From 8c8384c7118c84195e76773492dc1ded212cbf5c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 25 Oct 2022 17:55:50 +1100 Subject: [PATCH] fmp4: add support for muxing VP9 streams in cmaf, dash and iso fmp4 As specified in https://www.webmproject.org/vp9/mp4/ --- docs/plugins/gst_plugins_cache.json | 6 +-- mux/fmp4/src/fmp4mux/boxes.rs | 76 +++++++++++++++++++++++++++-- mux/fmp4/src/fmp4mux/imp.rs | 25 ++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 4ccaced5..20cba100 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -1587,7 +1587,7 @@ "long-name": "CMAFMux", "pad-templates": { "sink": { - "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", "presence": "always" }, @@ -1615,7 +1615,7 @@ "long-name": "DASHMP4Mux", "pad-templates": { "sink": { - "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", "presence": "always" }, @@ -1643,7 +1643,7 @@ "long-name": "ISOFMP4Mux", "pad-templates": { "sink_%%u": { - "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", "presence": "request" }, diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index 9b69fb36..6cc9e86e 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -731,7 +731,9 @@ fn write_hdlr( let s = caps.structure(0).unwrap(); let (handler_type, name) = match s.name() { - "video/x-h264" | "video/x-h265" | "image/jpeg" => (b"vide", b"VideoHandler\0".as_slice()), + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { + (b"vide", b"VideoHandler\0".as_slice()) + } "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { (b"soun", b"SoundHandler\0".as_slice()) } @@ -759,7 +761,7 @@ fn write_minf( let s = caps.structure(0).unwrap(); match s.name() { - "video/x-h264" | "video/x-h265" | "image/jpeg" => { + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { // Flags are always 1 for unspecified reasons write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))? } @@ -874,7 +876,9 @@ fn write_stsd( let s = caps.structure(0).unwrap(); match s.name() { - "video/x-h264" | "video/x-h265" | "image/jpeg" => write_visual_sample_entry(v, cfg, caps)?, + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { + write_visual_sample_entry(v, cfg, caps)? + } "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { write_audio_sample_entry(v, cfg, caps)? } @@ -925,6 +929,7 @@ fn write_visual_sample_entry( } } "image/jpeg" => b"jpeg", + "video/x-vp9" => b"vp09", _ => unreachable!(), }; @@ -993,6 +998,69 @@ fn write_visual_sample_entry( Ok(()) })?; } + "video/x-vp9" => { + let profile: u8 = match s.get::<&str>("profile").expect("no vp9 profile") { + "0" => Some(0), + "1" => Some(1), + "2" => Some(2), + "3" => Some(3), + _ => None, + } + .context("unsupported vp9 profile")?; + let colorimetry = gst_video::VideoColorimetry::from_str( + s.get::<&str>("colorimetry").expect("no colorimetry"), + ) + .context("failed to parse colorimetry")?; + let video_full_range = + colorimetry.range() == gst_video::VideoColorRange::Range0_255; + let chroma_format: u8 = + match s.get::<&str>("chroma-format").expect("no chroma-format") { + "4:2:0" => + // chroma-site is optional + { + match s + .get::<&str>("chroma-site") + .ok() + .and_then(|cs| gst_video::VideoChromaSite::from_str(cs).ok()) + { + Some(gst_video::VideoChromaSite::V_COSITED) => Some(0), + // COSITED + _ => Some(1), + } + } + "4:2:2" => Some(2), + "4:4:4" => Some(3), + _ => None, + } + .context("unsupported chroma-format")?; + let bit_depth: u8 = { + let bit_depth_luma = s.get::("bit-depth-luma").expect("no bit-depth-luma"); + let bit_depth_chroma = s + .get::("bit-depth-chroma") + .expect("no bit-depth-chroma"); + if bit_depth_luma != bit_depth_chroma { + return Err(anyhow!("bit-depth-luma and bit-depth-chroma have different values which is an unsupported configuration")); + } + bit_depth_luma as u8 + }; + write_full_box(v, b"vpcC", 1, 0, move |v| { + v.push(profile); + // XXX: hardcoded level 1 + v.push(10); + let mut byte: u8 = 0; + byte |= (bit_depth & 0xF) << 4; + byte |= (chroma_format & 0x7) << 1; + byte |= video_full_range as u8; + v.push(byte); + v.push(colorimetry.primaries().to_iso() as u8); + v.push(colorimetry.transfer().to_iso() as u8); + v.push(colorimetry.matrix().to_iso() as u8); + // 16-bit length field for codec initialization, unused + v.push(0); + v.push(0); + Ok(()) + })?; + } "image/jpeg" => { // Nothing to do here } @@ -1977,7 +2045,7 @@ pub(crate) fn create_mfra( } // Copy from std while this is still nightly-only -use std::fmt; +use std::{fmt, str::FromStr}; /// An iterator over slice in (non-overlapping) chunks separated by a predicate. /// diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index 7292e030..8f89633b 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -1454,6 +1454,7 @@ impl FMP4Mux { return Err(gst::FlowError::NotNegotiated); } } + "video/x-vp9" => (), "image/jpeg" => { intra_only = true; } @@ -2299,6 +2300,14 @@ impl ElementImpl for ISOFMP4Mux { .field("width", gst::IntRange::new(1, u16::MAX as i32)) .field("height", gst::IntRange::new(1, u16::MAX as i32)) .build(), + gst::Structure::builder("video/x-vp9") + .field("profile", gst::List::new(["0", "1", "2", "3"])) + .field("chroma-format", gst::List::new(["4:2:0", "4:2:2", "4:4:4"])) + .field("bit-depth-luma", gst::List::new([8u32, 10u32, 12u32])) + .field("bit-depth-chroma", gst::List::new([8u32, 10u32, 12u32])) + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), gst::Structure::builder("audio/mpeg") .field("mpegversion", 4i32) .field("stream-format", "raw") @@ -2381,6 +2390,14 @@ impl ElementImpl for CMAFMux { .field("width", gst::IntRange::new(1, u16::MAX as i32)) .field("height", gst::IntRange::new(1, u16::MAX as i32)) .build(), + gst::Structure::builder("video/x-vp9") + .field("profile", gst::List::new(["0", "1", "2", "3"])) + .field("chroma-format", gst::List::new(["4:2:0", "4:2:2", "4:4:4"])) + .field("bit-depth-luma", gst::List::new([8u32, 10u32, 12u32])) + .field("bit-depth-chroma", gst::List::new([8u32, 10u32, 12u32])) + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), gst::Structure::builder("audio/mpeg") .field("mpegversion", 4i32) .field("stream-format", "raw") @@ -2463,6 +2480,14 @@ impl ElementImpl for DASHMP4Mux { .field("width", gst::IntRange::::new(1, u16::MAX as i32)) .field("height", gst::IntRange::::new(1, u16::MAX as i32)) .build(), + gst::Structure::builder("video/x-vp9") + .field("profile", gst::List::new(["0", "1", "2", "3"])) + .field("chroma-format", gst::List::new(["4:2:0", "4:2:2", "4:4:4"])) + .field("bit-depth-luma", gst::List::new([8u32, 10u32, 12u32])) + .field("bit-depth-chroma", gst::List::new([8u32, 10u32, 12u32])) + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), gst::Structure::builder("audio/mpeg") .field("mpegversion", 4i32) .field("stream-format", "raw")