From f062b7cf0da027f19826c5e87ea27212c046c281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 7 Nov 2022 19:47:31 +0200 Subject: [PATCH] fmp4mux: Make media/trak timescales configurable And refactor a bit of code for easier extensibility. --- docs/plugins/gst_plugins_cache.json | 53 ++++++- mux/fmp4/Cargo.toml | 1 + mux/fmp4/src/fmp4mux/boxes.rs | 177 +++++++++++---------- mux/fmp4/src/fmp4mux/imp.rs | 238 ++++++++++++++++++++-------- mux/fmp4/src/fmp4mux/mod.rs | 54 ++++++- 5 files changed, 363 insertions(+), 160 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 4ebd6c25..2e79a5f3 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -1589,7 +1589,8 @@ "sink": { "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", - "presence": "always" + "presence": "always", + "type": "GstFMP4MuxPad" }, "src": { "caps": "video/quicktime:\n variant: cmaf\n", @@ -1617,7 +1618,8 @@ "sink": { "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", - "presence": "always" + "presence": "always", + "type": "GstFMP4MuxPad" }, "src": { "caps": "video/quicktime:\n variant: iso-fragmented\n", @@ -1645,7 +1647,8 @@ "sink_%%u": { "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", - "presence": "request" + "presence": "request", + "type": "GstFMP4MuxPad" }, "src": { "caps": "video/quicktime:\n variant: iso-fragmented\n", @@ -1673,7 +1676,8 @@ "sink_%%u": { "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nimage/jpeg:\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-alaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-mulaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-adpcm:\n layout: g726\n channels: 1\n rate: 8000\n bitrate: { (int)16000, (int)24000, (int)32000, (int)40000 }\napplication/x-onvif-metadata:\n parsed: true\n", "direction": "sink", - "presence": "request" + "presence": "request", + "type": "GstFMP4MuxPad" }, "src": { "caps": "video/quicktime:\n variant: iso-fragmented\n", @@ -1752,6 +1756,20 @@ "type": "guint64", "writable": true }, + "movie-timescale": { + "blurb": "Timescale to use for the movie (units per second, 0 is automatic)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint", + "writable": true + }, "write-mehd": { "blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)", "conditionally-available": false, @@ -1797,6 +1815,33 @@ "value": "2" } ] + }, + "GstFMP4MuxPad": { + "hierarchy": [ + "GstFMP4MuxPad", + "GstAggregatorPad", + "GstPad", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "trak-timescale": { + "blurb": "Timescale to use for the track (units per second, 0 is automatic)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint", + "writable": true + } + } } }, "package": "gst-plugin-fmp4", diff --git a/mux/fmp4/Cargo.toml b/mux/fmp4/Cargo.toml index 7de120c1..fb60b4c9 100644 --- a/mux/fmp4/Cargo.toml +++ b/mux/fmp4/Cargo.toml @@ -37,6 +37,7 @@ default = ["v1_18"] static = [] capi = [] v1_18 = ["gst-video/v1_18"] +doc = ["gst/v1_18"] [package.metadata.capi] min_version = "0.8.0" diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index 23546a35..b0a2ba01 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -355,7 +355,8 @@ fn brands_from_variant_and_caps<'a>( pub(super) fn create_fmp4_header(cfg: super::HeaderConfiguration) -> Result { let mut v = vec![]; - let (brand, compatible_brands) = brands_from_variant_and_caps(cfg.variant, cfg.streams.iter()); + let (brand, compatible_brands) = + brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.caps)); write_box(&mut v, b"ftyp", |v| { // major brand @@ -420,17 +421,17 @@ fn write_moov(v: &mut Vec, cfg: &super::HeaderConfiguration) -> Result<(), E write_full_box(v, b"mvhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { write_mvhd(v, cfg, creation_time) })?; - for (idx, caps) in cfg.streams.iter().enumerate() { + for (idx, stream) in cfg.streams.iter().enumerate() { write_box(v, b"trak", |v| { let mut references = vec![]; // Reference the video track for ONVIF metadata tracks if cfg.variant == super::Variant::ONVIF - && caps.structure(0).unwrap().name() == "application/x-onvif-metadata" + && stream.caps.structure(0).unwrap().name() == "application/x-onvif-metadata" { // Find the first video track - for (idx, caps) in cfg.streams.iter().enumerate() { - let s = caps.structure(0).unwrap(); + for (idx, other_stream) in cfg.streams.iter().enumerate() { + let s = other_stream.caps.structure(0).unwrap(); if matches!(s.name(), "video/x-h264" | "video/x-h265" | "image/jpeg") { references.push(TrackReference { @@ -442,7 +443,7 @@ fn write_moov(v: &mut Vec, cfg: &super::HeaderConfiguration) -> Result<(), E } } - write_trak(v, cfg, idx, caps, creation_time, &references) + write_trak(v, cfg, idx, stream, creation_time, &references) })?; } write_box(v, b"mvex", |v| write_mvex(v, cfg))?; @@ -480,6 +481,31 @@ fn caps_to_timescale(caps: &gst::CapsRef) -> u32 { } } +fn header_stream_to_timescale(stream: &super::HeaderStream) -> u32 { + if stream.trak_timescale > 0 { + stream.trak_timescale + } else { + caps_to_timescale(&stream.caps) + } +} + +fn header_configuration_to_timescale(cfg: &super::HeaderConfiguration) -> u32 { + if cfg.movie_timescale > 0 { + cfg.movie_timescale + } else { + // Use the reference track timescale + header_stream_to_timescale(&cfg.streams[0]) + } +} + +fn fragment_header_stream_to_timescale(stream: &super::FragmentHeaderStream) -> u32 { + if stream.trak_timescale > 0 { + stream.trak_timescale + } else { + caps_to_timescale(&stream.caps) + } +} + fn write_mvhd( v: &mut Vec, cfg: &super::HeaderConfiguration, @@ -489,8 +515,8 @@ fn write_mvhd( v.extend(creation_time.to_be_bytes()); // Modification time v.extend(creation_time.to_be_bytes()); - // Timescale: uses the reference track timescale - v.extend(caps_to_timescale(&cfg.streams[0]).to_be_bytes()); + // Timescale + v.extend(header_configuration_to_timescale(cfg).to_be_bytes()); // Duration v.extend(0u64.to_be_bytes()); @@ -540,7 +566,7 @@ fn write_trak( v: &mut Vec, cfg: &super::HeaderConfiguration, idx: usize, - caps: &gst::CapsRef, + stream: &super::HeaderStream, creation_time: u64, references: &[TrackReference], ) -> Result<(), Error> { @@ -549,13 +575,13 @@ fn write_trak( b"tkhd", FULL_BOX_VERSION_1, TKHD_FLAGS_TRACK_ENABLED | TKHD_FLAGS_TRACK_IN_MOVIE | TKHD_FLAGS_TRACK_IN_PREVIEW, - |v| write_tkhd(v, cfg, idx, caps, creation_time), + |v| write_tkhd(v, cfg, idx, stream, creation_time), )?; // TODO: write edts if necessary: for audio tracks to remove initialization samples // TODO: write edts optionally for negative DTS instead of offsetting the DTS - write_box(v, b"mdia", |v| write_mdia(v, cfg, caps, creation_time))?; + write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?; if !references.is_empty() { write_box(v, b"tref", |v| write_tref(v, cfg, references))?; @@ -568,7 +594,7 @@ fn write_tkhd( v: &mut Vec, _cfg: &super::HeaderConfiguration, idx: usize, - caps: &gst::CapsRef, + stream: &super::HeaderStream, creation_time: u64, ) -> Result<(), Error> { // Creation time @@ -591,7 +617,7 @@ fn write_tkhd( v.extend(0u16.to_be_bytes()); // Volume - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); match s.name() { "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { v.extend((1u16 << 8).to_be_bytes()) @@ -650,19 +676,19 @@ fn write_tkhd( fn write_mdia( v: &mut Vec, cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, creation_time: u64, ) -> Result<(), Error> { write_full_box(v, b"mdhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { - write_mdhd(v, cfg, caps, creation_time) + write_mdhd(v, cfg, stream, creation_time) })?; write_full_box(v, b"hdlr", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { - write_hdlr(v, cfg, caps) + write_hdlr(v, cfg, stream) })?; // TODO: write elng if needed - write_box(v, b"minf", |v| write_minf(v, cfg, caps))?; + write_box(v, b"minf", |v| write_minf(v, cfg, stream))?; Ok(()) } @@ -699,7 +725,7 @@ fn language_code(lang: impl std::borrow::Borrow<[u8; 3]>) -> u16 { fn write_mdhd( v: &mut Vec, _cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, creation_time: u64, ) -> Result<(), Error> { // Creation time @@ -707,7 +733,7 @@ fn write_mdhd( // Modification time v.extend(creation_time.to_be_bytes()); // Timescale - v.extend(caps_to_timescale(caps).to_be_bytes()); + v.extend(header_stream_to_timescale(stream).to_be_bytes()); // Duration v.extend(0u64.to_be_bytes()); @@ -724,12 +750,12 @@ fn write_mdhd( fn write_hdlr( v: &mut Vec, _cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { // Pre-defined v.extend([0u8; 4]); - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); let (handler_type, name) = match s.name() { "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { (b"vide", b"VideoHandler\0".as_slice()) @@ -756,9 +782,9 @@ fn write_hdlr( fn write_minf( v: &mut Vec, cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); match s.name() { "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { @@ -780,7 +806,7 @@ fn write_minf( write_box(v, b"dinf", |v| write_dinf(v, cfg))?; - write_box(v, b"stbl", |v| write_stbl(v, cfg, caps))?; + write_box(v, b"stbl", |v| write_stbl(v, cfg, stream))?; Ok(()) } @@ -833,10 +859,10 @@ fn write_dref(v: &mut Vec, _cfg: &super::HeaderConfiguration) -> Result<(), fn write_stbl( v: &mut Vec, cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { write_full_box(v, b"stsd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { - write_stsd(v, cfg, caps) + write_stsd(v, cfg, stream) })?; write_full_box(v, b"stts", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_stts(v, cfg) @@ -853,14 +879,10 @@ fn write_stbl( })?; // For video write a sync sample box as indication that not all samples are sync samples - let s = caps.structure(0).unwrap(); - match s.name() { - "video/x-h264" | "video/x-h265" | "video/x-vp9" => { - write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { - write_stss(v, cfg) - })? - } - _ => (), + if !stream.delta_frames.intra_only() { + write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stss(v, cfg) + })? } Ok(()) @@ -869,20 +891,20 @@ fn write_stbl( fn write_stsd( v: &mut Vec, cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { // Entry count v.extend(1u32.to_be_bytes()); - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); match s.name() { "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { - write_visual_sample_entry(v, cfg, caps)? + write_visual_sample_entry(v, cfg, stream)? } "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { - write_audio_sample_entry(v, cfg, caps)? + write_audio_sample_entry(v, cfg, stream)? } - "application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, caps)?, + "application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, stream)?, _ => unreachable!(), } @@ -908,9 +930,9 @@ fn write_sample_entry_box) -> Result>( fn write_visual_sample_entry( v: &mut Vec, _cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); let fourcc = match s.name() { "video/x-h264" => { let stream_format = s.get::<&str>("stream-format").context("no stream-format")?; @@ -1146,7 +1168,7 @@ fn write_visual_sample_entry( #[cfg(feature = "v1_18")] { - if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(caps) { + if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(&stream.caps) { write_box(v, b"clli", move |v| { v.extend((cll.max_content_light_level() as u16).to_be_bytes()); v.extend((cll.max_frame_average_light_level() as u16).to_be_bytes()); @@ -1154,7 +1176,7 @@ fn write_visual_sample_entry( })?; } - if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(caps) { + if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(&stream.caps) { write_box(v, b"mdcv", move |v| { for primary in mastering.display_primaries() { v.extend(primary.x.to_be_bytes()); @@ -1211,9 +1233,9 @@ fn write_visual_sample_entry( fn write_audio_sample_entry( v: &mut Vec, _cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); let fourcc = match s.name() { "audio/mpeg" => b"mp4a", "audio/x-opus" => b"Opus", @@ -1275,7 +1297,7 @@ fn write_audio_sample_entry( write_esds_aac(v, &map)?; } "audio/x-opus" => { - write_dops(v, caps)?; + write_dops(v, &stream.caps)?; } "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { // Nothing to do here @@ -1412,7 +1434,7 @@ fn write_esds_aac(v: &mut Vec, codec_data: &[u8]) -> Result<(), Error> { ) } -fn write_dops(v: &mut Vec, caps: &gst::CapsRef) -> Result<(), Error> { +fn write_dops(v: &mut Vec, caps: &gst::Caps) -> Result<(), Error> { let rate; let channels; let channel_mapping_family; @@ -1442,11 +1464,6 @@ fn write_dops(v: &mut Vec, caps: &gst::CapsRef) -> Result<(), Error> { ) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping)) .unwrap(); } else { - // FIXME: Workaround for below function taking a &Caps instead of &CapsRef - // SAFETY: This is OK because we only get an immutable reference and don't - // clone it, so nobody will be able to get a mutable reference to the caps. - let caps = unsafe { &*(&caps as *const &gst::CapsRef as *const gst::Caps) }; - ( rate, channels, @@ -1479,9 +1496,9 @@ fn write_dops(v: &mut Vec, caps: &gst::CapsRef) -> Result<(), Error> { fn write_xml_meta_data_sample_entry( v: &mut Vec, _cfg: &super::HeaderConfiguration, - caps: &gst::CapsRef, + stream: &super::HeaderStream, ) -> Result<(), Error> { - let s = caps.structure(0).unwrap(); + let s = stream.caps.structure(0).unwrap(); let namespace = match s.name() { "application/x-onvif-metadata" => b"http://www.onvif.org/ver10/schema", _ => unreachable!(), @@ -1560,7 +1577,7 @@ fn write_mvex(v: &mut Vec, cfg: &super::HeaderConfiguration) -> Result<(), E } } - for (idx, _caps) in cfg.streams.iter().enumerate() { + for (idx, _stream) in cfg.streams.iter().enumerate() { write_full_box(v, b"trex", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_trex(v, cfg, idx) })?; @@ -1571,7 +1588,7 @@ fn write_mvex(v: &mut Vec, cfg: &super::HeaderConfiguration) -> Result<(), E fn write_mehd(v: &mut Vec, cfg: &super::HeaderConfiguration) -> Result<(), Error> { // Use the reference track timescale - let timescale = caps_to_timescale(&cfg.streams[0]); + let timescale = header_configuration_to_timescale(cfg); let duration = cfg .duration @@ -1614,7 +1631,7 @@ pub(super) fn create_fmp4_fragment_header( let mut v = vec![]; let (brand, compatible_brands) = - brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.0)); + brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.caps)); write_box(&mut v, b"styp", |v| { // major brand @@ -1665,15 +1682,14 @@ fn write_moof( })?; let mut data_offset_offsets = vec![]; - for (idx, (caps, timing_info)) in cfg.streams.iter().enumerate() { + for (idx, stream) in cfg.streams.iter().enumerate() { // Skip tracks without any buffers for this fragment. - let timing_info = match timing_info { - None => continue, - Some(ref timing_info) => timing_info, - }; + if stream.start_time.is_none() { + continue; + } write_box(v, b"traf", |v| { - write_traf(v, cfg, &mut data_offset_offsets, idx, caps, timing_info) + write_traf(v, cfg, &mut data_offset_offsets, idx, stream) })?; } @@ -1688,11 +1704,8 @@ fn write_mfhd(v: &mut Vec, cfg: &super::FragmentHeaderConfiguration) -> Resu #[allow(clippy::identity_op)] #[allow(clippy::bool_to_int_with_if)] -fn sample_flags_from_buffer( - timing_info: &super::FragmentTimingInfo, - buffer: &gst::BufferRef, -) -> u32 { - if timing_info.delta_frames.intra_only() { +fn sample_flags_from_buffer(stream: &super::FragmentHeaderStream, buffer: &gst::BufferRef) -> u32 { + if stream.delta_frames.intra_only() { (0b00u32 << (16 + 10)) | // leading: unknown (0b10u32 << (16 + 8)) | // depends: no (0b10u32 << (16 + 6)) | // depended: no @@ -1743,7 +1756,7 @@ const SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT: u32 = 0x8_00; fn analyze_buffers( cfg: &super::FragmentHeaderConfiguration, idx: usize, - timing_info: &super::FragmentTimingInfo, + stream: &super::FragmentHeaderStream, timescale: u32, ) -> Result< ( @@ -1802,7 +1815,7 @@ fn analyze_buffers( tr_flags |= SAMPLE_DURATION_PRESENT; } - let f = sample_flags_from_buffer(timing_info, buffer); + let f = sample_flags_from_buffer(stream, buffer); if first_buffer_flags.is_none() { first_buffer_flags = Some(f); } else { @@ -1818,7 +1831,7 @@ fn analyze_buffers( } if let Some(composition_time_offset) = *composition_time_offset { - assert!(timing_info.delta_frames.requires_dts()); + assert!(stream.delta_frames.requires_dts()); if composition_time_offset != 0 { tr_flags |= SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT; } @@ -1869,10 +1882,9 @@ fn write_traf( cfg: &super::FragmentHeaderConfiguration, data_offset_offsets: &mut Vec, idx: usize, - caps: &gst::CapsRef, - timing_info: &super::FragmentTimingInfo, + stream: &super::FragmentHeaderStream, ) -> Result<(), Error> { - let timescale = caps_to_timescale(caps); + let timescale = fragment_header_stream_to_timescale(stream); // Analyze all buffers to know what values can be put into the tfhd for all samples and what // has to be stored for every single sample @@ -1883,7 +1895,7 @@ fn write_traf( default_duration, default_flags, negative_composition_time_offsets, - ) = analyze_buffers(cfg, idx, timing_info, timescale)?; + ) = analyze_buffers(cfg, idx, stream, timescale)?; assert!((tf_flags & DEFAULT_SAMPLE_SIZE_PRESENT == 0) ^ default_size.is_some()); assert!((tf_flags & DEFAULT_SAMPLE_DURATION_PRESENT == 0) ^ default_duration.is_some()); @@ -1893,7 +1905,7 @@ fn write_traf( write_tfhd(v, cfg, idx, default_size, default_duration, default_flags) })?; write_full_box(v, b"tfdt", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { - write_tfdt(v, cfg, idx, timing_info, timescale) + write_tfdt(v, cfg, idx, stream, timescale) })?; let mut current_data_offset = 0; @@ -1923,7 +1935,7 @@ fn write_traf( current_data_offset, tr_flags, timescale, - timing_info, + stream, run, ) }, @@ -1973,11 +1985,12 @@ fn write_tfdt( v: &mut Vec, _cfg: &super::FragmentHeaderConfiguration, _idx: usize, - timing_info: &super::FragmentTimingInfo, + stream: &super::FragmentHeaderStream, timescale: u32, ) -> Result<(), Error> { - let base_time = timing_info + let base_time = stream .start_time + .unwrap() .mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds()) .context("base time overflow")?; @@ -1993,7 +2006,7 @@ fn write_trun( current_data_offset: u32, tr_flags: u32, timescale: u32, - timing_info: &super::FragmentTimingInfo, + stream: &super::FragmentHeaderStream, buffers: &[Buffer], ) -> Result { // Sample count @@ -2004,7 +2017,7 @@ fn write_trun( v.extend(current_data_offset.to_be_bytes()); if (tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) != 0 { - v.extend(sample_flags_from_buffer(timing_info, &buffers[0].buffer).to_be_bytes()); + v.extend(sample_flags_from_buffer(stream, &buffers[0].buffer).to_be_bytes()); } for Buffer { @@ -2036,7 +2049,7 @@ fn write_trun( assert!((tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) == 0); // Sample flags - v.extend(sample_flags_from_buffer(timing_info, buffer).to_be_bytes()); + v.extend(sample_flags_from_buffer(stream, buffer).to_be_bytes()); } if (tr_flags & SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT) != 0 { diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index b0eb3628..8c3ede38 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -76,6 +76,7 @@ struct Settings { write_mehd: bool, interleave_bytes: Option, interleave_time: Option, + movie_timescale: u32, } impl Default for Settings { @@ -87,6 +88,7 @@ impl Default for Settings { write_mehd: DEFAULT_WRITE_MEHD, interleave_bytes: DEFAULT_INTERLEAVE_BYTES, interleave_time: DEFAULT_INTERLEAVE_TIME, + movie_timescale: 0, } } } @@ -122,7 +124,7 @@ struct Gop { } struct Stream { - sinkpad: gst_base::AggregatorPad, + sinkpad: super::FMP4MuxPad, caps: gst::Caps, delta_frames: DeltaFrames, @@ -617,11 +619,7 @@ impl FMP4Mux { ) -> Result< ( // Drained streams - Vec<( - gst::Caps, - Option, - VecDeque, - )>, + Vec<(super::FragmentHeaderStream, VecDeque)>, // Minimum earliest PTS position of all streams Option, // Minimum earliest PTS of all streams @@ -658,6 +656,8 @@ impl FMP4Mux { ); for (idx, stream) in state.streams.iter_mut().enumerate() { + let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone(); + assert!( timeout || at_eos @@ -742,7 +742,16 @@ impl FMP4Mux { "Draining no buffers", ); - drained_streams.push((stream.caps.clone(), None, VecDeque::new())); + drained_streams.push(( + super::FragmentHeaderStream { + caps: stream.caps.clone(), + start_time: None, + delta_frames: stream.delta_frames, + trak_timescale: stream_settings.trak_timescale, + }, + VecDeque::new(), + )); + continue; } @@ -876,11 +885,12 @@ impl FMP4Mux { } drained_streams.push(( - stream.caps.clone(), - Some(super::FragmentTimingInfo { - start_time, + super::FragmentHeaderStream { + caps: stream.caps.clone(), + start_time: Some(start_time), delta_frames: stream.delta_frames, - }), + trak_timescale: stream_settings.trak_timescale, + }, buffers, )); } @@ -897,11 +907,7 @@ impl FMP4Mux { fn preprocess_drained_streams_onvif( &self, state: &mut State, - drained_streams: &mut [( - gst::Caps, - Option, - VecDeque, - )], + drained_streams: &mut [(super::FragmentHeaderStream, VecDeque)], ) -> Result, gst::FlowError> { let aggregator = self.obj(); if aggregator.class().as_ref().variant != super::Variant::ONVIF { @@ -925,7 +931,7 @@ impl FMP4Mux { // If this is the first fragment then allow the first buffers to not have a reference // timestamp meta and backdate them if state.stream_header.is_none() { - for (idx, (_, _, drain_buffers)) in drained_streams.iter_mut().enumerate() { + for (idx, (_, drain_buffers)) in drained_streams.iter_mut().enumerate() { let (buffer_idx, utc_time, buffer) = match drain_buffers.iter().enumerate().find_map(|(idx, buffer)| { get_utc_time_from_buffer(&buffer.buffer) @@ -979,7 +985,7 @@ impl FMP4Mux { if state.start_utc_time.is_none() { let mut start_utc_time = None; - for (idx, (_, _, drain_buffers)) in drained_streams.iter().enumerate() { + for (idx, (_, drain_buffers)) in drained_streams.iter().enumerate() { for buffer in drain_buffers { let utc_time = match get_utc_time_from_buffer(&buffer.buffer) { None => { @@ -1010,7 +1016,7 @@ impl FMP4Mux { // Update all buffer timestamps based on the UTC time and offset to the start UTC time let start_utc_time = state.start_utc_time.unwrap(); - for (idx, (_, timing_info, drain_buffers)) in drained_streams.iter_mut().enumerate() { + for (idx, (stream, drain_buffers)) in drained_streams.iter_mut().enumerate() { let mut start_time = None; for buffer in drain_buffers.iter_mut() { @@ -1128,9 +1134,9 @@ impl FMP4Mux { if let Some(start_time) = start_time { gst::debug!(CAT, obj: state.streams[idx].sinkpad, "Fragment starting at UTC time {}", start_time); - timing_info.as_mut().unwrap().start_time = start_time; + *stream.start_time.as_mut().unwrap() = start_time; } else { - assert!(timing_info.is_none()); + assert!(stream.start_time.is_none()); } } @@ -1141,35 +1147,28 @@ impl FMP4Mux { fn interleave_buffers( &self, settings: &Settings, - mut drained_streams: Vec<( - gst::Caps, - Option, - VecDeque, - )>, - ) -> Result< - ( - Vec, - Vec<(gst::Caps, Option)>, - ), - gst::FlowError, - > { + mut drained_streams: Vec<(super::FragmentHeaderStream, VecDeque)>, + ) -> Result<(Vec, Vec), gst::FlowError> { let mut interleaved_buffers = - Vec::with_capacity(drained_streams.iter().map(|(_, _, bufs)| bufs.len()).sum()); - while let Some((_idx, (_, _, bufs))) = drained_streams.iter_mut().enumerate().min_by( - |(a_idx, (_, _, a)), (b_idx, (_, _, b))| { - let (a, b) = match (a.front(), b.front()) { - (None, None) => return std::cmp::Ordering::Equal, - (None, _) => return std::cmp::Ordering::Greater, - (_, None) => return std::cmp::Ordering::Less, - (Some(a), Some(b)) => (a, b), - }; + Vec::with_capacity(drained_streams.iter().map(|(_, bufs)| bufs.len()).sum()); + while let Some((_idx, (_, bufs))) = + drained_streams + .iter_mut() + .enumerate() + .min_by(|(a_idx, (_, a)), (b_idx, (_, b))| { + let (a, b) = match (a.front(), b.front()) { + (None, None) => return std::cmp::Ordering::Equal, + (None, _) => return std::cmp::Ordering::Greater, + (_, None) => return std::cmp::Ordering::Less, + (Some(a), Some(b)) => (a, b), + }; - match a.timestamp.cmp(&b.timestamp) { - std::cmp::Ordering::Equal => a_idx.cmp(b_idx), - cmp => cmp, - } - }, - ) { + match a.timestamp.cmp(&b.timestamp) { + std::cmp::Ordering::Equal => a_idx.cmp(b_idx), + cmp => cmp, + } + }) + { let start_time = match bufs.front() { None => { // No more buffers now @@ -1201,11 +1200,11 @@ impl FMP4Mux { } // All buffers should be consumed now - assert!(drained_streams.iter().all(|(_, _, bufs)| bufs.is_empty())); + assert!(drained_streams.iter().all(|(_, bufs)| bufs.is_empty())); let streams = drained_streams .into_iter() - .map(|(caps, timing_info, _)| (caps, timing_info)) + .map(|(stream, _)| stream) .collect::>(); Ok((interleaved_buffers, streams)) @@ -1217,7 +1216,7 @@ impl FMP4Mux { settings: &Settings, timeout: bool, at_eos: bool, - upstream_events: &mut Vec<(gst_base::AggregatorPad, gst::Event)>, + upstream_events: &mut Vec<(super::FMP4MuxPad, gst::Event)>, ) -> Result<(Option, Option), gst::FlowError> { if at_eos { gst::info!(CAT, imp: self, "Draining at EOS"); @@ -1241,7 +1240,7 @@ impl FMP4Mux { ) = self.drain_buffers(state, settings, timeout, at_eos)?; // Remove all GAP buffers before processing them further - for (_, timing_info, buffers) in &mut drained_streams { + for (stream, buffers) in &mut drained_streams { buffers.retain(|buf| { !buf.buffer.flags().contains(gst::BufferFlags::GAP) || !buf.buffer.flags().contains(gst::BufferFlags::DROPPABLE) @@ -1249,7 +1248,7 @@ impl FMP4Mux { }); if buffers.is_empty() { - *timing_info = None; + stream.start_time = None; } } @@ -1371,9 +1370,13 @@ impl FMP4Mux { // Write mfra only for the main stream, and if there are no buffers for the main stream // in this segment then don't write anything. - if let Some((_caps, Some(ref timing_info))) = streams.get(0) { + if let Some(super::FragmentHeaderStream { + start_time: Some(start_time), + .. + }) = streams.get(0) + { state.fragment_offsets.push(super::FragmentOffset { - time: timing_info.start_time, + time: *start_time, offset: moof_offset, }); } @@ -1432,7 +1435,7 @@ impl FMP4Mux { if settings.write_mfra && at_eos { gst::debug!(CAT, imp: self, "Writing mfra box"); - match boxes::create_mfra(&streams[0].0, &state.fragment_offsets) { + match boxes::create_mfra(&streams[0].caps, &state.fragment_offsets) { Ok(mut mfra) => { { let mfra = mfra.get_mut().unwrap(); @@ -1462,7 +1465,7 @@ impl FMP4Mux { .obj() .sink_pads() .into_iter() - .map(|pad| pad.downcast::().unwrap()) + .map(|pad| pad.downcast::().unwrap()) { let caps = match pad.current_caps() { Some(caps) => caps, @@ -1599,13 +1602,18 @@ impl FMP4Mux { let streams = state .streams .iter() - .map(|s| s.caps.clone()) + .map(|s| super::HeaderStream { + trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale, + delta_frames: s.delta_frames, + caps: s.caps.clone(), + }) .collect::>(); let mut buffer = boxes::create_fmp4_header(super::HeaderConfiguration { variant, update: at_eos, - streams: streams.as_slice(), + movie_timescale: settings.movie_timescale, + streams, write_mehd: settings.write_mehd, duration: if at_eos { duration } else { None }, start_utc_time: state @@ -1696,6 +1704,11 @@ impl ObjectImpl for FMP4Mux { .default_value(DEFAULT_INTERLEAVE_TIME.map(gst::ClockTime::nseconds).unwrap_or(u64::MAX)) .mutable_ready() .build(), + glib::ParamSpecUInt::builder("movie-timescale") + .nick("Movie Timescale") + .blurb("Timescale to use for the movie (units per second, 0 is automatic)") + .mutable_ready() + .build(), ] }); @@ -1745,6 +1758,11 @@ impl ObjectImpl for FMP4Mux { }; } + "movie-timescale" => { + let mut settings = self.settings.lock().unwrap(); + settings.movie_timescale = value.get().expect("type checked upstream"); + } + _ => unimplemented!(), } } @@ -1781,6 +1799,11 @@ impl ObjectImpl for FMP4Mux { settings.interleave_time.to_value() } + "movie-timescale" => { + let settings = self.settings.lock().unwrap(); + settings.movie_timescale.to_value() + } + _ => unimplemented!(), } } @@ -1954,8 +1977,6 @@ impl AggregatorImpl for FMP4Mux { } fn flush(&self) -> Result { - self.parent_flush()?; - let mut state = self.state.lock().unwrap(); for stream in &mut state.streams { @@ -1969,7 +1990,9 @@ impl AggregatorImpl for FMP4Mux { state.current_offset = 0; state.fragment_offsets.clear(); - Ok(gst::FlowSuccess::Ok) + drop(state); + + self.parent_flush() } fn stop(&self) -> Result<(), gst::ErrorMessage> { @@ -2346,7 +2369,7 @@ impl ElementImpl for ISOFMP4Mux { ) .unwrap(); - let sink_pad_template = gst::PadTemplate::new( + let sink_pad_template = gst::PadTemplate::with_gtype( "sink_%u", gst::PadDirection::Sink, gst::PadPresence::Request, @@ -2385,6 +2408,7 @@ impl ElementImpl for ISOFMP4Mux { ] .into_iter() .collect::(), + super::FMP4MuxPad::static_type(), ) .unwrap(); @@ -2441,7 +2465,7 @@ impl ElementImpl for CMAFMux { ) .unwrap(); - let sink_pad_template = gst::PadTemplate::new( + let sink_pad_template = gst::PadTemplate::with_gtype( "sink", gst::PadDirection::Sink, gst::PadPresence::Always, @@ -2467,6 +2491,7 @@ impl ElementImpl for CMAFMux { ] .into_iter() .collect::(), + super::FMP4MuxPad::static_type(), ) .unwrap(); @@ -2523,7 +2548,7 @@ impl ElementImpl for DASHMP4Mux { ) .unwrap(); - let sink_pad_template = gst::PadTemplate::new( + let sink_pad_template = gst::PadTemplate::with_gtype( "sink", gst::PadDirection::Sink, gst::PadPresence::Always, @@ -2562,6 +2587,7 @@ impl ElementImpl for DASHMP4Mux { ] .into_iter() .collect::(), + super::FMP4MuxPad::static_type(), ) .unwrap(); @@ -2618,7 +2644,7 @@ impl ElementImpl for ONVIFFMP4Mux { ) .unwrap(); - let sink_pad_template = gst::PadTemplate::new( + let sink_pad_template = gst::PadTemplate::with_gtype( "sink_%u", gst::PadDirection::Sink, gst::PadPresence::Request, @@ -2665,6 +2691,7 @@ impl ElementImpl for ONVIFFMP4Mux { ] .into_iter() .collect::(), + super::FMP4MuxPad::static_type(), ) .unwrap(); @@ -2680,3 +2707,82 @@ impl AggregatorImpl for ONVIFFMP4Mux {} impl FMP4MuxImpl for ONVIFFMP4Mux { const VARIANT: super::Variant = super::Variant::ONVIF; } + +#[derive(Default, Clone)] +struct PadSettings { + trak_timescale: u32, +} + +#[derive(Default)] +pub(crate) struct FMP4MuxPad { + settings: Mutex, +} + +#[glib::object_subclass] +impl ObjectSubclass for FMP4MuxPad { + const NAME: &'static str = "GstFMP4MuxPad"; + type Type = super::FMP4MuxPad; + type ParentType = gst_base::AggregatorPad; +} + +impl ObjectImpl for FMP4MuxPad { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![glib::ParamSpecUInt::builder("trak-timescale") + .nick("Track Timescale") + .blurb("Timescale to use for the track (units per second, 0 is automatic)") + .mutable_ready() + .build()] + }); + + &PROPERTIES + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "trak-timescale" => { + let mut settings = self.settings.lock().unwrap(); + settings.trak_timescale = value.get().expect("type checked upstream"); + } + + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "trak-timescale" => { + let settings = self.settings.lock().unwrap(); + settings.trak_timescale.to_value() + } + + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for FMP4MuxPad {} + +impl PadImpl for FMP4MuxPad {} + +impl AggregatorPadImpl for FMP4MuxPad { + fn flush(&self, aggregator: &gst_base::Aggregator) -> Result { + let mux = aggregator.downcast_ref::().unwrap(); + let mut mux_state = mux.imp().state.lock().unwrap(); + + for stream in &mut mux_state.streams { + if stream.sinkpad == *self.obj() { + stream.queued_gops.clear(); + stream.dts_offset = None; + stream.current_position = gst::ClockTime::ZERO; + stream.current_utc_time = gst::ClockTime::ZERO; + stream.fragment_filled = false; + break; + } + } + + drop(mux_state); + + self.parent_flush(aggregator) + } +} diff --git a/mux/fmp4/src/fmp4mux/mod.rs b/mux/fmp4/src/fmp4mux/mod.rs index 9973be71..396d8b81 100644 --- a/mux/fmp4/src/fmp4mux/mod.rs +++ b/mux/fmp4/src/fmp4mux/mod.rs @@ -12,6 +12,10 @@ use gst::prelude::*; mod boxes; mod imp; +glib::wrapper! { + pub(crate) struct FMP4MuxPad(ObjectSubclass) @extends gst_base::AggregatorPad, gst::Pad, gst::Object; +} + glib::wrapper! { pub(crate) struct FMP4Mux(ObjectSubclass) @extends gst_base::Aggregator, gst::Element, gst::Object; } @@ -33,8 +37,12 @@ glib::wrapper! { } pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { - FMP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); - HeaderUpdateMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + #[cfg(feature = "doc")] + { + FMP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + FMP4MuxPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + HeaderUpdateMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + } gst::Element::register( Some(plugin), "isofmp4mux", @@ -64,33 +72,63 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { } #[derive(Debug)] -pub(crate) struct HeaderConfiguration<'a> { +pub(crate) struct HeaderConfiguration { variant: Variant, update: bool, + + /// Pre-defined movie timescale if not 0. + movie_timescale: u32, + /// First caps must be the video/reference stream. Must be in the order the tracks are going to /// be used later for the fragments too. - streams: &'a [gst::Caps], + streams: Vec, + write_mehd: bool, duration: Option, + /// Start UTC time in ONVIF mode. /// Since Jan 1 1601 in 100ns units. start_utc_time: Option, } +#[derive(Debug)] +pub(crate) struct HeaderStream { + /// Caps of this stream + caps: gst::Caps, + + /// Set if this is an intra-only stream + delta_frames: DeltaFrames, + + /// Pre-defined trak timescale if not 0. + trak_timescale: u32, +} + #[derive(Debug)] pub(crate) struct FragmentHeaderConfiguration<'a> { variant: Variant, + + /// Sequence number for this fragment. sequence_number: u32, - streams: &'a [(gst::Caps, Option)], + + streams: &'a [FragmentHeaderStream], buffers: &'a [Buffer], } #[derive(Debug)] -pub(crate) struct FragmentTimingInfo { - /// Start time of this fragment - start_time: gst::ClockTime, +pub(crate) struct FragmentHeaderStream { + /// Caps of this stream + caps: gst::Caps, + /// Set if this is an intra-only stream delta_frames: DeltaFrames, + + /// Pre-defined trak timescale if not 0. + trak_timescale: u32, + + /// Start time of this fragment + /// + /// `None` if this stream has no buffers in this fragment. + start_time: Option, } #[derive(Debug, Copy, Clone)]