From c0e4e9c41b8f3041060cada81cad76c1b689f8f8 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 9 Jan 2024 08:15:02 -0300 Subject: [PATCH 1/4] fmp4mux: Add support to write edts to handle audio priming --- mux/fmp4/src/fmp4mux/boxes.rs | 37 +++++++- mux/fmp4/src/fmp4mux/imp.rs | 173 ++++++++++++++++++++++++++++++++-- mux/fmp4/src/fmp4mux/mod.rs | 9 ++ 3 files changed, 211 insertions(+), 8 deletions(-) diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index 55c8c520..4a2e3b9c 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -6,6 +6,7 @@ // // SPDX-License-Identifier: MPL-2.0 +use crate::fmp4mux::imp::CAT; use gst::prelude::*; use anyhow::{anyhow, bail, Context, Error}; @@ -547,6 +548,35 @@ struct TrackReference { track_ids: Vec, } +fn write_edts(v: &mut Vec, stream: &super::HeaderStream) -> Result<(), Error> { + write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?; + + Ok(()) +} + +fn write_elst(v: &mut Vec, stream: &super::HeaderStream) -> Result<(), Error> { + // Entry count + v.extend((stream.elst_infos.len() as u32).to_be_bytes()); + + for elst_info in &stream.elst_infos { + v.extend( + elst_info + .duration + .expect("Should have been set by `get_elst_infos`") + .to_be_bytes(), + ); + + // Media time + v.extend(elst_info.start.to_be_bytes()); + + // Media rate + v.extend(1u16.to_be_bytes()); + v.extend(0u16.to_be_bytes()); + } + + Ok(()) +} + fn write_trak( v: &mut Vec, cfg: &super::HeaderConfiguration, @@ -563,10 +593,13 @@ fn write_trak( |v| write_tkhd(v, cfg, idx, stream, creation_time), )?; - // TODO: write edts if necessary: for audio tracks to remove initialization samples // TODO: write edts optionally for negative DTS instead of offsetting the DTS - write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?; + if !stream.elst_infos.is_empty() { + if let Err(e) = write_edts(v, stream) { + gst::warning!(CAT, "Failed to write edts: {}", e); + } + } if !references.is_empty() { write_box(v, b"tref", |v| write_tref(v, cfg, references))?; diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index fc941468..8b8e98cd 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -12,6 +12,7 @@ use gst::subclass::prelude::*; use gst_base::prelude::*; use gst_base::subclass::prelude::*; +use anyhow::Context; use std::collections::VecDeque; use std::mem; use std::sync::Mutex; @@ -86,7 +87,7 @@ fn utc_time_to_running_time( .and_then(|res| res.positive()) } -static CAT: Lazy = Lazy::new(|| { +pub static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( "fmp4mux", gst::DebugColorFlags::empty(), @@ -222,6 +223,83 @@ struct Stream { /// Mapping between running time and UTC time in ONVIF mode. running_time_utc_time_mapping: Option<(gst::Signed, gst::ClockTime)>, + + /// Earliest PTS of the whole stream + earliest_pts: Option, + /// Current end PTS of the whole stream + end_pts: Option, + + /// Edit list entries for this stream. + elst_infos: Vec, +} + +impl Stream { + fn get_elst_infos(&self) -> Result, anyhow::Error> { + let mut elst_infos = self.elst_infos.clone(); + let timescale = self.timescale(); + let earliest_pts = self.earliest_pts.unwrap_or(gst::ClockTime::ZERO); + let end_pts = self + .end_pts + .unwrap_or(gst::ClockTime::from_nseconds(u64::MAX - 1)); + + let mut iter = elst_infos.iter_mut().peekable(); + while let Some(&mut ref mut elst_info) = iter.next() { + if elst_info.duration.unwrap_or(0u64) == 0u64 { + elst_info.duration = if let Some(next) = iter.peek_mut() { + Some((next.start - elst_info.start) as u64) + } else { + Some( + (end_pts - earliest_pts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?, + ) + } + } + } + + Ok(elst_infos) + } + + fn timescale(&self) -> u32 { + let trak_timescale = { self.sinkpad.imp().settings.lock().unwrap().trak_timescale }; + + if trak_timescale > 0 { + return trak_timescale; + } + + let s = self.caps.structure(0).unwrap(); + + if let Ok(fps) = s.get::("framerate") { + if fps.numer() == 0 { + return 10_000; + } + + if fps.denom() != 1 && fps.denom() != 1001 { + if let Some(fps) = (fps.denom() as u64) + .nseconds() + .mul_div_round(1_000_000_000, fps.numer() as u64) + .and_then(gst_video::guess_framerate) + { + return (fps.numer() as u32) + .mul_div_round(100, fps.denom() as u32) + .unwrap_or(10_000); + } + } + + if fps.denom() == 1001 { + fps.numer() as u32 + } else { + (fps.numer() as u32) + .mul_div_round(100, fps.denom() as u32) + .unwrap_or(10_000) + } + } else if let Ok(rate) = s.get::("rate") { + rate as u32 + } else { + 10_000 + } + } } #[derive(Default)] @@ -270,6 +348,64 @@ pub(crate) struct FMP4Mux { } impl FMP4Mux { + fn add_elst_info( + &self, + buffer: &PreQueuedBuffer, + stream: &mut Stream, + ) -> Result<(), anyhow::Error> { + if let Some(cmeta) = buffer.buffer.meta::() { + let timescale = stream + .caps + .structure(0) + .unwrap() + .get::("rate") + .unwrap_or_else(|_| stream.timescale() as i32); + + let gstclocktime_to_samples = move |v: gst::ClockTime| { + v.nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("Invalid start in the AudioClipMeta") + }; + + let generic_to_samples = move |t| -> Result, anyhow::Error> { + if let gst::GenericFormattedValue::Default(Some(v)) = t { + let v: u64 = v.into(); + Ok(Some(v).filter(|x| x != &0u64)) + } else if let gst::GenericFormattedValue::Time(Some(v)) = t { + Ok(Some(gstclocktime_to_samples(v)?)) + } else { + Ok(None) + } + }; + + let start: Option = generic_to_samples(cmeta.start())?; + let end: Option = generic_to_samples(cmeta.end())?; + if end.is_none() && start.is_none() { + return Err(anyhow::anyhow!( + "No start or end time in `default` format in the AudioClipingMeta" + )); + } + + let start = if let Some(start) = generic_to_samples(cmeta.start())? { + start + gstclocktime_to_samples(buffer.pts)? + } else { + 0 + }; + let duration = if let Some(e) = end { + Some(gstclocktime_to_samples(buffer.end_pts)? - e) + } else { + None + }; + + stream.elst_infos.push(super::ElstInfo { + start: start as i64, + duration, + }); + } + + Ok(()) + } + /// Checks if a buffer is valid according to the stream configuration. fn check_buffer( buffer: &gst::BufferRef, @@ -361,6 +497,14 @@ impl FMP4Mux { gst::ClockTime::ZERO }); + if stream.earliest_pts.opt_gt(pts).unwrap_or(true) { + stream.end_pts = Some(pts); + } + + if stream.end_pts.opt_lt(end_pts).unwrap_or(true) { + stream.end_pts = Some(end_pts); + } + let (dts, end_dts) = if !stream.delta_frames.requires_dts() { (None, None) } else { @@ -592,7 +736,13 @@ impl FMP4Mux { assert!(stream.running_time_utc_time_mapping.is_some()); } - stream.pre_queue.pop_front().unwrap() + let buffer = stream.pre_queue.pop_front().unwrap(); + + if let Err(err) = self.add_elst_info(&buffer, stream) { + gst::error!(CAT, "Failed to add elst info: {:#}", err); + } + + buffer } /// Finds the stream that has the earliest buffer queued. @@ -2615,6 +2765,9 @@ impl FMP4Mux { dts_offset: None, current_position: gst::ClockTime::ZERO, running_time_utc_time_mapping: None, + earliest_pts: None, + end_pts: None, + elst_infos: Vec::new(), }); } @@ -2678,10 +2831,18 @@ impl FMP4Mux { let streams = state .streams .iter() - .map(|s| super::HeaderStream { - trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale, - delta_frames: s.delta_frames, - caps: s.caps.clone(), + .map(|s| { + let trak_timescale = { s.sinkpad.imp().settings.lock().unwrap().trak_timescale }; + super::HeaderStream { + trak_timescale, + delta_frames: s.delta_frames, + caps: s.caps.clone(), + elst_infos: s.get_elst_infos().unwrap_or_else(|e| { + gst::error!(CAT, "Could not prepare edit lists: {e:?}"); + + Vec::new() + }), + } }) .collect::>(); diff --git a/mux/fmp4/src/fmp4mux/mod.rs b/mux/fmp4/src/fmp4mux/mod.rs index bb50c289..3dce44b5 100644 --- a/mux/fmp4/src/fmp4mux/mod.rs +++ b/mux/fmp4/src/fmp4mux/mod.rs @@ -91,6 +91,12 @@ pub(crate) struct HeaderConfiguration { start_utc_time: Option, } +#[derive(Debug, Clone)] +pub(crate) struct ElstInfo { + start: i64, + duration: Option, +} + #[derive(Debug)] pub(crate) struct HeaderStream { /// Caps of this stream @@ -101,6 +107,9 @@ pub(crate) struct HeaderStream { /// Pre-defined trak timescale if not 0. trak_timescale: u32, + + /// Edit list clipping information + elst_infos: Vec, } #[derive(Debug)] From ff0da29f19c9cc1938420855f3745391d7c285b4 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 9 Jan 2024 08:15:28 -0300 Subject: [PATCH 2/4] fmp4mux: Add support for GstAggregator::start-time-selector==set Taking it into account so the encoded stream start time is what was set in `GstAggregator::start-time`, respecting what was specified by the user. --- mux/fmp4/src/fmp4mux/imp.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index 8b8e98cd..c98b5098 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -2025,6 +2025,13 @@ impl FMP4Mux { settings.chunk_duration.map(|duration| chunk_start_pts + duration).display(), ); + let obj = self.obj(); + let start_time_offset = + if obj.start_time_selection() == gst_base::AggregatorStartTimeSelection::Set { + gst::ClockTime::from_nseconds(AggregatorExt::start_time(&*obj)) + } else { + gst::ClockTime::ZERO + }; for (idx, stream) in state.streams.iter_mut().enumerate() { let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone(); @@ -2209,7 +2216,7 @@ impl FMP4Mux { drained_streams.push(( super::FragmentHeaderStream { caps: stream.caps.clone(), - start_time: Some(start_time), + start_time: Some(start_time + start_time_offset), delta_frames: stream.delta_frames, trak_timescale: stream_settings.trak_timescale, }, From ecbf46e82be5ee37b8658511ce37b629d4889283 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 9 Jan 2024 08:47:51 -0300 Subject: [PATCH 3/4] mp4mux: Add support for edit lists So we properly handle audio priming --- mux/mp4/src/mp4mux/boxes.rs | 135 ++++------------------ mux/mp4/src/mp4mux/imp.rs | 216 +++++++++++++++++++++++++++++++++++- mux/mp4/src/mp4mux/mod.rs | 19 ++-- 3 files changed, 245 insertions(+), 125 deletions(-) diff --git a/mux/mp4/src/mp4mux/boxes.rs b/mux/mp4/src/mp4mux/boxes.rs index 4e2188b0..53ee2f03 100644 --- a/mux/mp4/src/mp4mux/boxes.rs +++ b/mux/mp4/src/mp4mux/boxes.rs @@ -210,50 +210,12 @@ fn write_moov(v: &mut Vec, header: &super::Header) -> Result<(), Error> { Ok(()) } -fn stream_to_timescale(stream: &super::Stream) -> u32 { - if stream.trak_timescale > 0 { - stream.trak_timescale - } else { - let s = stream.caps.structure(0).unwrap(); - - if let Ok(fps) = s.get::("framerate") { - if fps.numer() == 0 { - return 10_000; - } - - if fps.denom() != 1 && fps.denom() != 1001 { - if let Some(fps) = (fps.denom() as u64) - .nseconds() - .mul_div_round(1_000_000_000, fps.numer() as u64) - .and_then(gst_video::guess_framerate) - { - return (fps.numer() as u32) - .mul_div_round(100, fps.denom() as u32) - .unwrap_or(10_000); - } - } - - if fps.denom() == 1001 { - fps.numer() as u32 - } else { - (fps.numer() as u32) - .mul_div_round(100, fps.denom() as u32) - .unwrap_or(10_000) - } - } else if let Ok(rate) = s.get::("rate") { - rate as u32 - } else { - 10_000 - } - } -} - fn header_to_timescale(header: &super::Header) -> u32 { if header.movie_timescale > 0 { header.movie_timescale } else { // Use the reference track timescale - stream_to_timescale(&header.streams[0]) + header.streams[0].timescale } } @@ -337,7 +299,7 @@ fn write_trak( if !references.is_empty() { write_box(v, b"tref", |v| write_tref(v, header, references))?; } - write_box(v, b"edts", |v| write_edts(v, header, stream))?; + write_box(v, b"edts", |v| write_edts(v, stream))?; Ok(()) } @@ -474,7 +436,7 @@ fn write_mdhd( stream: &super::Stream, creation_time: u64, ) -> Result<(), Error> { - let timescale = stream_to_timescale(stream); + let timescale = stream.timescale; // Creation time v.extend(creation_time.to_be_bytes()); @@ -1366,7 +1328,7 @@ fn write_stts( _header: &super::Header, stream: &super::Stream, ) -> Result<(), Error> { - let timescale = stream_to_timescale(stream); + let timescale = stream.timescale; let entry_count_position = v.len(); // Entry count, rewritten in the end @@ -1420,7 +1382,7 @@ fn write_ctts( stream: &super::Stream, version: u8, ) -> Result<(), Error> { - let timescale = stream_to_timescale(stream); + let timescale = stream.timescale; let entry_count_position = v.len(); // Entry count, rewritten in the end @@ -1490,7 +1452,7 @@ fn write_cslg( _header: &super::Header, stream: &super::Stream, ) -> Result<(), Error> { - let timescale = stream_to_timescale(stream); + let timescale = stream.timescale; let (min_ctts, max_ctts) = stream .chunks @@ -1706,82 +1668,31 @@ fn write_tref( Ok(()) } -fn write_edts( - v: &mut Vec, - header: &super::Header, - stream: &super::Stream, -) -> Result<(), Error> { - write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| { - write_elst(v, header, stream) - })?; +fn write_edts(v: &mut Vec, stream: &super::Stream) -> Result<(), Error> { + write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?; Ok(()) } -fn write_elst( - v: &mut Vec, - header: &super::Header, - stream: &super::Stream, -) -> Result<(), Error> { - // In movie header timescale - let timescale = header_to_timescale(header); +fn write_elst(v: &mut Vec, stream: &super::Stream) -> Result<(), Error> { + // Entry count + v.extend((stream.elst_infos.len() as u32).to_be_bytes()); - let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap(); + for elst_info in &stream.elst_infos { + v.extend( + elst_info + .duration + .expect("Should have been set by `get_elst_infos`") + .to_be_bytes(), + ); - if min_earliest_pts != stream.earliest_pts { - let gap = (stream.earliest_pts - min_earliest_pts) - .nseconds() - .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) - .context("too big gap")?; + // Media time + v.extend(elst_info.start.to_be_bytes()); - if gap > 0 { - // Entry count - v.extend(2u32.to_be_bytes()); - - // First entry for the gap - - // Edit duration - v.extend(gap.to_be_bytes()); - - // Media time - v.extend((-1i64).to_be_bytes()); - - // Media rate - v.extend(1u16.to_be_bytes()); - v.extend(0u16.to_be_bytes()); - } else { - // Entry count - v.extend(1u32.to_be_bytes()); - } - } else { - // Entry count - v.extend(1u32.to_be_bytes()); + // Media rate + v.extend(1u16.to_be_bytes()); + v.extend(0u16.to_be_bytes()); } - // Edit duration - let duration = (stream.end_pts - stream.earliest_pts) - .nseconds() - .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) - .context("too big track duration")?; - v.extend(duration.to_be_bytes()); - - // Media time - if let Some(start_dts) = stream.start_dts { - let shift = (gst::Signed::Positive(stream.earliest_pts) - start_dts) - .nseconds() - .positive() - .unwrap_or(0) - .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) - .context("too big track duration")?; - - v.extend(shift.to_be_bytes()); - } else { - v.extend(0u64.to_be_bytes()); - } - - // Media rate - v.extend(1u16.to_be_bytes()); - v.extend(0u16.to_be_bytes()); - Ok(()) } diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs index 6ea81c19..3701a772 100644 --- a/mux/mp4/src/mp4mux/imp.rs +++ b/mux/mp4/src/mp4mux/imp.rs @@ -6,6 +6,7 @@ // // SPDX-License-Identifier: MPL-2.0 +use anyhow::{anyhow, Context}; use gst::glib; use gst::prelude::*; use gst::subclass::prelude::*; @@ -128,6 +129,10 @@ struct Stream { /// Earliest PTS. earliest_pts: Option, + + /// Edit list entries for this stream. + elst_infos: Vec, + /// Current end PTS. end_pts: Option, @@ -135,6 +140,121 @@ struct Stream { running_time_utc_time_mapping: Option<(gst::Signed, gst::ClockTime)>, } +impl Stream { + fn get_elst_infos( + &self, + min_earliest_pts: gst::ClockTime, + ) -> Result, anyhow::Error> { + let mut elst_infos = self.elst_infos.clone(); + let timescale = self.timescale(); + let earliest_pts = self + .earliest_pts + .expect("Streams without earliest_pts should have been skipped"); + let end_pts = self + .end_pts + .expect("Streams without end_pts should have been skipped"); + + // If no elst info were set, use the whole track + if self.elst_infos.is_empty() { + let start = if let Some(start_dts) = self.start_dts { + ((gst::Signed::Positive(earliest_pts) - start_dts) + .nseconds() + .positive() + .unwrap_or(0) + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?) as i64 + } else { + 0i64 + }; + + elst_infos.push(super::ElstInfo { + start, + duration: Some( + (end_pts - earliest_pts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?, + ), + }); + } + + // Add a gap at the beginning if needed + if min_earliest_pts != earliest_pts { + let gap_duration = (earliest_pts - min_earliest_pts).nseconds()) + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big gap")?; + + if gap_duration > 0 { + elst_infos.insert( + 0, + super::ElstInfo { + start: -1, + duration: Some(gap_duration), + }, + ); + } + } + + let mut iter = elst_infos.iter_mut().peekable(); + while let Some(&mut ref mut elst_info) = iter.next() { + if elst_info.duration.unwrap_or(0u64) == 0u64 { + elst_info.duration = if let Some(next) = iter.peek_mut() { + Some((next.start - elst_info.start) as u64) + } else { + Some( + (end_pts - earliest_pts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?, + ) + } + } + } + + Ok(elst_infos) + } + + fn timescale(&self) -> u32 { + let trak_timescale = { self.sinkpad.imp().settings.lock().unwrap().trak_timescale }; + + if trak_timescale > 0 { + return trak_timescale; + } + + let s = self.caps.structure(0).unwrap(); + + if let Ok(fps) = s.get::("framerate") { + if fps.numer() == 0 { + return 10_000; + } + + if fps.denom() != 1 && fps.denom() != 1001 { + if let Some(fps) = (fps.denom() as u64) + .nseconds() + .mul_div_round(1_000_000_000, fps.numer() as u64) + .and_then(gst_video::guess_framerate) + { + return (fps.numer() as u32) + .mul_div_round(100, fps.denom() as u32) + .unwrap_or(10_000); + } + } + + if fps.denom() == 1001 { + fps.numer() as u32 + } else { + (fps.numer() as u32) + .mul_div_round(100, fps.denom() as u32) + .unwrap_or(10_000) + } + } else if let Ok(rate) = s.get::("rate") { + rate as u32 + } else { + 10_000 + } + } +} + #[derive(Default)] struct State { /// List of streams when the muxer was started. @@ -184,6 +304,77 @@ impl MP4Mux { Ok(()) } + fn add_elst_info( + &self, + buffer: &PendingBuffer, + stream: &mut Stream, + ) -> Result<(), anyhow::Error> { + let cmeta = if let Some(cmeta) = buffer.buffer.meta::() { + cmeta + } else { + return Ok(()); + }; + + let timescale = stream + .caps + .structure(0) + .unwrap() + .get::("rate") + .unwrap_or_else(|_| stream.timescale() as i32); + + let gstclocktime_to_samples = move |v: gst::ClockTime| { + v.nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("Invalid start in the AudioClipMeta") + }; + + let generic_to_samples = move |t| -> Result, anyhow::Error> { + if let gst::GenericFormattedValue::Default(Some(v)) = t { + let v: u64 = v.into(); + Ok(Some(v).filter(|x| x != &0u64)) + } else if let gst::GenericFormattedValue::Time(Some(v)) = t { + Ok(Some(gstclocktime_to_samples(v)?)) + } else { + Ok(None) + } + }; + + let start: Option = generic_to_samples(cmeta.start())?; + let end: Option = generic_to_samples(cmeta.end())?; + + if end.is_none() && start.is_none() { + return Err(anyhow!( + "No start or end time in `default` format in the AudioClipingMeta" + )); + } + + let start = if let Some(start) = generic_to_samples(cmeta.start())? { + start + gstclocktime_to_samples(buffer.pts)? + } else { + 0 + }; + let duration = if let Some(e) = end { + Some( + gstclocktime_to_samples(buffer.pts)? + + gstclocktime_to_samples( + buffer + .duration + .context("No duration on buffer, we can't add edit list")?, + )? + - e, + ) + } else { + None + }; + + stream.elst_infos.push(super::ElstInfo { + start: start as i64, + duration, + }); + + Ok(()) + } + fn peek_buffer( &self, sinkpad: &super::MP4MuxPad, @@ -815,6 +1006,11 @@ impl MP4Mux { let duration = buffer.duration.unwrap(); let composition_time_offset = buffer.composition_time_offset; + + if let Err(err) = self.add_elst_info(&buffer, stream) { + gst::error!(CAT, "Failed to add elst info: {:#}", err); + } + let mut buffer = buffer.buffer; stream.queued_chunk_time += duration; @@ -930,6 +1126,7 @@ impl MP4Mux { queued_chunk_bytes: 0, start_dts: None, earliest_pts: None, + elst_infos: Default::default(), end_pts: None, running_time_utc_time_mapping: None, }); @@ -1319,9 +1516,14 @@ impl AggregatorImpl for MP4Mux { state.mdat_size ); + let min_earliest_pts = state + .streams + .iter() + .filter_map(|s| s.earliest_pts) + .min() + .unwrap(); let mut streams = Vec::with_capacity(state.streams.len()); for stream in state.streams.drain(..) { - let pad_settings = stream.sinkpad.imp().settings.lock().unwrap().clone(); let (earliest_pts, end_pts) = match Option::zip(stream.earliest_pts, stream.end_pts) { Some(res) => res, @@ -1331,16 +1533,22 @@ impl AggregatorImpl for MP4Mux { streams.push(super::Stream { caps: stream.caps.clone(), delta_frames: stream.delta_frames, - trak_timescale: pad_settings.trak_timescale, - start_dts: stream.start_dts, + timescale: stream.timescale(), earliest_pts, end_pts, + elst_infos: stream + .get_elst_infos(min_earliest_pts) + .unwrap_or_else(|e| { + gst::error!(CAT, "Could not prepare edit lists: {e:?}"); + + Vec::new() + }), chunks: stream.chunks, }); } let moov = boxes::create_moov(super::Header { - variant: self.obj().class().as_ref().variant, + variant: obj.class().as_ref().variant, movie_timescale: settings.movie_timescale, streams, }) diff --git a/mux/mp4/src/mp4mux/mod.rs b/mux/mp4/src/mp4mux/mod.rs index 5dd012bb..b35b42ce 100644 --- a/mux/mp4/src/mp4mux/mod.rs +++ b/mux/mp4/src/mp4mux/mod.rs @@ -97,6 +97,12 @@ pub(crate) struct Chunk { samples: Vec, } +#[derive(Debug, Clone)] +pub(crate) struct ElstInfo { + start: i64, + duration: Option, +} + #[derive(Debug)] pub(crate) struct Stream { /// Caps of this stream @@ -106,15 +112,7 @@ pub(crate) struct Stream { delta_frames: DeltaFrames, /// Pre-defined trak timescale if not 0. - trak_timescale: u32, - - /// Start DTS - /// - /// If this is negative then an edit list entry is needed to - /// make all sample times positive. - /// - /// This is `None` for streams that have no concept of DTS. - start_dts: Option>, + timescale: u32, /// Earliest PTS /// @@ -126,6 +124,9 @@ pub(crate) struct Stream { /// All the chunks stored for this stream chunks: Vec, + + /// Edit list clipping information + elst_infos: Vec, } #[derive(Debug)] From 9155a57722e36bd76fd4e12c8b499a891b189fa2 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 19 Jan 2024 15:14:03 -0300 Subject: [PATCH 4/4] mp4mux: Add support for GstAggregator::start-time-selector==set Taking it into account so the encoded stream start time is what was set in `GstAggregator::start-time`, respecting what was specified by the user. --- mux/mp4/src/mp4mux/imp.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs index 3701a772..cd3ace96 100644 --- a/mux/mp4/src/mp4mux/imp.rs +++ b/mux/mp4/src/mp4mux/imp.rs @@ -143,6 +143,7 @@ struct Stream { impl Stream { fn get_elst_infos( &self, + start_time: gst::ClockTime, min_earliest_pts: gst::ClockTime, ) -> Result, anyhow::Error> { let mut elst_infos = self.elst_infos.clone(); @@ -179,10 +180,11 @@ impl Stream { } // Add a gap at the beginning if needed - if min_earliest_pts != earliest_pts { - let gap_duration = (earliest_pts - min_earliest_pts).nseconds()) - .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) - .context("too big gap")?; + if min_earliest_pts != earliest_pts || start_time > gst::ClockTime::ZERO { + let gap_duration = (start_time.nseconds() + + (earliest_pts - min_earliest_pts).nseconds()) + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big gap")?; if gap_duration > 0 { elst_infos.insert( @@ -1516,6 +1518,13 @@ impl AggregatorImpl for MP4Mux { state.mdat_size ); + let obj = self.obj(); + let start_time = + if obj.start_time_selection() == gst_base::AggregatorStartTimeSelection::Set { + gst::ClockTime::from_nseconds(AggregatorExt::start_time(&*obj)) + } else { + gst::ClockTime::ZERO + }; let min_earliest_pts = state .streams .iter() @@ -1537,7 +1546,7 @@ impl AggregatorImpl for MP4Mux { earliest_pts, end_pts, elst_infos: stream - .get_elst_infos(min_earliest_pts) + .get_elst_infos(start_time, min_earliest_pts) .unwrap_or_else(|e| { gst::error!(CAT, "Could not prepare edit lists: {e:?}");