Merge branch 'feat-fmp4-sidx-boxes' into 'main'

fmp4mux: add sidx support

See merge request gstreamer/gst-plugins-rs!1133

mux/fmp4/src/fmp4mux/boxes.rs

@@ -1707,8 +1707,81 @@ pub(super) fn create_fmp4_fragment_header(
     }
 
     let styp_len = v.len();
+    let mut sidx_offsets: Option<Vec<(usize, usize)>> = None;
 
-    let data_offset_offsets = write_box(&mut v, b"moof", |v| write_moof(v, &cfg))?;
+    if cfg.write_sidx && !cfg.chunk {
+        let mut start_sidx_offset = v.len();
+        sidx_offsets = Some(
+            cfg.streams
+                .iter()
+                .enumerate()
+                .map(|(idx, stream)| {
+                    let timescale = fragment_header_stream_to_timescale(stream);
+                    let earliest_presentation_time = stream
+                        .start_time
+                        .unwrap()
+                        .mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                        .expect("base time overflow");
+
+                    write_full_box(
+                        &mut v,
+                        b"sidx",
+                        FULL_BOX_VERSION_1,
+                        FULL_BOX_FLAGS_NONE,
+                        |v| {
+                            // reference ID / Track ID
+                            v.extend((idx as u32 + 1).to_be_bytes());
+                            // timescale
+                            v.extend((timescale as u32).to_be_bytes());
+                            // earliest_presentation_time
+                            v.extend(earliest_presentation_time.to_be_bytes());
+                            // first_offset, rewritten afterwards
+                            v.extend(0u64.to_be_bytes());
+                            // reserved
+                            v.extend(0u16.to_be_bytes());
+                            // ref count == 1, single reference for non-chunked mode
+                            v.extend(1u16.to_be_bytes());
+
+                            // filled afterwards
+                            // type + size
+                            v.extend(0u32.to_be_bytes());
+                            // duration
+                            v.extend(0u32.to_be_bytes());
+
+                            // starts with SAP, type, delta_time
+                            let mut sap_byte: u32 = 0;
+                            sap_byte |= if cfg
+                                .buffers
+                                .first()
+                                .unwrap()
+                                .buffer
+                                .flags()
+                                .contains(gst::BufferFlags::DELTA_UNIT)
+                            {
+                                0u32 << 31 // is delta frame
+                            } else {
+                                1u32 << 31 // is keyframe
+                            };
+                            v.extend(sap_byte.to_be_bytes());
+
+                            Ok(())
+                        },
+                    )
+                    .expect("written sidx");
+
+                    let end_sidx_offset = v.len();
+                    let ret = (start_sidx_offset, end_sidx_offset);
+                    start_sidx_offset = end_sidx_offset;
+                    ret
+                })
+                .collect()
+        );
+    }
+
+    let first_moof_offset = v.len();
+
+    let (data_offset_offsets, buffers_sizes, buffers_durations) =
+        write_box(&mut v, b"moof", |v| write_moof(v, &cfg))?;
 
     let size = cfg
         .buffers
@@ -1724,7 +1797,7 @@ pub(super) fn create_fmp4_fragment_header(
         v.extend((size + 16).to_be_bytes());
     }
 
-    let data_offset = v.len() - styp_len;
+    let data_offset = v.len() - first_moof_offset;
     for data_offset_offset in data_offset_offsets {
         let val = u32::from_be_bytes(v[data_offset_offset..][..4].try_into()?)
             .checked_add(u32::try_from(data_offset)?)
@@ -1732,30 +1805,66 @@ pub(super) fn create_fmp4_fragment_header(
         v[data_offset_offset..][..4].copy_from_slice(&val.to_be_bytes());
     }
 
+    if cfg.write_sidx && !cfg.chunk {
+        for ((sidx_start, end), (buffers_size, buffers_duration)) in sidx_offsets
+            .unwrap()
+            .into_iter()
+            .zip(buffers_sizes.into_iter().zip(buffers_durations))
+        {
+            let first_offset = (first_moof_offset - end) as u64;
+            // first_offset
+            v[sidx_start + 28..][..8].copy_from_slice(&first_offset.to_be_bytes());
+            // (1b) reference_type + (31b) referenced_size
+            let reference = &mut buffers_size.to_be_bytes();
+            let ref_type_byte = reference.get_mut(0).unwrap();
+            *ref_type_byte &= !(1u8 << 7);  // set reference_type to 0
+            v[sidx_start + 40..][..4].copy_from_slice(reference);
+            // subsegment duration
+            v[sidx_start + 44..][..4].copy_from_slice(&buffers_duration.to_be_bytes());
+        }
+    }
+
     Ok((gst::Buffer::from_mut_slice(v), styp_len as u64))
 }
 
 fn write_moof(
     v: &mut Vec<u8>,
     cfg: &super::FragmentHeaderConfiguration,
-) -> Result<Vec<usize>, Error> {
+) -> Result<(Vec<usize>, Vec<u32>, Vec<u32>), Error> {
     write_full_box(v, b"mfhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
         write_mfhd(v, cfg)
     })?;
 
     let mut data_offset_offsets = vec![];
+    let mut buffers_total_sizes = vec![];
+    let mut buffers_total_durations = vec![];
+
     for (idx, stream) in cfg.streams.iter().enumerate() {
         // Skip tracks without any buffers for this fragment.
         if stream.start_time.is_none() {
+            buffers_total_sizes.push(0);
+            buffers_total_durations.push(0);
             continue;
         }
 
         write_box(v, b"traf", |v| {
-            write_traf(v, cfg, &mut data_offset_offsets, idx, stream)
+            write_traf(
+                v,
+                cfg,
+                &mut data_offset_offsets,
+                &mut buffers_total_sizes,
+                &mut buffers_total_durations,
+                idx,
+                stream,
+            )
         })?;
     }
 
-    Ok(data_offset_offsets)
+    Ok((
+        data_offset_offsets,
+        buffers_total_sizes,
+        buffers_total_durations,
+    ))
 }
 
 fn write_mfhd(v: &mut Vec<u8>, cfg: &super::FragmentHeaderConfiguration) -> Result<(), Error> {
@@ -1834,6 +1943,10 @@ fn analyze_buffers(
         Option<u32>,
         // negative composition time offsets
         bool,
+        // total size
+        u32,
+        // total duration
+        u32,
     ),
     Error,
 > {
@@ -1845,6 +1958,9 @@ fn analyze_buffers(
     let mut first_buffer_flags = None;
     let mut flags = None;
 
+    let mut total_size = 0u32;
+    let mut total_duration = 0u32;
+
     let mut negative_composition_time_offsets = false;
 
     for Buffer {
@@ -1861,6 +1977,7 @@ fn analyze_buffers(
         if Some(buffer.size() as u32) != size {
             tr_flags |= SAMPLE_SIZE_PRESENT;
         }
+        total_size += buffer.size() as u32;
 
         let sample_duration = u32::try_from(
             sample_duration
@@ -1876,6 +1993,7 @@ fn analyze_buffers(
         if Some(sample_duration) != duration {
             tr_flags |= SAMPLE_DURATION_PRESENT;
         }
+        total_duration += sample_duration as u32;
 
         let f = sample_flags_from_buffer(stream, buffer);
         if first_buffer_flags.is_none() {
@@ -1941,6 +2059,8 @@ fn analyze_buffers(
         duration,
         flags,
         negative_composition_time_offsets,
+        total_size,
+        total_duration,
     ))
 }
 
@@ -1949,6 +2069,8 @@ fn write_traf(
     v: &mut Vec<u8>,
     cfg: &super::FragmentHeaderConfiguration,
     data_offset_offsets: &mut Vec<usize>,
+    buffers_total_sizes: &mut Vec<u32>,
+    buffers_total_durations: &mut Vec<u32>,
     idx: usize,
     stream: &super::FragmentHeaderStream,
 ) -> Result<(), Error> {
@@ -1963,6 +2085,8 @@ fn write_traf(
         default_duration,
         default_flags,
         negative_composition_time_offsets,
+        total_size,
+        total_duration,
     ) = analyze_buffers(cfg, idx, stream, timescale)?;
 
     assert!((tf_flags & DEFAULT_SAMPLE_SIZE_PRESENT == 0) ^ default_size.is_some());
@@ -2021,6 +2145,8 @@ fn write_traf(
 
     // TODO: saio, saiz, sbgp, sgpd, subs?
 
+    buffers_total_sizes.push(total_size);
+    buffers_total_durations.push(total_duration);
     Ok(())
 }
 

mux/fmp4/src/fmp4mux/imp.rs

@@ -99,6 +99,7 @@ const DEFAULT_CHUNK_DURATION: Option<gst::ClockTime> = gst::ClockTime::NONE;
 const DEFAULT_HEADER_UPDATE_MODE: super::HeaderUpdateMode = super::HeaderUpdateMode::None;
 const DEFAULT_WRITE_MFRA: bool = false;
 const DEFAULT_WRITE_MEHD: bool = false;
+const DEFAULT_WRITE_SIDX: bool = false;
 const DEFAULT_INTERLEAVE_BYTES: Option<u64> = None;
 const DEFAULT_INTERLEAVE_TIME: Option<gst::ClockTime> = Some(gst::ClockTime::from_mseconds(250));
 
@@ -109,6 +110,7 @@ struct Settings {
     header_update_mode: super::HeaderUpdateMode,
     write_mfra: bool,
     write_mehd: bool,
+    write_sidx: bool,
     interleave_bytes: Option<u64>,
     interleave_time: Option<gst::ClockTime>,
     movie_timescale: u32,
@@ -123,6 +125,7 @@ impl Default for Settings {
             header_update_mode: DEFAULT_HEADER_UPDATE_MODE,
             write_mfra: DEFAULT_WRITE_MFRA,
             write_mehd: DEFAULT_WRITE_MEHD,
+            write_sidx: DEFAULT_WRITE_SIDX,
             interleave_bytes: DEFAULT_INTERLEAVE_BYTES,
             interleave_time: DEFAULT_INTERLEAVE_TIME,
             movie_timescale: 0,
@@ -2331,7 +2334,6 @@ impl FMP4Mux {
         }
 
         // TODO: Write prft boxes before moof
-        // TODO: Write sidx boxes before moof and rewrite once offsets are known
 
         // First sequence number must be 1
         if state.sequence_number == 0 {
@@ -2350,6 +2352,7 @@ impl FMP4Mux {
                 chunk: !fragment_start,
                 streams: streams.as_slice(),
                 buffers: interleaved_buffers.as_slice(),
+                write_sidx: settings.write_sidx,
             })
             .map_err(|err| {
                 gst::error!(
@@ -2866,6 +2869,12 @@ impl ObjectImpl for FMP4Mux {
                     .default_value(DEFAULT_WRITE_MFRA)
                     .mutable_ready()
                     .build(),
+                glib::ParamSpecBoolean::builder("write-sidx")
+                    .nick("Write sidx boxes")
+                    .blurb("Write segment index boxes with track buffers details and references before moof (sidx boxes are not updated in chunked mode)")
+                    .default_value(DEFAULT_WRITE_SIDX)
+                    .mutable_ready()
+                    .build(),
                 glib::ParamSpecUInt64::builder("interleave-bytes")
                     .nick("Interleave Bytes")
                     .blurb("Interleave between streams in bytes")
@@ -2932,6 +2941,11 @@ impl ObjectImpl for FMP4Mux {
                 settings.write_mehd = value.get().expect("type checked upstream");
             }
 
+            "write-sidx" => {
+                let mut settings = self.settings.lock().unwrap();
+                settings.write_sidx = value.get().expect("type checked upstream");
+            }
+
             "interleave-bytes" => {
                 let mut settings = self.settings.lock().unwrap();
                 settings.interleave_bytes = match value.get().expect("type checked upstream") {
@@ -2984,6 +2998,11 @@ impl ObjectImpl for FMP4Mux {
                 settings.write_mehd.to_value()
             }
 
+            "write-sidx" => {
+                let settings = self.settings.lock().unwrap();
+                settings.write_sidx.to_value()
+            }
+
             "interleave-bytes" => {
                 let settings = self.settings.lock().unwrap();
                 settings.interleave_bytes.unwrap_or(0).to_value()

mux/fmp4/src/fmp4mux/mod.rs

@@ -115,6 +115,9 @@ pub(crate) struct FragmentHeaderConfiguration<'a> {
 
     streams: &'a [FragmentHeaderStream],
     buffers: &'a [Buffer],
+
+    /// If sidx boxes should be written before moof
+    write_sidx: bool,
 }
 
 #[derive(Debug)]