mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-05-20 01:08:06 +00:00
Code Issues Projects Releases Wiki Activity

fmp4mux: Make media/trak timescales configurable

Browse source 
And refactor a bit of code for easier extensibility.
This commit is contained in:
Sebastian Dröge 2022-11-07 19:47:31 +02:00 committed by Sebastian Dröge
parent e87251c7d9
commit f062b7cf0d
5 changed files with 363 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
docs/plugins/gst_plugins_cache.json
View file

@ -1589,7 +1589,8 @@
"sink": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n",
"direction": "sink",
"presence": "always"
"presence": "always",
"type": "GstFMP4MuxPad"
},
"src": {
"caps": "video/quicktime:\n variant: cmaf\n",
@ -1617,7 +1618,8 @@
"sink": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n",
"direction": "sink",
"presence": "always"
"presence": "always",
"type": "GstFMP4MuxPad"
},
"src": {
"caps": "video/quicktime:\n variant: iso-fragmented\n",
@ -1645,7 +1647,8 @@
"sink_%%u": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n",
"direction": "sink",
"presence": "request"
"presence": "request",
"type": "GstFMP4MuxPad"
},
"src": {
"caps": "video/quicktime:\n variant: iso-fragmented\n",
@ -1673,7 +1676,8 @@
"sink_%%u": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nimage/jpeg:\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-alaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-mulaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-adpcm:\n layout: g726\n channels: 1\n rate: 8000\n bitrate: { (int)16000, (int)24000, (int)32000, (int)40000 }\napplication/x-onvif-metadata:\n parsed: true\n",
"direction": "sink",
"presence": "request"
"presence": "request",
"type": "GstFMP4MuxPad"
},
"src": {
"caps": "video/quicktime:\n variant: iso-fragmented\n",
@ -1752,6 +1756,20 @@
"type": "guint64",
"writable": true
},
"movie-timescale": {
"blurb": "Timescale to use for the movie (units per second, 0 is automatic)",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "0",
"max": "-1",
"min": "0",
"mutable": "ready",
"readable": true,
"type": "guint",
"writable": true
},
"write-mehd": {
"blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)",
"conditionally-available": false,
@ -1797,6 +1815,33 @@
"value": "2"
}
]
},
"GstFMP4MuxPad": {
"hierarchy": [
"GstFMP4MuxPad",
"GstAggregatorPad",
"GstPad",
"GstObject",
"GInitiallyUnowned",
"GObject"
],
"kind": "object",
"properties": {
"trak-timescale": {
"blurb": "Timescale to use for the track (units per second, 0 is automatic)",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "0",
"max": "-1",
"min": "0",
"mutable": "ready",
"readable": true,
"type": "guint",
"writable": true
}
}
}
},
"package": "gst-plugin-fmp4",

1
mux/fmp4/Cargo.toml
View file

@ -37,6 +37,7 @@ default = ["v1_18"]
static = []
capi = []
v1_18 = ["gst-video/v1_18"]
doc = ["gst/v1_18"]
[package.metadata.capi]
min_version = "0.8.0"

177
mux/fmp4/src/fmp4mux/boxes.rs
View file

@ -355,7 +355,8 @@ fn brands_from_variant_and_caps<'a>(
pub(super) fn create_fmp4_header(cfg: super::HeaderConfiguration) -> Result<gst::Buffer, Error> {
let mut v = vec![];
let (brand, compatible_brands) = brands_from_variant_and_caps(cfg.variant, cfg.streams.iter());
let (brand, compatible_brands) =
brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.caps));
write_box(&mut v, b"ftyp", |v| {
// major brand
@ -420,17 +421,17 @@ fn write_moov(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
write_full_box(v, b"mvhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| {
write_mvhd(v, cfg, creation_time)
})?;
for (idx, caps) in cfg.streams.iter().enumerate() {
for (idx, stream) in cfg.streams.iter().enumerate() {
write_box(v, b"trak", |v| {
let mut references = vec![];
// Reference the video track for ONVIF metadata tracks
if cfg.variant == super::Variant::ONVIF
&& caps.structure(0).unwrap().name() == "application/x-onvif-metadata"
&& stream.caps.structure(0).unwrap().name() == "application/x-onvif-metadata"
{
// Find the first video track
for (idx, caps) in cfg.streams.iter().enumerate() {
let s = caps.structure(0).unwrap();
for (idx, other_stream) in cfg.streams.iter().enumerate() {
let s = other_stream.caps.structure(0).unwrap();
if matches!(s.name(), "video/x-h264" | "video/x-h265" | "image/jpeg") {
references.push(TrackReference {
@ -442,7 +443,7 @@ fn write_moov(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
}
}
write_trak(v, cfg, idx, caps, creation_time, &references)
write_trak(v, cfg, idx, stream, creation_time, &references)
})?;
}
write_box(v, b"mvex", |v| write_mvex(v, cfg))?;
@ -480,6 +481,31 @@ fn caps_to_timescale(caps: &gst::CapsRef) -> u32 {
}
}
fn header_stream_to_timescale(stream: &super::HeaderStream) -> u32 {
if stream.trak_timescale > 0 {
stream.trak_timescale
} else {
caps_to_timescale(&stream.caps)
}
}
fn header_configuration_to_timescale(cfg: &super::HeaderConfiguration) -> u32 {
if cfg.movie_timescale > 0 {
cfg.movie_timescale
} else {
// Use the reference track timescale
header_stream_to_timescale(&cfg.streams[0])
}
}
fn fragment_header_stream_to_timescale(stream: &super::FragmentHeaderStream) -> u32 {
if stream.trak_timescale > 0 {
stream.trak_timescale
} else {
caps_to_timescale(&stream.caps)
}
}
fn write_mvhd(
v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration,
@ -489,8 +515,8 @@ fn write_mvhd(
v.extend(creation_time.to_be_bytes());
// Modification time
v.extend(creation_time.to_be_bytes());
// Timescale: uses the reference track timescale
v.extend(caps_to_timescale(&cfg.streams[0]).to_be_bytes());
// Timescale
v.extend(header_configuration_to_timescale(cfg).to_be_bytes());
// Duration
v.extend(0u64.to_be_bytes());
@ -540,7 +566,7 @@ fn write_trak(
v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration,
idx: usize,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
creation_time: u64,
references: &[TrackReference],
) -> Result<(), Error> {
@ -549,13 +575,13 @@ fn write_trak(
b"tkhd",
FULL_BOX_VERSION_1,
TKHD_FLAGS_TRACK_ENABLED | TKHD_FLAGS_TRACK_IN_MOVIE | TKHD_FLAGS_TRACK_IN_PREVIEW,
|v| write_tkhd(v, cfg, idx, caps, creation_time),
|v| write_tkhd(v, cfg, idx, stream, creation_time),
)?;
// TODO: write edts if necessary: for audio tracks to remove initialization samples
// TODO: write edts optionally for negative DTS instead of offsetting the DTS
write_box(v, b"mdia", |v| write_mdia(v, cfg, caps, creation_time))?;
write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?;
if !references.is_empty() {
write_box(v, b"tref", |v| write_tref(v, cfg, references))?;
@ -568,7 +594,7 @@ fn write_tkhd(
v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration,
idx: usize,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
creation_time: u64,
) -> Result<(), Error> {
// Creation time
@ -591,7 +617,7 @@ fn write_tkhd(
v.extend(0u16.to_be_bytes());
// Volume
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
match s.name() {
"audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
v.extend((1u16 << 8).to_be_bytes())
@ -650,19 +676,19 @@ fn write_tkhd(
fn write_mdia(
v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
creation_time: u64,
) -> Result<(), Error> {
write_full_box(v, b"mdhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| {
write_mdhd(v, cfg, caps, creation_time)
write_mdhd(v, cfg, stream, creation_time)
})?;
write_full_box(v, b"hdlr", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_hdlr(v, cfg, caps)
write_hdlr(v, cfg, stream)
})?;
// TODO: write elng if needed
write_box(v, b"minf", |v| write_minf(v, cfg, caps))?;
write_box(v, b"minf", |v| write_minf(v, cfg, stream))?;
Ok(())
}
@ -699,7 +725,7 @@ fn language_code(lang: impl std::borrow::Borrow<[u8; 3]>) -> u16 {
fn write_mdhd(
v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
creation_time: u64,
) -> Result<(), Error> {
// Creation time
@ -707,7 +733,7 @@ fn write_mdhd(
// Modification time
v.extend(creation_time.to_be_bytes());
// Timescale
v.extend(caps_to_timescale(caps).to_be_bytes());
v.extend(header_stream_to_timescale(stream).to_be_bytes());
// Duration
v.extend(0u64.to_be_bytes());
@ -724,12 +750,12 @@ fn write_mdhd(
fn write_hdlr(
v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
// Pre-defined
v.extend([0u8; 4]);
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
let (handler_type, name) = match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
(b"vide", b"VideoHandler\0".as_slice())
@ -756,9 +782,9 @@ fn write_hdlr(
fn write_minf(
v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
@ -780,7 +806,7 @@ fn write_minf(
write_box(v, b"dinf", |v| write_dinf(v, cfg))?;
write_box(v, b"stbl", |v| write_stbl(v, cfg, caps))?;
write_box(v, b"stbl", |v| write_stbl(v, cfg, stream))?;
Ok(())
}
@ -833,10 +859,10 @@ fn write_dref(v: &mut Vec<u8>, _cfg: &super::HeaderConfiguration) -> Result<(),
fn write_stbl(
v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
write_full_box(v, b"stsd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_stsd(v, cfg, caps)
write_stsd(v, cfg, stream)
})?;
write_full_box(v, b"stts", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_stts(v, cfg)
@ -853,14 +879,10 @@ fn write_stbl(
})?;
// For video write a sync sample box as indication that not all samples are sync samples
let s = caps.structure(0).unwrap();
match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" => {
write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_stss(v, cfg)
})?
}
_ => (),
if !stream.delta_frames.intra_only() {
write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_stss(v, cfg)
})?
}
Ok(())
@ -869,20 +891,20 @@ fn write_stbl(
fn write_stsd(
v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
// Entry count
v.extend(1u32.to_be_bytes());
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
write_visual_sample_entry(v, cfg, caps)?
write_visual_sample_entry(v, cfg, stream)?
}
"audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
write_audio_sample_entry(v, cfg, caps)?
write_audio_sample_entry(v, cfg, stream)?
}
"application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, caps)?,
"application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, stream)?,
_ => unreachable!(),
}
@ -908,9 +930,9 @@ fn write_sample_entry_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>(
fn write_visual_sample_entry(
v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
let fourcc = match s.name() {
"video/x-h264" => {
let stream_format = s.get::<&str>("stream-format").context("no stream-format")?;
@ -1146,7 +1168,7 @@ fn write_visual_sample_entry(
#[cfg(feature = "v1_18")]
{
if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(caps) {
if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(&stream.caps) {
write_box(v, b"clli", move |v| {
v.extend((cll.max_content_light_level() as u16).to_be_bytes());
v.extend((cll.max_frame_average_light_level() as u16).to_be_bytes());
@ -1154,7 +1176,7 @@ fn write_visual_sample_entry(
})?;
}
if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(caps) {
if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(&stream.caps) {
write_box(v, b"mdcv", move |v| {
for primary in mastering.display_primaries() {
v.extend(primary.x.to_be_bytes());
@ -1211,9 +1233,9 @@ fn write_visual_sample_entry(
fn write_audio_sample_entry(
v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
let fourcc = match s.name() {
"audio/mpeg" => b"mp4a",
"audio/x-opus" => b"Opus",
@ -1275,7 +1297,7 @@ fn write_audio_sample_entry(
write_esds_aac(v, &map)?;
}
"audio/x-opus" => {
write_dops(v, caps)?;
write_dops(v, &stream.caps)?;
}
"audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
// Nothing to do here
@ -1412,7 +1434,7 @@ fn write_esds_aac(v: &mut Vec<u8>, codec_data: &[u8]) -> Result<(), Error> {
)
}
fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> {
fn write_dops(v: &mut Vec<u8>, caps: &gst::Caps) -> Result<(), Error> {
let rate;
let channels;
let channel_mapping_family;
@ -1442,11 +1464,6 @@ fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> {
) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping))
.unwrap();
} else {
// FIXME: Workaround for below function taking a &Caps instead of &CapsRef
// SAFETY: This is OK because we only get an immutable reference and don't
// clone it, so nobody will be able to get a mutable reference to the caps.
let caps = unsafe { &*(&caps as *const &gst::CapsRef as *const gst::Caps) };
(
rate,
channels,
@ -1479,9 +1496,9 @@ fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> {
fn write_xml_meta_data_sample_entry(
v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef,
stream: &super::HeaderStream,
) -> Result<(), Error> {
let s = caps.structure(0).unwrap();
let s = stream.caps.structure(0).unwrap();
let namespace = match s.name() {
"application/x-onvif-metadata" => b"http://www.onvif.org/ver10/schema",
_ => unreachable!(),
@ -1560,7 +1577,7 @@ fn write_mvex(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
}
}
for (idx, _caps) in cfg.streams.iter().enumerate() {
for (idx, _stream) in cfg.streams.iter().enumerate() {
write_full_box(v, b"trex", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_trex(v, cfg, idx)
})?;
@ -1571,7 +1588,7 @@ fn write_mvex(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
fn write_mehd(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), Error> {
// Use the reference track timescale
let timescale = caps_to_timescale(&cfg.streams[0]);
let timescale = header_configuration_to_timescale(cfg);
let duration = cfg
.duration
@ -1614,7 +1631,7 @@ pub(super) fn create_fmp4_fragment_header(
let mut v = vec![];
let (brand, compatible_brands) =
brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.0));
brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.caps));
write_box(&mut v, b"styp", |v| {
// major brand
@ -1665,15 +1682,14 @@ fn write_moof(
})?;
let mut data_offset_offsets = vec![];
for (idx, (caps, timing_info)) in cfg.streams.iter().enumerate() {
for (idx, stream) in cfg.streams.iter().enumerate() {
// Skip tracks without any buffers for this fragment.
let timing_info = match timing_info {
None => continue,
Some(ref timing_info) => timing_info,
};
if stream.start_time.is_none() {
continue;
}
write_box(v, b"traf", |v| {
write_traf(v, cfg, &mut data_offset_offsets, idx, caps, timing_info)
write_traf(v, cfg, &mut data_offset_offsets, idx, stream)
})?;
}
@ -1688,11 +1704,8 @@ fn write_mfhd(v: &mut Vec<u8>, cfg: &super::FragmentHeaderConfiguration) -> Resu
#[allow(clippy::identity_op)]
#[allow(clippy::bool_to_int_with_if)]
fn sample_flags_from_buffer(
timing_info: &super::FragmentTimingInfo,
buffer: &gst::BufferRef,
) -> u32 {
if timing_info.delta_frames.intra_only() {
fn sample_flags_from_buffer(stream: &super::FragmentHeaderStream, buffer: &gst::BufferRef) -> u32 {
if stream.delta_frames.intra_only() {
(0b00u32 << (16 + 10)) | // leading: unknown
(0b10u32 << (16 + 8)) | // depends: no
(0b10u32 << (16 + 6)) | // depended: no
@ -1743,7 +1756,7 @@ const SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT: u32 = 0x8_00;
fn analyze_buffers(
cfg: &super::FragmentHeaderConfiguration,
idx: usize,
timing_info: &super::FragmentTimingInfo,
stream: &super::FragmentHeaderStream,
timescale: u32,
) -> Result<
(
@ -1802,7 +1815,7 @@ fn analyze_buffers(
tr_flags |= SAMPLE_DURATION_PRESENT;
}
let f = sample_flags_from_buffer(timing_info, buffer);
let f = sample_flags_from_buffer(stream, buffer);
if first_buffer_flags.is_none() {
first_buffer_flags = Some(f);
} else {
@ -1818,7 +1831,7 @@ fn analyze_buffers(
}
if let Some(composition_time_offset) = *composition_time_offset {
assert!(timing_info.delta_frames.requires_dts());
assert!(stream.delta_frames.requires_dts());
if composition_time_offset != 0 {
tr_flags |= SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT;
}
@ -1869,10 +1882,9 @@ fn write_traf(
cfg: &super::FragmentHeaderConfiguration,
data_offset_offsets: &mut Vec<usize>,
idx: usize,
caps: &gst::CapsRef,
timing_info: &super::FragmentTimingInfo,
stream: &super::FragmentHeaderStream,
) -> Result<(), Error> {
let timescale = caps_to_timescale(caps);
let timescale = fragment_header_stream_to_timescale(stream);
// Analyze all buffers to know what values can be put into the tfhd for all samples and what
// has to be stored for every single sample
@ -1883,7 +1895,7 @@ fn write_traf(
default_duration,
default_flags,
negative_composition_time_offsets,
) = analyze_buffers(cfg, idx, timing_info, timescale)?;
) = analyze_buffers(cfg, idx, stream, timescale)?;
assert!((tf_flags & DEFAULT_SAMPLE_SIZE_PRESENT == 0) ^ default_size.is_some());
assert!((tf_flags & DEFAULT_SAMPLE_DURATION_PRESENT == 0) ^ default_duration.is_some());
@ -1893,7 +1905,7 @@ fn write_traf(
write_tfhd(v, cfg, idx, default_size, default_duration, default_flags)
})?;
write_full_box(v, b"tfdt", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| {
write_tfdt(v, cfg, idx, timing_info, timescale)
write_tfdt(v, cfg, idx, stream, timescale)
})?;
let mut current_data_offset = 0;
@ -1923,7 +1935,7 @@ fn write_traf(
current_data_offset,
tr_flags,
timescale,
timing_info,
stream,
run,
)
},
@ -1973,11 +1985,12 @@ fn write_tfdt(
v: &mut Vec<u8>,
_cfg: &super::FragmentHeaderConfiguration,
_idx: usize,
timing_info: &super::FragmentTimingInfo,
stream: &super::FragmentHeaderStream,
timescale: u32,
) -> Result<(), Error> {
let base_time = timing_info
let base_time = stream
.start_time
.unwrap()
.mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("base time overflow")?;
@ -1993,7 +2006,7 @@ fn write_trun(
current_data_offset: u32,
tr_flags: u32,
timescale: u32,
timing_info: &super::FragmentTimingInfo,
stream: &super::FragmentHeaderStream,
buffers: &[Buffer],
) -> Result<usize, Error> {
// Sample count
@ -2004,7 +2017,7 @@ fn write_trun(
v.extend(current_data_offset.to_be_bytes());
if (tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) != 0 {
v.extend(sample_flags_from_buffer(timing_info, &buffers[0].buffer).to_be_bytes());
v.extend(sample_flags_from_buffer(stream, &buffers[0].buffer).to_be_bytes());
}
for Buffer {
@ -2036,7 +2049,7 @@ fn write_trun(
assert!((tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) == 0);
// Sample flags
v.extend(sample_flags_from_buffer(timing_info, buffer).to_be_bytes());
v.extend(sample_flags_from_buffer(stream, buffer).to_be_bytes());
}
if (tr_flags & SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT) != 0 {

238
mux/fmp4/src/fmp4mux/imp.rs
View file

@ -76,6 +76,7 @@ struct Settings {
write_mehd: bool,
interleave_bytes: Option<u64>,
interleave_time: Option<gst::ClockTime>,
movie_timescale: u32,
}
impl Default for Settings {
@ -87,6 +88,7 @@ impl Default for Settings {
write_mehd: DEFAULT_WRITE_MEHD,
interleave_bytes: DEFAULT_INTERLEAVE_BYTES,
interleave_time: DEFAULT_INTERLEAVE_TIME,
movie_timescale: 0,
}
}
}
@ -122,7 +124,7 @@ struct Gop {
}
struct Stream {
sinkpad: gst_base::AggregatorPad,
sinkpad: super::FMP4MuxPad,
caps: gst::Caps,
delta_frames: DeltaFrames,
@ -617,11 +619,7 @@ impl FMP4Mux {
) -> Result<
(
// Drained streams
Vec<(
gst::Caps,
Option<super::FragmentTimingInfo>,
VecDeque<Buffer>,
)>,
Vec<(super::FragmentHeaderStream, VecDeque<Buffer>)>,
// Minimum earliest PTS position of all streams
Option<gst::ClockTime>,
// Minimum earliest PTS of all streams
@ -658,6 +656,8 @@ impl FMP4Mux {
);
for (idx, stream) in state.streams.iter_mut().enumerate() {
let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
assert!(
timeout
|| at_eos
@ -742,7 +742,16 @@ impl FMP4Mux {
"Draining no buffers",
);
drained_streams.push((stream.caps.clone(), None, VecDeque::new()));
drained_streams.push((
super::FragmentHeaderStream {
caps: stream.caps.clone(),
start_time: None,
delta_frames: stream.delta_frames,
trak_timescale: stream_settings.trak_timescale,
},
VecDeque::new(),
));
continue;
}
@ -876,11 +885,12 @@ impl FMP4Mux {
}
drained_streams.push((
stream.caps.clone(),
Some(super::FragmentTimingInfo {
start_time,
super::FragmentHeaderStream {
caps: stream.caps.clone(),
start_time: Some(start_time),
delta_frames: stream.delta_frames,
}),
trak_timescale: stream_settings.trak_timescale,
},
buffers,
));
}
@ -897,11 +907,7 @@ impl FMP4Mux {
fn preprocess_drained_streams_onvif(
&self,
state: &mut State,
drained_streams: &mut [(
gst::Caps,
Option<super::FragmentTimingInfo>,
VecDeque<Buffer>,
)],
drained_streams: &mut [(super::FragmentHeaderStream, VecDeque<Buffer>)],
) -> Result<Option<gst::ClockTime>, gst::FlowError> {
let aggregator = self.obj();
if aggregator.class().as_ref().variant != super::Variant::ONVIF {
@ -925,7 +931,7 @@ impl FMP4Mux {
// If this is the first fragment then allow the first buffers to not have a reference
// timestamp meta and backdate them
if state.stream_header.is_none() {
for (idx, (_, _, drain_buffers)) in drained_streams.iter_mut().enumerate() {
for (idx, (_, drain_buffers)) in drained_streams.iter_mut().enumerate() {
let (buffer_idx, utc_time, buffer) =
match drain_buffers.iter().enumerate().find_map(|(idx, buffer)| {
get_utc_time_from_buffer(&buffer.buffer)
@ -979,7 +985,7 @@ impl FMP4Mux {
if state.start_utc_time.is_none() {
let mut start_utc_time = None;
for (idx, (_, _, drain_buffers)) in drained_streams.iter().enumerate() {
for (idx, (_, drain_buffers)) in drained_streams.iter().enumerate() {
for buffer in drain_buffers {
let utc_time = match get_utc_time_from_buffer(&buffer.buffer) {
None => {
@ -1010,7 +1016,7 @@ impl FMP4Mux {
// Update all buffer timestamps based on the UTC time and offset to the start UTC time
let start_utc_time = state.start_utc_time.unwrap();
for (idx, (_, timing_info, drain_buffers)) in drained_streams.iter_mut().enumerate() {
for (idx, (stream, drain_buffers)) in drained_streams.iter_mut().enumerate() {
let mut start_time = None;
for buffer in drain_buffers.iter_mut() {
@ -1128,9 +1134,9 @@ impl FMP4Mux {
if let Some(start_time) = start_time {
gst::debug!(CAT, obj: state.streams[idx].sinkpad, "Fragment starting at UTC time {}", start_time);
timing_info.as_mut().unwrap().start_time = start_time;
*stream.start_time.as_mut().unwrap() = start_time;
} else {
assert!(timing_info.is_none());
assert!(stream.start_time.is_none());
}
}
@ -1141,35 +1147,28 @@ impl FMP4Mux {
fn interleave_buffers(
&self,
settings: &Settings,
mut drained_streams: Vec<(
gst::Caps,
Option<super::FragmentTimingInfo>,
VecDeque<Buffer>,
)>,
) -> Result<
(
Vec<Buffer>,
Vec<(gst::Caps, Option<super::FragmentTimingInfo>)>,
),
gst::FlowError,
> {
mut drained_streams: Vec<(super::FragmentHeaderStream, VecDeque<Buffer>)>,
) -> Result<(Vec<Buffer>, Vec<super::FragmentHeaderStream>), gst::FlowError> {
let mut interleaved_buffers =
Vec::with_capacity(drained_streams.iter().map(|(_, _, bufs)| bufs.len()).sum());
while let Some((_idx, (_, _, bufs))) = drained_streams.iter_mut().enumerate().min_by(
|(a_idx, (_, _, a)), (b_idx, (_, _, b))| {
let (a, b) = match (a.front(), b.front()) {
(None, None) => return std::cmp::Ordering::Equal,
(None, _) => return std::cmp::Ordering::Greater,
(_, None) => return std::cmp::Ordering::Less,
(Some(a), Some(b)) => (a, b),
};
Vec::with_capacity(drained_streams.iter().map(|(_, bufs)| bufs.len()).sum());
while let Some((_idx, (_, bufs))) =
drained_streams
.iter_mut()
.enumerate()
.min_by(|(a_idx, (_, a)), (b_idx, (_, b))| {
let (a, b) = match (a.front(), b.front()) {
(None, None) => return std::cmp::Ordering::Equal,
(None, _) => return std::cmp::Ordering::Greater,
(_, None) => return std::cmp::Ordering::Less,
(Some(a), Some(b)) => (a, b),
};
match a.timestamp.cmp(&b.timestamp) {
std::cmp::Ordering::Equal => a_idx.cmp(b_idx),
cmp => cmp,
}
},
) {
match a.timestamp.cmp(&b.timestamp) {
std::cmp::Ordering::Equal => a_idx.cmp(b_idx),
cmp => cmp,
}
})
{
let start_time = match bufs.front() {
None => {
// No more buffers now
@ -1201,11 +1200,11 @@ impl FMP4Mux {
}
// All buffers should be consumed now
assert!(drained_streams.iter().all(|(_, _, bufs)| bufs.is_empty()));
assert!(drained_streams.iter().all(|(_, bufs)| bufs.is_empty()));
let streams = drained_streams
.into_iter()
.map(|(caps, timing_info, _)| (caps, timing_info))
.map(|(stream, _)| stream)
.collect::<Vec<_>>();
Ok((interleaved_buffers, streams))
@ -1217,7 +1216,7 @@ impl FMP4Mux {
settings: &Settings,
timeout: bool,
at_eos: bool,
upstream_events: &mut Vec<(gst_base::AggregatorPad, gst::Event)>,
upstream_events: &mut Vec<(super::FMP4MuxPad, gst::Event)>,
) -> Result<(Option<gst::Caps>, Option<gst::BufferList>), gst::FlowError> {
if at_eos {
gst::info!(CAT, imp: self, "Draining at EOS");
@ -1241,7 +1240,7 @@ impl FMP4Mux {
) = self.drain_buffers(state, settings, timeout, at_eos)?;
// Remove all GAP buffers before processing them further
for (_, timing_info, buffers) in &mut drained_streams {
for (stream, buffers) in &mut drained_streams {
buffers.retain(|buf| {
!buf.buffer.flags().contains(gst::BufferFlags::GAP)
|| !buf.buffer.flags().contains(gst::BufferFlags::DROPPABLE)
@ -1249,7 +1248,7 @@ impl FMP4Mux {
});
if buffers.is_empty() {
*timing_info = None;
stream.start_time = None;
}
}
@ -1371,9 +1370,13 @@ impl FMP4Mux {
// Write mfra only for the main stream, and if there are no buffers for the main stream
// in this segment then don't write anything.
if let Some((_caps, Some(ref timing_info))) = streams.get(0) {
if let Some(super::FragmentHeaderStream {
start_time: Some(start_time),
..
}) = streams.get(0)
{
state.fragment_offsets.push(super::FragmentOffset {
time: timing_info.start_time,
time: *start_time,
offset: moof_offset,
});
}
@ -1432,7 +1435,7 @@ impl FMP4Mux {
if settings.write_mfra && at_eos {
gst::debug!(CAT, imp: self, "Writing mfra box");
match boxes::create_mfra(&streams[0].0, &state.fragment_offsets) {
match boxes::create_mfra(&streams[0].caps, &state.fragment_offsets) {
Ok(mut mfra) => {
{
let mfra = mfra.get_mut().unwrap();
@ -1462,7 +1465,7 @@ impl FMP4Mux {
.obj()
.sink_pads()
.into_iter()
.map(|pad| pad.downcast::<gst_base::AggregatorPad>().unwrap())
.map(|pad| pad.downcast::<super::FMP4MuxPad>().unwrap())
{
let caps = match pad.current_caps() {
Some(caps) => caps,
@ -1599,13 +1602,18 @@ impl FMP4Mux {
let streams = state
.streams
.iter()
.map(|s| s.caps.clone())
.map(|s| super::HeaderStream {
trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale,
delta_frames: s.delta_frames,
caps: s.caps.clone(),
})
.collect::<Vec<_>>();
let mut buffer = boxes::create_fmp4_header(super::HeaderConfiguration {
variant,
update: at_eos,
streams: streams.as_slice(),
movie_timescale: settings.movie_timescale,
streams,
write_mehd: settings.write_mehd,
duration: if at_eos { duration } else { None },
start_utc_time: state
@ -1696,6 +1704,11 @@ impl ObjectImpl for FMP4Mux {
.default_value(DEFAULT_INTERLEAVE_TIME.map(gst::ClockTime::nseconds).unwrap_or(u64::MAX))
.mutable_ready()
.build(),
glib::ParamSpecUInt::builder("movie-timescale")
.nick("Movie Timescale")
.blurb("Timescale to use for the movie (units per second, 0 is automatic)")
.mutable_ready()
.build(),
]
});
@ -1745,6 +1758,11 @@ impl ObjectImpl for FMP4Mux {
};
}
"movie-timescale" => {
let mut settings = self.settings.lock().unwrap();
settings.movie_timescale = value.get().expect("type checked upstream");
}
_ => unimplemented!(),
}
}
@ -1781,6 +1799,11 @@ impl ObjectImpl for FMP4Mux {
settings.interleave_time.to_value()
}
"movie-timescale" => {
let settings = self.settings.lock().unwrap();
settings.movie_timescale.to_value()
}
_ => unimplemented!(),
}
}
@ -1954,8 +1977,6 @@ impl AggregatorImpl for FMP4Mux {
}
fn flush(&self) -> Result<gst::FlowSuccess, gst::FlowError> {
self.parent_flush()?;
let mut state = self.state.lock().unwrap();
for stream in &mut state.streams {
@ -1969,7 +1990,9 @@ impl AggregatorImpl for FMP4Mux {
state.current_offset = 0;
state.fragment_offsets.clear();
Ok(gst::FlowSuccess::Ok)
drop(state);
self.parent_flush()
}
fn stop(&self) -> Result<(), gst::ErrorMessage> {
@ -2346,7 +2369,7 @@ impl ElementImpl for ISOFMP4Mux {
)
.unwrap();
let sink_pad_template = gst::PadTemplate::new(
let sink_pad_template = gst::PadTemplate::with_gtype(
"sink_%u",
gst::PadDirection::Sink,
gst::PadPresence::Request,
@ -2385,6 +2408,7 @@ impl ElementImpl for ISOFMP4Mux {
]
.into_iter()
.collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
)
.unwrap();
@ -2441,7 +2465,7 @@ impl ElementImpl for CMAFMux {
)
.unwrap();
let sink_pad_template = gst::PadTemplate::new(
let sink_pad_template = gst::PadTemplate::with_gtype(
"sink",
gst::PadDirection::Sink,
gst::PadPresence::Always,
@ -2467,6 +2491,7 @@ impl ElementImpl for CMAFMux {
]
.into_iter()
.collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
)
.unwrap();
@ -2523,7 +2548,7 @@ impl ElementImpl for DASHMP4Mux {
)
.unwrap();
let sink_pad_template = gst::PadTemplate::new(
let sink_pad_template = gst::PadTemplate::with_gtype(
"sink",
gst::PadDirection::Sink,
gst::PadPresence::Always,
@ -2562,6 +2587,7 @@ impl ElementImpl for DASHMP4Mux {
]
.into_iter()
.collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
)
.unwrap();
@ -2618,7 +2644,7 @@ impl ElementImpl for ONVIFFMP4Mux {
)
.unwrap();
let sink_pad_template = gst::PadTemplate::new(
let sink_pad_template = gst::PadTemplate::with_gtype(
"sink_%u",
gst::PadDirection::Sink,
gst::PadPresence::Request,
@ -2665,6 +2691,7 @@ impl ElementImpl for ONVIFFMP4Mux {
]
.into_iter()
.collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
)
.unwrap();
@ -2680,3 +2707,82 @@ impl AggregatorImpl for ONVIFFMP4Mux {}
impl FMP4MuxImpl for ONVIFFMP4Mux {
const VARIANT: super::Variant = super::Variant::ONVIF;
}
#[derive(Default, Clone)]
struct PadSettings {
trak_timescale: u32,
}
#[derive(Default)]
pub(crate) struct FMP4MuxPad {
settings: Mutex<PadSettings>,
}
#[glib::object_subclass]
impl ObjectSubclass for FMP4MuxPad {
const NAME: &'static str = "GstFMP4MuxPad";
type Type = super::FMP4MuxPad;
type ParentType = gst_base::AggregatorPad;
}
impl ObjectImpl for FMP4MuxPad {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![glib::ParamSpecUInt::builder("trak-timescale")
.nick("Track Timescale")
.blurb("Timescale to use for the track (units per second, 0 is automatic)")
.mutable_ready()
.build()]
});
&PROPERTIES
}
fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) {
match pspec.name() {
"trak-timescale" => {
let mut settings = self.settings.lock().unwrap();
settings.trak_timescale = value.get().expect("type checked upstream");
}
_ => unimplemented!(),
}
}
fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
match pspec.name() {
"trak-timescale" => {
let settings = self.settings.lock().unwrap();
settings.trak_timescale.to_value()
}
_ => unimplemented!(),
}
}
}
impl GstObjectImpl for FMP4MuxPad {}
impl PadImpl for FMP4MuxPad {}
impl AggregatorPadImpl for FMP4MuxPad {
fn flush(&self, aggregator: &gst_base::Aggregator) -> Result<gst::FlowSuccess, gst::FlowError> {
let mux = aggregator.downcast_ref::<super::FMP4Mux>().unwrap();
let mut mux_state = mux.imp().state.lock().unwrap();
for stream in &mut mux_state.streams {
if stream.sinkpad == *self.obj() {
stream.queued_gops.clear();
stream.dts_offset = None;
stream.current_position = gst::ClockTime::ZERO;
stream.current_utc_time = gst::ClockTime::ZERO;
stream.fragment_filled = false;
break;
}
}
drop(mux_state);
self.parent_flush(aggregator)
}
}

54
mux/fmp4/src/fmp4mux/mod.rs
View file

@ -12,6 +12,10 @@ use gst::prelude::*;
mod boxes;
mod imp;
glib::wrapper! {
pub(crate) struct FMP4MuxPad(ObjectSubclass<imp::FMP4MuxPad>) @extends gst_base::AggregatorPad, gst::Pad, gst::Object;
}
glib::wrapper! {
pub(crate) struct FMP4Mux(ObjectSubclass<imp::FMP4Mux>) @extends gst_base::Aggregator, gst::Element, gst::Object;
}
@ -33,8 +37,12 @@ glib::wrapper! {
}
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
FMP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
HeaderUpdateMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
#[cfg(feature = "doc")]
{
FMP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
FMP4MuxPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
HeaderUpdateMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
}
gst::Element::register(
Some(plugin),
"isofmp4mux",
@ -64,33 +72,63 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
}
#[derive(Debug)]
pub(crate) struct HeaderConfiguration<'a> {
pub(crate) struct HeaderConfiguration {
variant: Variant,
update: bool,
/// Pre-defined movie timescale if not 0.
movie_timescale: u32,
/// First caps must be the video/reference stream. Must be in the order the tracks are going to
/// be used later for the fragments too.
streams: &'a [gst::Caps],
streams: Vec<HeaderStream>,
write_mehd: bool,
duration: Option<gst::ClockTime>,
/// Start UTC time in ONVIF mode.
/// Since Jan 1 1601 in 100ns units.
start_utc_time: Option<u64>,
}
#[derive(Debug)]
pub(crate) struct HeaderStream {
/// Caps of this stream
caps: gst::Caps,
/// Set if this is an intra-only stream
delta_frames: DeltaFrames,
/// Pre-defined trak timescale if not 0.
trak_timescale: u32,
}
#[derive(Debug)]
pub(crate) struct FragmentHeaderConfiguration<'a> {
variant: Variant,
/// Sequence number for this fragment.
sequence_number: u32,
streams: &'a [(gst::Caps, Option<FragmentTimingInfo>)],
streams: &'a [FragmentHeaderStream],
buffers: &'a [Buffer],
}
#[derive(Debug)]
pub(crate) struct FragmentTimingInfo {
/// Start time of this fragment
start_time: gst::ClockTime,
pub(crate) struct FragmentHeaderStream {
/// Caps of this stream
caps: gst::Caps,
/// Set if this is an intra-only stream
delta_frames: DeltaFrames,
/// Pre-defined trak timescale if not 0.
trak_timescale: u32,
/// Start time of this fragment
///
/// `None` if this stream has no buffers in this fragment.
start_time: Option<gst::ClockTime>,
}
#[derive(Debug, Copy, Clone)]