mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-06-02 13:29:50 +00:00
Code Issues Projects Releases Wiki Activity

fmp4mux: Make media/trak timescales configurable

Browse source 
And refactor a bit of code for easier extensibility.
This commit is contained in:
Sebastian Dröge 2022-11-07 19:47:31 +02:00 committed by Sebastian Dröge
parent e87251c7d9
commit f062b7cf0d
5 changed files with 363 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
docs/plugins/gst_plugins_cache.json
View file

@ -1589,7 +1589,8 @@
"sink": { "sink": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n",
"direction": "sink", "direction": "sink",
"presence": "always" "presence": "always",
"type": "GstFMP4MuxPad"
}, },
"src": { "src": {
"caps": "video/quicktime:\n variant: cmaf\n", "caps": "video/quicktime:\n variant: cmaf\n",
@ -1617,7 +1618,8 @@
"sink": { "sink": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n",
"direction": "sink", "direction": "sink",
"presence": "always" "presence": "always",
"type": "GstFMP4MuxPad"
}, },
"src": { "src": {
"caps": "video/quicktime:\n variant: iso-fragmented\n", "caps": "video/quicktime:\n variant: iso-fragmented\n",
@ -1645,7 +1647,8 @@
"sink_%%u": { "sink_%%u": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n",
"direction": "sink", "direction": "sink",
"presence": "request" "presence": "request",
"type": "GstFMP4MuxPad"
}, },
"src": { "src": {
"caps": "video/quicktime:\n variant: iso-fragmented\n", "caps": "video/quicktime:\n variant: iso-fragmented\n",
@ -1673,7 +1676,8 @@
"sink_%%u": { "sink_%%u": {
"caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nimage/jpeg:\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-alaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-mulaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-adpcm:\n layout: g726\n channels: 1\n rate: 8000\n bitrate: { (int)16000, (int)24000, (int)32000, (int)40000 }\napplication/x-onvif-metadata:\n parsed: true\n", "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nimage/jpeg:\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-alaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-mulaw:\n channels: [ 1, 2 ]\n rate: [ 1, 2147483647 ]\naudio/x-adpcm:\n layout: g726\n channels: 1\n rate: 8000\n bitrate: { (int)16000, (int)24000, (int)32000, (int)40000 }\napplication/x-onvif-metadata:\n parsed: true\n",
"direction": "sink", "direction": "sink",
"presence": "request" "presence": "request",
"type": "GstFMP4MuxPad"
}, },
"src": { "src": {
"caps": "video/quicktime:\n variant: iso-fragmented\n", "caps": "video/quicktime:\n variant: iso-fragmented\n",
@ -1752,6 +1756,20 @@
"type": "guint64", "type": "guint64",
"writable": true "writable": true
}, },
"movie-timescale": {
"blurb": "Timescale to use for the movie (units per second, 0 is automatic)",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "0",
"max": "-1",
"min": "0",
"mutable": "ready",
"readable": true,
"type": "guint",
"writable": true
},
"write-mehd": { "write-mehd": {
"blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)", "blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)",
"conditionally-available": false, "conditionally-available": false,
@ -1797,6 +1815,33 @@
"value": "2" "value": "2"
} }
] ]
},
"GstFMP4MuxPad": {
"hierarchy": [
"GstFMP4MuxPad",
"GstAggregatorPad",
"GstPad",
"GstObject",
"GInitiallyUnowned",
"GObject"
],
"kind": "object",
"properties": {
"trak-timescale": {
"blurb": "Timescale to use for the track (units per second, 0 is automatic)",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "0",
"max": "-1",
"min": "0",
"mutable": "ready",
"readable": true,
"type": "guint",
"writable": true
}
}
} }
}, },
"package": "gst-plugin-fmp4", "package": "gst-plugin-fmp4",

1
mux/fmp4/Cargo.toml
View file

@ -37,6 +37,7 @@ default = ["v1_18"]
static = [] static = []
capi = [] capi = []
v1_18 = ["gst-video/v1_18"] v1_18 = ["gst-video/v1_18"]
doc = ["gst/v1_18"]
[package.metadata.capi] [package.metadata.capi]
min_version = "0.8.0" min_version = "0.8.0"

177
mux/fmp4/src/fmp4mux/boxes.rs
View file

@ -355,7 +355,8 @@ fn brands_from_variant_and_caps<'a>(
pub(super) fn create_fmp4_header(cfg: super::HeaderConfiguration) -> Result<gst::Buffer, Error> { pub(super) fn create_fmp4_header(cfg: super::HeaderConfiguration) -> Result<gst::Buffer, Error> {
let mut v = vec![]; let mut v = vec![];
let (brand, compatible_brands) = brands_from_variant_and_caps(cfg.variant, cfg.streams.iter()); let (brand, compatible_brands) =
brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.caps));
write_box(&mut v, b"ftyp", |v| { write_box(&mut v, b"ftyp", |v| {
// major brand // major brand
@ -420,17 +421,17 @@ fn write_moov(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
write_full_box(v, b"mvhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"mvhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| {
write_mvhd(v, cfg, creation_time) write_mvhd(v, cfg, creation_time)
})?; })?;
for (idx, caps) in cfg.streams.iter().enumerate() { for (idx, stream) in cfg.streams.iter().enumerate() {
write_box(v, b"trak", |v| { write_box(v, b"trak", |v| {
let mut references = vec![]; let mut references = vec![];
// Reference the video track for ONVIF metadata tracks // Reference the video track for ONVIF metadata tracks
if cfg.variant == super::Variant::ONVIF if cfg.variant == super::Variant::ONVIF
&& caps.structure(0).unwrap().name() == "application/x-onvif-metadata" && stream.caps.structure(0).unwrap().name() == "application/x-onvif-metadata"
{ {
// Find the first video track // Find the first video track
for (idx, caps) in cfg.streams.iter().enumerate() { for (idx, other_stream) in cfg.streams.iter().enumerate() {
let s = caps.structure(0).unwrap(); let s = other_stream.caps.structure(0).unwrap();
if matches!(s.name(), "video/x-h264" | "video/x-h265" | "image/jpeg") { if matches!(s.name(), "video/x-h264" | "video/x-h265" | "image/jpeg") {
references.push(TrackReference { references.push(TrackReference {
@ -442,7 +443,7 @@ fn write_moov(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
} }
} }
write_trak(v, cfg, idx, caps, creation_time, &references) write_trak(v, cfg, idx, stream, creation_time, &references)
})?; })?;
} }
write_box(v, b"mvex", |v| write_mvex(v, cfg))?; write_box(v, b"mvex", |v| write_mvex(v, cfg))?;
@ -480,6 +481,31 @@ fn caps_to_timescale(caps: &gst::CapsRef) -> u32 {
} }
} }
fn header_stream_to_timescale(stream: &super::HeaderStream) -> u32 {
if stream.trak_timescale > 0 {
stream.trak_timescale
} else {
caps_to_timescale(&stream.caps)
}
}
fn header_configuration_to_timescale(cfg: &super::HeaderConfiguration) -> u32 {
if cfg.movie_timescale > 0 {
cfg.movie_timescale
} else {
// Use the reference track timescale
header_stream_to_timescale(&cfg.streams[0])
}
}
fn fragment_header_stream_to_timescale(stream: &super::FragmentHeaderStream) -> u32 {
if stream.trak_timescale > 0 {
stream.trak_timescale
} else {
caps_to_timescale(&stream.caps)
}
}
fn write_mvhd( fn write_mvhd(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
@ -489,8 +515,8 @@ fn write_mvhd(
v.extend(creation_time.to_be_bytes()); v.extend(creation_time.to_be_bytes());
// Modification time // Modification time
v.extend(creation_time.to_be_bytes()); v.extend(creation_time.to_be_bytes());
// Timescale: uses the reference track timescale // Timescale
v.extend(caps_to_timescale(&cfg.streams[0]).to_be_bytes()); v.extend(header_configuration_to_timescale(cfg).to_be_bytes());
// Duration // Duration
v.extend(0u64.to_be_bytes()); v.extend(0u64.to_be_bytes());
@ -540,7 +566,7 @@ fn write_trak(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
idx: usize, idx: usize,
caps: &gst::CapsRef, stream: &super::HeaderStream,
creation_time: u64, creation_time: u64,
references: &[TrackReference], references: &[TrackReference],
) -> Result<(), Error> { ) -> Result<(), Error> {
@ -549,13 +575,13 @@ fn write_trak(
b"tkhd", b"tkhd",
FULL_BOX_VERSION_1, FULL_BOX_VERSION_1,
TKHD_FLAGS_TRACK_ENABLED | TKHD_FLAGS_TRACK_IN_MOVIE | TKHD_FLAGS_TRACK_IN_PREVIEW, TKHD_FLAGS_TRACK_ENABLED | TKHD_FLAGS_TRACK_IN_MOVIE | TKHD_FLAGS_TRACK_IN_PREVIEW,
|v| write_tkhd(v, cfg, idx, caps, creation_time), |v| write_tkhd(v, cfg, idx, stream, creation_time),
)?; )?;
// TODO: write edts if necessary: for audio tracks to remove initialization samples // TODO: write edts if necessary: for audio tracks to remove initialization samples
// TODO: write edts optionally for negative DTS instead of offsetting the DTS // TODO: write edts optionally for negative DTS instead of offsetting the DTS
write_box(v, b"mdia", |v| write_mdia(v, cfg, caps, creation_time))?; write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?;
if !references.is_empty() { if !references.is_empty() {
write_box(v, b"tref", |v| write_tref(v, cfg, references))?; write_box(v, b"tref", |v| write_tref(v, cfg, references))?;
@ -568,7 +594,7 @@ fn write_tkhd(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration, _cfg: &super::HeaderConfiguration,
idx: usize, idx: usize,
caps: &gst::CapsRef, stream: &super::HeaderStream,
creation_time: u64, creation_time: u64,
) -> Result<(), Error> { ) -> Result<(), Error> {
// Creation time // Creation time
@ -591,7 +617,7 @@ fn write_tkhd(
v.extend(0u16.to_be_bytes()); v.extend(0u16.to_be_bytes());
// Volume // Volume
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
match s.name() { match s.name() {
"audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
v.extend((1u16 << 8).to_be_bytes()) v.extend((1u16 << 8).to_be_bytes())
@ -650,19 +676,19 @@ fn write_tkhd(
fn write_mdia( fn write_mdia(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
creation_time: u64, creation_time: u64,
) -> Result<(), Error> { ) -> Result<(), Error> {
write_full_box(v, b"mdhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"mdhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| {
write_mdhd(v, cfg, caps, creation_time) write_mdhd(v, cfg, stream, creation_time)
})?; })?;
write_full_box(v, b"hdlr", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"hdlr", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_hdlr(v, cfg, caps) write_hdlr(v, cfg, stream)
})?; })?;
// TODO: write elng if needed // TODO: write elng if needed
write_box(v, b"minf", |v| write_minf(v, cfg, caps))?; write_box(v, b"minf", |v| write_minf(v, cfg, stream))?;
Ok(()) Ok(())
} }
@ -699,7 +725,7 @@ fn language_code(lang: impl std::borrow::Borrow<[u8; 3]>) -> u16 {
fn write_mdhd( fn write_mdhd(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration, _cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
creation_time: u64, creation_time: u64,
) -> Result<(), Error> { ) -> Result<(), Error> {
// Creation time // Creation time
@ -707,7 +733,7 @@ fn write_mdhd(
// Modification time // Modification time
v.extend(creation_time.to_be_bytes()); v.extend(creation_time.to_be_bytes());
// Timescale // Timescale
v.extend(caps_to_timescale(caps).to_be_bytes()); v.extend(header_stream_to_timescale(stream).to_be_bytes());
// Duration // Duration
v.extend(0u64.to_be_bytes()); v.extend(0u64.to_be_bytes());
@ -724,12 +750,12 @@ fn write_mdhd(
fn write_hdlr( fn write_hdlr(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration, _cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
// Pre-defined // Pre-defined
v.extend([0u8; 4]); v.extend([0u8; 4]);
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
let (handler_type, name) = match s.name() { let (handler_type, name) = match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
(b"vide", b"VideoHandler\0".as_slice()) (b"vide", b"VideoHandler\0".as_slice())
@ -756,9 +782,9 @@ fn write_hdlr(
fn write_minf( fn write_minf(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
match s.name() { match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
@ -780,7 +806,7 @@ fn write_minf(
write_box(v, b"dinf", |v| write_dinf(v, cfg))?; write_box(v, b"dinf", |v| write_dinf(v, cfg))?;
write_box(v, b"stbl", |v| write_stbl(v, cfg, caps))?; write_box(v, b"stbl", |v| write_stbl(v, cfg, stream))?;
Ok(()) Ok(())
} }
@ -833,10 +859,10 @@ fn write_dref(v: &mut Vec<u8>, _cfg: &super::HeaderConfiguration) -> Result<(),
fn write_stbl( fn write_stbl(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
write_full_box(v, b"stsd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"stsd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_stsd(v, cfg, caps) write_stsd(v, cfg, stream)
})?; })?;
write_full_box(v, b"stts", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"stts", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_stts(v, cfg) write_stts(v, cfg)
@ -853,14 +879,10 @@ fn write_stbl(
})?; })?;
// For video write a sync sample box as indication that not all samples are sync samples // For video write a sync sample box as indication that not all samples are sync samples
let s = caps.structure(0).unwrap(); if !stream.delta_frames.intra_only() {
match s.name() { write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
"video/x-h264" | "video/x-h265" | "video/x-vp9" => { write_stss(v, cfg)
write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { })?
write_stss(v, cfg)
})?
}
_ => (),
} }
Ok(()) Ok(())
@ -869,20 +891,20 @@ fn write_stbl(
fn write_stsd( fn write_stsd(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
// Entry count // Entry count
v.extend(1u32.to_be_bytes()); v.extend(1u32.to_be_bytes());
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
match s.name() { match s.name() {
"video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
write_visual_sample_entry(v, cfg, caps)? write_visual_sample_entry(v, cfg, stream)?
} }
"audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
write_audio_sample_entry(v, cfg, caps)? write_audio_sample_entry(v, cfg, stream)?
} }
"application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, caps)?, "application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, stream)?,
_ => unreachable!(), _ => unreachable!(),
} }
@ -908,9 +930,9 @@ fn write_sample_entry_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>(
fn write_visual_sample_entry( fn write_visual_sample_entry(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration, _cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
let fourcc = match s.name() { let fourcc = match s.name() {
"video/x-h264" => { "video/x-h264" => {
let stream_format = s.get::<&str>("stream-format").context("no stream-format")?; let stream_format = s.get::<&str>("stream-format").context("no stream-format")?;
@ -1146,7 +1168,7 @@ fn write_visual_sample_entry(
#[cfg(feature = "v1_18")] #[cfg(feature = "v1_18")]
{ {
if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(caps) { if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(&stream.caps) {
write_box(v, b"clli", move |v| { write_box(v, b"clli", move |v| {
v.extend((cll.max_content_light_level() as u16).to_be_bytes()); v.extend((cll.max_content_light_level() as u16).to_be_bytes());
v.extend((cll.max_frame_average_light_level() as u16).to_be_bytes()); v.extend((cll.max_frame_average_light_level() as u16).to_be_bytes());
@ -1154,7 +1176,7 @@ fn write_visual_sample_entry(
})?; })?;
} }
if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(caps) { if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(&stream.caps) {
write_box(v, b"mdcv", move |v| { write_box(v, b"mdcv", move |v| {
for primary in mastering.display_primaries() { for primary in mastering.display_primaries() {
v.extend(primary.x.to_be_bytes()); v.extend(primary.x.to_be_bytes());
@ -1211,9 +1233,9 @@ fn write_visual_sample_entry(
fn write_audio_sample_entry( fn write_audio_sample_entry(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration, _cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
let fourcc = match s.name() { let fourcc = match s.name() {
"audio/mpeg" => b"mp4a", "audio/mpeg" => b"mp4a",
"audio/x-opus" => b"Opus", "audio/x-opus" => b"Opus",
@ -1275,7 +1297,7 @@ fn write_audio_sample_entry(
write_esds_aac(v, &map)?; write_esds_aac(v, &map)?;
} }
"audio/x-opus" => { "audio/x-opus" => {
write_dops(v, caps)?; write_dops(v, &stream.caps)?;
} }
"audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
// Nothing to do here // Nothing to do here
@ -1412,7 +1434,7 @@ fn write_esds_aac(v: &mut Vec<u8>, codec_data: &[u8]) -> Result<(), Error> {
) )
} }
fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> { fn write_dops(v: &mut Vec<u8>, caps: &gst::Caps) -> Result<(), Error> {
let rate; let rate;
let channels; let channels;
let channel_mapping_family; let channel_mapping_family;
@ -1442,11 +1464,6 @@ fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> {
) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping)) ) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping))
.unwrap(); .unwrap();
} else { } else {
// FIXME: Workaround for below function taking a &Caps instead of &CapsRef
// SAFETY: This is OK because we only get an immutable reference and don't
// clone it, so nobody will be able to get a mutable reference to the caps.
let caps = unsafe { &*(&caps as *const &gst::CapsRef as *const gst::Caps) };
( (
rate, rate,
channels, channels,
@ -1479,9 +1496,9 @@ fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> {
fn write_xml_meta_data_sample_entry( fn write_xml_meta_data_sample_entry(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::HeaderConfiguration, _cfg: &super::HeaderConfiguration,
caps: &gst::CapsRef, stream: &super::HeaderStream,
) -> Result<(), Error> { ) -> Result<(), Error> {
let s = caps.structure(0).unwrap(); let s = stream.caps.structure(0).unwrap();
let namespace = match s.name() { let namespace = match s.name() {
"application/x-onvif-metadata" => b"http://www.onvif.org/ver10/schema", "application/x-onvif-metadata" => b"http://www.onvif.org/ver10/schema",
_ => unreachable!(), _ => unreachable!(),
@ -1560,7 +1577,7 @@ fn write_mvex(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
} }
} }
for (idx, _caps) in cfg.streams.iter().enumerate() { for (idx, _stream) in cfg.streams.iter().enumerate() {
write_full_box(v, b"trex", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"trex", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_trex(v, cfg, idx) write_trex(v, cfg, idx)
})?; })?;
@ -1571,7 +1588,7 @@ fn write_mvex(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
fn write_mehd(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), Error> { fn write_mehd(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), Error> {
// Use the reference track timescale // Use the reference track timescale
let timescale = caps_to_timescale(&cfg.streams[0]); let timescale = header_configuration_to_timescale(cfg);
let duration = cfg let duration = cfg
.duration .duration
@ -1614,7 +1631,7 @@ pub(super) fn create_fmp4_fragment_header(
let mut v = vec![]; let mut v = vec![];
let (brand, compatible_brands) = let (brand, compatible_brands) =
brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.0)); brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.caps));
write_box(&mut v, b"styp", |v| { write_box(&mut v, b"styp", |v| {
// major brand // major brand
@ -1665,15 +1682,14 @@ fn write_moof(
})?; })?;
let mut data_offset_offsets = vec![]; let mut data_offset_offsets = vec![];
for (idx, (caps, timing_info)) in cfg.streams.iter().enumerate() { for (idx, stream) in cfg.streams.iter().enumerate() {
// Skip tracks without any buffers for this fragment. // Skip tracks without any buffers for this fragment.
let timing_info = match timing_info { if stream.start_time.is_none() {
None => continue, continue;
Some(ref timing_info) => timing_info, }
};
write_box(v, b"traf", |v| { write_box(v, b"traf", |v| {
write_traf(v, cfg, &mut data_offset_offsets, idx, caps, timing_info) write_traf(v, cfg, &mut data_offset_offsets, idx, stream)
})?; })?;
} }
@ -1688,11 +1704,8 @@ fn write_mfhd(v: &mut Vec<u8>, cfg: &super::FragmentHeaderConfiguration) -> Resu
#[allow(clippy::identity_op)] #[allow(clippy::identity_op)]
#[allow(clippy::bool_to_int_with_if)] #[allow(clippy::bool_to_int_with_if)]
fn sample_flags_from_buffer( fn sample_flags_from_buffer(stream: &super::FragmentHeaderStream, buffer: &gst::BufferRef) -> u32 {
timing_info: &super::FragmentTimingInfo, if stream.delta_frames.intra_only() {
buffer: &gst::BufferRef,
) -> u32 {
if timing_info.delta_frames.intra_only() {
(0b00u32 << (16 + 10)) | // leading: unknown (0b00u32 << (16 + 10)) | // leading: unknown
(0b10u32 << (16 + 8)) | // depends: no (0b10u32 << (16 + 8)) | // depends: no
(0b10u32 << (16 + 6)) | // depended: no (0b10u32 << (16 + 6)) | // depended: no
@ -1743,7 +1756,7 @@ const SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT: u32 = 0x8_00;
fn analyze_buffers( fn analyze_buffers(
cfg: &super::FragmentHeaderConfiguration, cfg: &super::FragmentHeaderConfiguration,
idx: usize, idx: usize,
timing_info: &super::FragmentTimingInfo, stream: &super::FragmentHeaderStream,
timescale: u32, timescale: u32,
) -> Result< ) -> Result<
( (
@ -1802,7 +1815,7 @@ fn analyze_buffers(
tr_flags |= SAMPLE_DURATION_PRESENT; tr_flags |= SAMPLE_DURATION_PRESENT;
} }
let f = sample_flags_from_buffer(timing_info, buffer); let f = sample_flags_from_buffer(stream, buffer);
if first_buffer_flags.is_none() { if first_buffer_flags.is_none() {
first_buffer_flags = Some(f); first_buffer_flags = Some(f);
} else { } else {
@ -1818,7 +1831,7 @@ fn analyze_buffers(
} }
if let Some(composition_time_offset) = *composition_time_offset { if let Some(composition_time_offset) = *composition_time_offset {
assert!(timing_info.delta_frames.requires_dts()); assert!(stream.delta_frames.requires_dts());
if composition_time_offset != 0 { if composition_time_offset != 0 {
tr_flags |= SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT; tr_flags |= SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT;
} }
@ -1869,10 +1882,9 @@ fn write_traf(
cfg: &super::FragmentHeaderConfiguration, cfg: &super::FragmentHeaderConfiguration,
data_offset_offsets: &mut Vec<usize>, data_offset_offsets: &mut Vec<usize>,
idx: usize, idx: usize,
caps: &gst::CapsRef, stream: &super::FragmentHeaderStream,
timing_info: &super::FragmentTimingInfo,
) -> Result<(), Error> { ) -> Result<(), Error> {
let timescale = caps_to_timescale(caps); let timescale = fragment_header_stream_to_timescale(stream);
// Analyze all buffers to know what values can be put into the tfhd for all samples and what // Analyze all buffers to know what values can be put into the tfhd for all samples and what
// has to be stored for every single sample // has to be stored for every single sample
@ -1883,7 +1895,7 @@ fn write_traf(
default_duration, default_duration,
default_flags, default_flags,
negative_composition_time_offsets, negative_composition_time_offsets,
) = analyze_buffers(cfg, idx, timing_info, timescale)?; ) = analyze_buffers(cfg, idx, stream, timescale)?;
assert!((tf_flags & DEFAULT_SAMPLE_SIZE_PRESENT == 0) ^ default_size.is_some()); assert!((tf_flags & DEFAULT_SAMPLE_SIZE_PRESENT == 0) ^ default_size.is_some());
assert!((tf_flags & DEFAULT_SAMPLE_DURATION_PRESENT == 0) ^ default_duration.is_some()); assert!((tf_flags & DEFAULT_SAMPLE_DURATION_PRESENT == 0) ^ default_duration.is_some());
@ -1893,7 +1905,7 @@ fn write_traf(
write_tfhd(v, cfg, idx, default_size, default_duration, default_flags) write_tfhd(v, cfg, idx, default_size, default_duration, default_flags)
})?; })?;
write_full_box(v, b"tfdt", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { write_full_box(v, b"tfdt", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| {
write_tfdt(v, cfg, idx, timing_info, timescale) write_tfdt(v, cfg, idx, stream, timescale)
})?; })?;
let mut current_data_offset = 0; let mut current_data_offset = 0;
@ -1923,7 +1935,7 @@ fn write_traf(
current_data_offset, current_data_offset,
tr_flags, tr_flags,
timescale, timescale,
timing_info, stream,
run, run,
) )
}, },
@ -1973,11 +1985,12 @@ fn write_tfdt(
v: &mut Vec<u8>, v: &mut Vec<u8>,
_cfg: &super::FragmentHeaderConfiguration, _cfg: &super::FragmentHeaderConfiguration,
_idx: usize, _idx: usize,
timing_info: &super::FragmentTimingInfo, stream: &super::FragmentHeaderStream,
timescale: u32, timescale: u32,
) -> Result<(), Error> { ) -> Result<(), Error> {
let base_time = timing_info let base_time = stream
.start_time .start_time
.unwrap()
.mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds()) .mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("base time overflow")?; .context("base time overflow")?;
@ -1993,7 +2006,7 @@ fn write_trun(
current_data_offset: u32, current_data_offset: u32,
tr_flags: u32, tr_flags: u32,
timescale: u32, timescale: u32,
timing_info: &super::FragmentTimingInfo, stream: &super::FragmentHeaderStream,
buffers: &[Buffer], buffers: &[Buffer],
) -> Result<usize, Error> { ) -> Result<usize, Error> {
// Sample count // Sample count
@ -2004,7 +2017,7 @@ fn write_trun(
v.extend(current_data_offset.to_be_bytes()); v.extend(current_data_offset.to_be_bytes());
if (tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) != 0 { if (tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) != 0 {
v.extend(sample_flags_from_buffer(timing_info, &buffers[0].buffer).to_be_bytes()); v.extend(sample_flags_from_buffer(stream, &buffers[0].buffer).to_be_bytes());
} }
for Buffer { for Buffer {
@ -2036,7 +2049,7 @@ fn write_trun(
assert!((tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) == 0); assert!((tr_flags & FIRST_SAMPLE_FLAGS_PRESENT) == 0);
// Sample flags // Sample flags
v.extend(sample_flags_from_buffer(timing_info, buffer).to_be_bytes()); v.extend(sample_flags_from_buffer(stream, buffer).to_be_bytes());
} }
if (tr_flags & SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT) != 0 { if (tr_flags & SAMPLE_COMPOSITION_TIME_OFFSET_PRESENT) != 0 {

238
mux/fmp4/src/fmp4mux/imp.rs
View file

@ -76,6 +76,7 @@ struct Settings {
write_mehd: bool, write_mehd: bool,
interleave_bytes: Option<u64>, interleave_bytes: Option<u64>,
interleave_time: Option<gst::ClockTime>, interleave_time: Option<gst::ClockTime>,
movie_timescale: u32,
} }
impl Default for Settings { impl Default for Settings {
@ -87,6 +88,7 @@ impl Default for Settings {
write_mehd: DEFAULT_WRITE_MEHD, write_mehd: DEFAULT_WRITE_MEHD,
interleave_bytes: DEFAULT_INTERLEAVE_BYTES, interleave_bytes: DEFAULT_INTERLEAVE_BYTES,
interleave_time: DEFAULT_INTERLEAVE_TIME, interleave_time: DEFAULT_INTERLEAVE_TIME,
movie_timescale: 0,
} }
} }
} }
@ -122,7 +124,7 @@ struct Gop {
} }
struct Stream { struct Stream {
sinkpad: gst_base::AggregatorPad, sinkpad: super::FMP4MuxPad,
caps: gst::Caps, caps: gst::Caps,
delta_frames: DeltaFrames, delta_frames: DeltaFrames,
@ -617,11 +619,7 @@ impl FMP4Mux {
) -> Result< ) -> Result<
( (
// Drained streams // Drained streams
Vec<( Vec<(super::FragmentHeaderStream, VecDeque<Buffer>)>,
gst::Caps,
Option<super::FragmentTimingInfo>,
VecDeque<Buffer>,
)>,
// Minimum earliest PTS position of all streams // Minimum earliest PTS position of all streams
Option<gst::ClockTime>, Option<gst::ClockTime>,
// Minimum earliest PTS of all streams // Minimum earliest PTS of all streams
@ -658,6 +656,8 @@ impl FMP4Mux {
); );
for (idx, stream) in state.streams.iter_mut().enumerate() { for (idx, stream) in state.streams.iter_mut().enumerate() {
let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
assert!( assert!(
timeout timeout
|| at_eos || at_eos
@ -742,7 +742,16 @@ impl FMP4Mux {
"Draining no buffers", "Draining no buffers",
); );
drained_streams.push((stream.caps.clone(), None, VecDeque::new())); drained_streams.push((
super::FragmentHeaderStream {
caps: stream.caps.clone(),
start_time: None,
delta_frames: stream.delta_frames,
trak_timescale: stream_settings.trak_timescale,
},
VecDeque::new(),
));
continue; continue;
} }
@ -876,11 +885,12 @@ impl FMP4Mux {
} }
drained_streams.push(( drained_streams.push((
stream.caps.clone(), super::FragmentHeaderStream {
Some(super::FragmentTimingInfo { caps: stream.caps.clone(),
start_time, start_time: Some(start_time),
delta_frames: stream.delta_frames, delta_frames: stream.delta_frames,
}), trak_timescale: stream_settings.trak_timescale,
},
buffers, buffers,
)); ));
} }
@ -897,11 +907,7 @@ impl FMP4Mux {
fn preprocess_drained_streams_onvif( fn preprocess_drained_streams_onvif(
&self, &self,
state: &mut State, state: &mut State,
drained_streams: &mut [( drained_streams: &mut [(super::FragmentHeaderStream, VecDeque<Buffer>)],
gst::Caps,
Option<super::FragmentTimingInfo>,
VecDeque<Buffer>,
)],
) -> Result<Option<gst::ClockTime>, gst::FlowError> { ) -> Result<Option<gst::ClockTime>, gst::FlowError> {
let aggregator = self.obj(); let aggregator = self.obj();
if aggregator.class().as_ref().variant != super::Variant::ONVIF { if aggregator.class().as_ref().variant != super::Variant::ONVIF {
@ -925,7 +931,7 @@ impl FMP4Mux {
// If this is the first fragment then allow the first buffers to not have a reference // If this is the first fragment then allow the first buffers to not have a reference
// timestamp meta and backdate them // timestamp meta and backdate them
if state.stream_header.is_none() { if state.stream_header.is_none() {
for (idx, (_, _, drain_buffers)) in drained_streams.iter_mut().enumerate() { for (idx, (_, drain_buffers)) in drained_streams.iter_mut().enumerate() {
let (buffer_idx, utc_time, buffer) = let (buffer_idx, utc_time, buffer) =
match drain_buffers.iter().enumerate().find_map(|(idx, buffer)| { match drain_buffers.iter().enumerate().find_map(|(idx, buffer)| {
get_utc_time_from_buffer(&buffer.buffer) get_utc_time_from_buffer(&buffer.buffer)
@ -979,7 +985,7 @@ impl FMP4Mux {
if state.start_utc_time.is_none() { if state.start_utc_time.is_none() {
let mut start_utc_time = None; let mut start_utc_time = None;
for (idx, (_, _, drain_buffers)) in drained_streams.iter().enumerate() { for (idx, (_, drain_buffers)) in drained_streams.iter().enumerate() {
for buffer in drain_buffers { for buffer in drain_buffers {
let utc_time = match get_utc_time_from_buffer(&buffer.buffer) { let utc_time = match get_utc_time_from_buffer(&buffer.buffer) {
None => { None => {
@ -1010,7 +1016,7 @@ impl FMP4Mux {
// Update all buffer timestamps based on the UTC time and offset to the start UTC time // Update all buffer timestamps based on the UTC time and offset to the start UTC time
let start_utc_time = state.start_utc_time.unwrap(); let start_utc_time = state.start_utc_time.unwrap();
for (idx, (_, timing_info, drain_buffers)) in drained_streams.iter_mut().enumerate() { for (idx, (stream, drain_buffers)) in drained_streams.iter_mut().enumerate() {
let mut start_time = None; let mut start_time = None;
for buffer in drain_buffers.iter_mut() { for buffer in drain_buffers.iter_mut() {
@ -1128,9 +1134,9 @@ impl FMP4Mux {
if let Some(start_time) = start_time { if let Some(start_time) = start_time {
gst::debug!(CAT, obj: state.streams[idx].sinkpad, "Fragment starting at UTC time {}", start_time); gst::debug!(CAT, obj: state.streams[idx].sinkpad, "Fragment starting at UTC time {}", start_time);
timing_info.as_mut().unwrap().start_time = start_time; *stream.start_time.as_mut().unwrap() = start_time;
} else { } else {
assert!(timing_info.is_none()); assert!(stream.start_time.is_none());
} }
} }
@ -1141,35 +1147,28 @@ impl FMP4Mux {
fn interleave_buffers( fn interleave_buffers(
&self, &self,
settings: &Settings, settings: &Settings,
mut drained_streams: Vec<( mut drained_streams: Vec<(super::FragmentHeaderStream, VecDeque<Buffer>)>,
gst::Caps, ) -> Result<(Vec<Buffer>, Vec<super::FragmentHeaderStream>), gst::FlowError> {
Option<super::FragmentTimingInfo>,
VecDeque<Buffer>,
)>,
) -> Result<
(
Vec<Buffer>,
Vec<(gst::Caps, Option<super::FragmentTimingInfo>)>,
),
gst::FlowError,
> {
let mut interleaved_buffers = let mut interleaved_buffers =
Vec::with_capacity(drained_streams.iter().map(|(_, _, bufs)| bufs.len()).sum()); Vec::with_capacity(drained_streams.iter().map(|(_, bufs)| bufs.len()).sum());
while let Some((_idx, (_, _, bufs))) = drained_streams.iter_mut().enumerate().min_by( while let Some((_idx, (_, bufs))) =
|(a_idx, (_, _, a)), (b_idx, (_, _, b))| { drained_streams
let (a, b) = match (a.front(), b.front()) { .iter_mut()
(None, None) => return std::cmp::Ordering::Equal, .enumerate()
(None, _) => return std::cmp::Ordering::Greater, .min_by(|(a_idx, (_, a)), (b_idx, (_, b))| {
(_, None) => return std::cmp::Ordering::Less, let (a, b) = match (a.front(), b.front()) {
(Some(a), Some(b)) => (a, b), (None, None) => return std::cmp::Ordering::Equal,
}; (None, _) => return std::cmp::Ordering::Greater,
(_, None) => return std::cmp::Ordering::Less,
(Some(a), Some(b)) => (a, b),
};
match a.timestamp.cmp(&b.timestamp) { match a.timestamp.cmp(&b.timestamp) {
std::cmp::Ordering::Equal => a_idx.cmp(b_idx), std::cmp::Ordering::Equal => a_idx.cmp(b_idx),
cmp => cmp, cmp => cmp,
} }
}, })
) { {
let start_time = match bufs.front() { let start_time = match bufs.front() {
None => { None => {
// No more buffers now // No more buffers now
@ -1201,11 +1200,11 @@ impl FMP4Mux {
} }
// All buffers should be consumed now // All buffers should be consumed now
assert!(drained_streams.iter().all(|(_, _, bufs)| bufs.is_empty())); assert!(drained_streams.iter().all(|(_, bufs)| bufs.is_empty()));
let streams = drained_streams let streams = drained_streams
.into_iter() .into_iter()
.map(|(caps, timing_info, _)| (caps, timing_info)) .map(|(stream, _)| stream)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
Ok((interleaved_buffers, streams)) Ok((interleaved_buffers, streams))
@ -1217,7 +1216,7 @@ impl FMP4Mux {
settings: &Settings, settings: &Settings,
timeout: bool, timeout: bool,
at_eos: bool, at_eos: bool,
upstream_events: &mut Vec<(gst_base::AggregatorPad, gst::Event)>, upstream_events: &mut Vec<(super::FMP4MuxPad, gst::Event)>,
) -> Result<(Option<gst::Caps>, Option<gst::BufferList>), gst::FlowError> { ) -> Result<(Option<gst::Caps>, Option<gst::BufferList>), gst::FlowError> {
if at_eos { if at_eos {
gst::info!(CAT, imp: self, "Draining at EOS"); gst::info!(CAT, imp: self, "Draining at EOS");
@ -1241,7 +1240,7 @@ impl FMP4Mux {
) = self.drain_buffers(state, settings, timeout, at_eos)?; ) = self.drain_buffers(state, settings, timeout, at_eos)?;
// Remove all GAP buffers before processing them further // Remove all GAP buffers before processing them further
for (_, timing_info, buffers) in &mut drained_streams { for (stream, buffers) in &mut drained_streams {
buffers.retain(|buf| { buffers.retain(|buf| {
!buf.buffer.flags().contains(gst::BufferFlags::GAP) !buf.buffer.flags().contains(gst::BufferFlags::GAP)
|| !buf.buffer.flags().contains(gst::BufferFlags::DROPPABLE) || !buf.buffer.flags().contains(gst::BufferFlags::DROPPABLE)
@ -1249,7 +1248,7 @@ impl FMP4Mux {
}); });
if buffers.is_empty() { if buffers.is_empty() {
*timing_info = None; stream.start_time = None;
} }
} }
@ -1371,9 +1370,13 @@ impl FMP4Mux {
// Write mfra only for the main stream, and if there are no buffers for the main stream // Write mfra only for the main stream, and if there are no buffers for the main stream
// in this segment then don't write anything. // in this segment then don't write anything.
if let Some((_caps, Some(ref timing_info))) = streams.get(0) { if let Some(super::FragmentHeaderStream {
start_time: Some(start_time),
..
}) = streams.get(0)
{
state.fragment_offsets.push(super::FragmentOffset { state.fragment_offsets.push(super::FragmentOffset {
time: timing_info.start_time, time: *start_time,
offset: moof_offset, offset: moof_offset,
}); });
} }
@ -1432,7 +1435,7 @@ impl FMP4Mux {
if settings.write_mfra && at_eos { if settings.write_mfra && at_eos {
gst::debug!(CAT, imp: self, "Writing mfra box"); gst::debug!(CAT, imp: self, "Writing mfra box");
match boxes::create_mfra(&streams[0].0, &state.fragment_offsets) { match boxes::create_mfra(&streams[0].caps, &state.fragment_offsets) {
Ok(mut mfra) => { Ok(mut mfra) => {
{ {
let mfra = mfra.get_mut().unwrap(); let mfra = mfra.get_mut().unwrap();
@ -1462,7 +1465,7 @@ impl FMP4Mux {
.obj() .obj()
.sink_pads() .sink_pads()
.into_iter() .into_iter()
.map(|pad| pad.downcast::<gst_base::AggregatorPad>().unwrap()) .map(|pad| pad.downcast::<super::FMP4MuxPad>().unwrap())
{ {
let caps = match pad.current_caps() { let caps = match pad.current_caps() {
Some(caps) => caps, Some(caps) => caps,
@ -1599,13 +1602,18 @@ impl FMP4Mux {
let streams = state let streams = state
.streams .streams
.iter() .iter()
.map(|s| s.caps.clone()) .map(|s| super::HeaderStream {
trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale,
delta_frames: s.delta_frames,
caps: s.caps.clone(),
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut buffer = boxes::create_fmp4_header(super::HeaderConfiguration { let mut buffer = boxes::create_fmp4_header(super::HeaderConfiguration {
variant, variant,
update: at_eos, update: at_eos,
streams: streams.as_slice(), movie_timescale: settings.movie_timescale,
streams,
write_mehd: settings.write_mehd, write_mehd: settings.write_mehd,
duration: if at_eos { duration } else { None }, duration: if at_eos { duration } else { None },
start_utc_time: state start_utc_time: state
@ -1696,6 +1704,11 @@ impl ObjectImpl for FMP4Mux {
.default_value(DEFAULT_INTERLEAVE_TIME.map(gst::ClockTime::nseconds).unwrap_or(u64::MAX)) .default_value(DEFAULT_INTERLEAVE_TIME.map(gst::ClockTime::nseconds).unwrap_or(u64::MAX))
.mutable_ready() .mutable_ready()
.build(), .build(),
glib::ParamSpecUInt::builder("movie-timescale")
.nick("Movie Timescale")
.blurb("Timescale to use for the movie (units per second, 0 is automatic)")
.mutable_ready()
.build(),
] ]
}); });
@ -1745,6 +1758,11 @@ impl ObjectImpl for FMP4Mux {
}; };
} }
"movie-timescale" => {
let mut settings = self.settings.lock().unwrap();
settings.movie_timescale = value.get().expect("type checked upstream");
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -1781,6 +1799,11 @@ impl ObjectImpl for FMP4Mux {
settings.interleave_time.to_value() settings.interleave_time.to_value()
} }
"movie-timescale" => {
let settings = self.settings.lock().unwrap();
settings.movie_timescale.to_value()
}
_ => unimplemented!(), _ => unimplemented!(),
} }
} }
@ -1954,8 +1977,6 @@ impl AggregatorImpl for FMP4Mux {
} }
fn flush(&self) -> Result<gst::FlowSuccess, gst::FlowError> { fn flush(&self) -> Result<gst::FlowSuccess, gst::FlowError> {
self.parent_flush()?;
let mut state = self.state.lock().unwrap(); let mut state = self.state.lock().unwrap();
for stream in &mut state.streams { for stream in &mut state.streams {
@ -1969,7 +1990,9 @@ impl AggregatorImpl for FMP4Mux {
state.current_offset = 0; state.current_offset = 0;
state.fragment_offsets.clear(); state.fragment_offsets.clear();
Ok(gst::FlowSuccess::Ok) drop(state);
self.parent_flush()
} }
fn stop(&self) -> Result<(), gst::ErrorMessage> { fn stop(&self) -> Result<(), gst::ErrorMessage> {
@ -2346,7 +2369,7 @@ impl ElementImpl for ISOFMP4Mux {
) )
.unwrap(); .unwrap();
let sink_pad_template = gst::PadTemplate::new( let sink_pad_template = gst::PadTemplate::with_gtype(
"sink_%u", "sink_%u",
gst::PadDirection::Sink, gst::PadDirection::Sink,
gst::PadPresence::Request, gst::PadPresence::Request,
@ -2385,6 +2408,7 @@ impl ElementImpl for ISOFMP4Mux {
] ]
.into_iter() .into_iter()
.collect::<gst::Caps>(), .collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
) )
.unwrap(); .unwrap();
@ -2441,7 +2465,7 @@ impl ElementImpl for CMAFMux {
) )
.unwrap(); .unwrap();
let sink_pad_template = gst::PadTemplate::new( let sink_pad_template = gst::PadTemplate::with_gtype(
"sink", "sink",
gst::PadDirection::Sink, gst::PadDirection::Sink,
gst::PadPresence::Always, gst::PadPresence::Always,
@ -2467,6 +2491,7 @@ impl ElementImpl for CMAFMux {
] ]
.into_iter() .into_iter()
.collect::<gst::Caps>(), .collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
) )
.unwrap(); .unwrap();
@ -2523,7 +2548,7 @@ impl ElementImpl for DASHMP4Mux {
) )
.unwrap(); .unwrap();
let sink_pad_template = gst::PadTemplate::new( let sink_pad_template = gst::PadTemplate::with_gtype(
"sink", "sink",
gst::PadDirection::Sink, gst::PadDirection::Sink,
gst::PadPresence::Always, gst::PadPresence::Always,
@ -2562,6 +2587,7 @@ impl ElementImpl for DASHMP4Mux {
] ]
.into_iter() .into_iter()
.collect::<gst::Caps>(), .collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
) )
.unwrap(); .unwrap();
@ -2618,7 +2644,7 @@ impl ElementImpl for ONVIFFMP4Mux {
) )
.unwrap(); .unwrap();
let sink_pad_template = gst::PadTemplate::new( let sink_pad_template = gst::PadTemplate::with_gtype(
"sink_%u", "sink_%u",
gst::PadDirection::Sink, gst::PadDirection::Sink,
gst::PadPresence::Request, gst::PadPresence::Request,
@ -2665,6 +2691,7 @@ impl ElementImpl for ONVIFFMP4Mux {
] ]
.into_iter() .into_iter()
.collect::<gst::Caps>(), .collect::<gst::Caps>(),
super::FMP4MuxPad::static_type(),
) )
.unwrap(); .unwrap();
@ -2680,3 +2707,82 @@ impl AggregatorImpl for ONVIFFMP4Mux {}
impl FMP4MuxImpl for ONVIFFMP4Mux { impl FMP4MuxImpl for ONVIFFMP4Mux {
const VARIANT: super::Variant = super::Variant::ONVIF; const VARIANT: super::Variant = super::Variant::ONVIF;
} }
#[derive(Default, Clone)]
struct PadSettings {
trak_timescale: u32,
}
#[derive(Default)]
pub(crate) struct FMP4MuxPad {
settings: Mutex<PadSettings>,
}
#[glib::object_subclass]
impl ObjectSubclass for FMP4MuxPad {
const NAME: &'static str = "GstFMP4MuxPad";
type Type = super::FMP4MuxPad;
type ParentType = gst_base::AggregatorPad;
}
impl ObjectImpl for FMP4MuxPad {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![glib::ParamSpecUInt::builder("trak-timescale")
.nick("Track Timescale")
.blurb("Timescale to use for the track (units per second, 0 is automatic)")
.mutable_ready()
.build()]
});
&PROPERTIES
}
fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) {
match pspec.name() {
"trak-timescale" => {
let mut settings = self.settings.lock().unwrap();
settings.trak_timescale = value.get().expect("type checked upstream");
}
_ => unimplemented!(),
}
}
fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
match pspec.name() {
"trak-timescale" => {
let settings = self.settings.lock().unwrap();
settings.trak_timescale.to_value()
}
_ => unimplemented!(),
}
}
}
impl GstObjectImpl for FMP4MuxPad {}
impl PadImpl for FMP4MuxPad {}
impl AggregatorPadImpl for FMP4MuxPad {
fn flush(&self, aggregator: &gst_base::Aggregator) -> Result<gst::FlowSuccess, gst::FlowError> {
let mux = aggregator.downcast_ref::<super::FMP4Mux>().unwrap();
let mut mux_state = mux.imp().state.lock().unwrap();
for stream in &mut mux_state.streams {
if stream.sinkpad == *self.obj() {
stream.queued_gops.clear();
stream.dts_offset = None;
stream.current_position = gst::ClockTime::ZERO;
stream.current_utc_time = gst::ClockTime::ZERO;
stream.fragment_filled = false;
break;
}
}
drop(mux_state);
self.parent_flush(aggregator)
}
}

54
mux/fmp4/src/fmp4mux/mod.rs
View file

@ -12,6 +12,10 @@ use gst::prelude::*;
mod boxes; mod boxes;
mod imp; mod imp;
glib::wrapper! {
pub(crate) struct FMP4MuxPad(ObjectSubclass<imp::FMP4MuxPad>) @extends gst_base::AggregatorPad, gst::Pad, gst::Object;
}
glib::wrapper! { glib::wrapper! {
pub(crate) struct FMP4Mux(ObjectSubclass<imp::FMP4Mux>) @extends gst_base::Aggregator, gst::Element, gst::Object; pub(crate) struct FMP4Mux(ObjectSubclass<imp::FMP4Mux>) @extends gst_base::Aggregator, gst::Element, gst::Object;
} }
@ -33,8 +37,12 @@ glib::wrapper! {
} }
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
FMP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); #[cfg(feature = "doc")]
HeaderUpdateMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); {
FMP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
FMP4MuxPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
HeaderUpdateMode::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
}
gst::Element::register( gst::Element::register(
Some(plugin), Some(plugin),
"isofmp4mux", "isofmp4mux",
@ -64,33 +72,63 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
} }
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct HeaderConfiguration<'a> { pub(crate) struct HeaderConfiguration {
variant: Variant, variant: Variant,
update: bool, update: bool,
/// Pre-defined movie timescale if not 0.
movie_timescale: u32,
/// First caps must be the video/reference stream. Must be in the order the tracks are going to /// First caps must be the video/reference stream. Must be in the order the tracks are going to
/// be used later for the fragments too. /// be used later for the fragments too.
streams: &'a [gst::Caps], streams: Vec<HeaderStream>,
write_mehd: bool, write_mehd: bool,
duration: Option<gst::ClockTime>, duration: Option<gst::ClockTime>,
/// Start UTC time in ONVIF mode. /// Start UTC time in ONVIF mode.
/// Since Jan 1 1601 in 100ns units. /// Since Jan 1 1601 in 100ns units.
start_utc_time: Option<u64>, start_utc_time: Option<u64>,
} }
#[derive(Debug)]
pub(crate) struct HeaderStream {
/// Caps of this stream
caps: gst::Caps,
/// Set if this is an intra-only stream
delta_frames: DeltaFrames,
/// Pre-defined trak timescale if not 0.
trak_timescale: u32,
}
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct FragmentHeaderConfiguration<'a> { pub(crate) struct FragmentHeaderConfiguration<'a> {
variant: Variant, variant: Variant,
/// Sequence number for this fragment.
sequence_number: u32, sequence_number: u32,
streams: &'a [(gst::Caps, Option<FragmentTimingInfo>)],
streams: &'a [FragmentHeaderStream],
buffers: &'a [Buffer], buffers: &'a [Buffer],
} }
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct FragmentTimingInfo { pub(crate) struct FragmentHeaderStream {
/// Start time of this fragment /// Caps of this stream
start_time: gst::ClockTime, caps: gst::Caps,
/// Set if this is an intra-only stream /// Set if this is an intra-only stream
delta_frames: DeltaFrames, delta_frames: DeltaFrames,
/// Pre-defined trak timescale if not 0.
trak_timescale: u32,
/// Start time of this fragment
///
/// `None` if this stream has no buffers in this fragment.
start_time: Option<gst::ClockTime>,
} }
#[derive(Debug, Copy, Clone)] #[derive(Debug, Copy, Clone)]