mirror of
https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git
synced 2024-05-12 21:32:39 +00:00
Merge branch 'webrtc-raw-audio' into 'main'
webrtc: add raw payload support See merge request gstreamer/gst-plugins-rs!1501
This commit is contained in:
commit
f480bc4cd0
|
@ -8448,7 +8448,7 @@
|
|||
"construct": false,
|
||||
"construct-only": false,
|
||||
"controllable": false,
|
||||
"default": "audio/x-opus",
|
||||
"default": "audio/x-opus; audio/x-raw, format=(string)S24BE, layout=(GstAudioLayout)interleaved; audio/x-raw, format=(string)S16BE, layout=(GstAudioLayout)interleaved; audio/x-raw, format=(string)U8, layout=(GstAudioLayout)interleaved",
|
||||
"mutable": "ready",
|
||||
"readable": true,
|
||||
"type": "GstCaps",
|
||||
|
@ -8631,7 +8631,7 @@
|
|||
"construct": false,
|
||||
"construct-only": false,
|
||||
"controllable": false,
|
||||
"default": "video/x-vp8; video/x-h264; video/x-vp9; video/x-h265",
|
||||
"default": "video/x-vp8; video/x-h264; video/x-vp9; video/x-h265; video/x-raw, format=(string){ RGB, RGBA, BGR, BGRA, AYUV, UYVY, I420, Y41B, UYVP }",
|
||||
"mutable": "ready",
|
||||
"readable": true,
|
||||
"type": "GstCaps",
|
||||
|
@ -8758,7 +8758,7 @@
|
|||
"kind": "object",
|
||||
"properties": {
|
||||
"audio-codecs": {
|
||||
"blurb": "Names of audio codecs to be be used during the SDP negotiation. Valid values: [OPUS]",
|
||||
"blurb": "Names of audio codecs to be be used during the SDP negotiation. Valid values: [OPUS, L24, L16, L8]",
|
||||
"conditionally-available": false,
|
||||
"construct": false,
|
||||
"construct-only": false,
|
||||
|
@ -8838,7 +8838,7 @@
|
|||
"writable": true
|
||||
},
|
||||
"video-codecs": {
|
||||
"blurb": "Names of video codecs to be be used during the SDP negotiation. Valid values: [VP8, H264, VP9, H265]",
|
||||
"blurb": "Names of video codecs to be be used during the SDP negotiation. Valid values: [VP8, H264, VP9, H265, RAW]",
|
||||
"conditionally-available": false,
|
||||
"construct": false,
|
||||
"construct-only": false,
|
||||
|
|
|
@ -34,6 +34,8 @@ mode and an example based on RTSP instead of WebRTC.
|
|||
The examples can also be used for [RFC 7273] NTP or PTP clock signalling and
|
||||
synchronization.
|
||||
|
||||
Finally, raw payloads (e.g. L24 audio) can be negotiated.
|
||||
|
||||
[RFC 6051]: https://datatracker.ietf.org/doc/html/rfc6051
|
||||
[RFC 7273]: https://datatracker.ietf.org/doc/html/rfc7273
|
||||
[Instantaneous RTP synchronization...]: https://coaxion.net/blog/2022/05/instantaneous-rtp-synchronization-retrieval-of-absolute-sender-clock-times-with-gstreamer/
|
||||
|
@ -129,3 +131,31 @@ cargo r --example webrtc-precise-sync-recv -- --expect-clock-signalling
|
|||
cargo r --example webrtc-precise-sync-send -- --clock ntp --do-clock-signalling \
|
||||
--video-streams 0 --audio-streams 2
|
||||
```
|
||||
|
||||
### Raw payload
|
||||
|
||||
The sender can be instructed to send raw payloads.
|
||||
|
||||
This command will stream two stereo L24 streams:
|
||||
|
||||
```shell
|
||||
cargo r --example webrtc-precise-sync-send -- \
|
||||
--video-streams 0 --audio-streams 2 \
|
||||
--audio-caps 'audio/x-raw,format=S24BE,rate=48000,channels=2'
|
||||
```
|
||||
|
||||
Launch the receiver with:
|
||||
|
||||
```shell
|
||||
cargo r --example webrtc-precise-sync-recv
|
||||
```
|
||||
|
||||
This can be used to stream multiple RAW video streams forcing width and
|
||||
allowing fallback to VP8 & OPUS if remote doesn't support raw payloads:
|
||||
|
||||
```shell
|
||||
cargo r --example webrtc-precise-sync-send -- \
|
||||
--video-streams 2 --audio-streams 1 \
|
||||
--video-caps 'video/x-raw,format=I420,width=400;video/x-vp8' \
|
||||
--audio-caps 'audio/x-raw,format=S24BE,rate=48000,channels=2;video/x-opus'
|
||||
```
|
||||
|
|
|
@ -43,6 +43,18 @@ struct Args {
|
|||
#[clap(long, help = "RTP jitterbuffer latency (ms)", default_value = "40")]
|
||||
pub rtp_latency: u32,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
help = "Force accepted audio codecs. See 'webrtcsrc' 'audio-codecs' property (ex. 'OPUS'). Accepts several occurrences."
|
||||
)]
|
||||
pub audio_codecs: Vec<String>,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
help = "Force accepted video codecs. See 'webrtcsrc' 'video-codecs' property (ex. 'VP8'). Accepts several occurrences."
|
||||
)]
|
||||
pub video_codecs: Vec<String>,
|
||||
|
||||
#[clap(long, help = "Signalling server host", default_value = "localhost")]
|
||||
pub server: String,
|
||||
|
||||
|
@ -123,6 +135,14 @@ fn spawn_consumer(
|
|||
webrtcsrc.set_property("do-retransmission", false);
|
||||
}
|
||||
|
||||
if !args.audio_codecs.is_empty() {
|
||||
webrtcsrc.set_property("audio-codecs", gst::Array::new(&args.audio_codecs));
|
||||
}
|
||||
|
||||
if !args.video_codecs.is_empty() {
|
||||
webrtcsrc.set_property("video-codecs", gst::Array::new(&args.video_codecs));
|
||||
}
|
||||
|
||||
bin.add(&webrtcsrc).context("Adding webrtcsrc")?;
|
||||
|
||||
let signaller = webrtcsrc.property::<gst::glib::Object>("signaller");
|
||||
|
|
|
@ -45,6 +45,12 @@ struct Args {
|
|||
)]
|
||||
pub video_streams: usize,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
help = "Force audio caps (ex. for L24 'audio/x-raw,format=S24BE,rate=48000,channels=2')"
|
||||
)]
|
||||
pub audio_caps: Option<String>,
|
||||
|
||||
#[clap(long, help = "Force video caps (ex. 'video/x-h264')")]
|
||||
pub video_caps: Option<String>,
|
||||
|
||||
|
@ -135,6 +141,8 @@ impl App {
|
|||
}
|
||||
|
||||
async fn prepare(&mut self) -> anyhow::Result<()> {
|
||||
use std::str::FromStr;
|
||||
|
||||
debug!("Preparing");
|
||||
|
||||
self.pipeline = Some(gst::Pipeline::new());
|
||||
|
@ -217,6 +225,19 @@ impl App {
|
|||
});
|
||||
}
|
||||
|
||||
let raw_audio_caps = if let Some(ref audio_caps) = self.args.audio_caps {
|
||||
let caps = gst::Caps::from_str(audio_caps).context("Parsing audio caps")?;
|
||||
webrtcsink.set_property("audio-caps", &caps);
|
||||
|
||||
// Reuse the first user defined caps for the raw caps
|
||||
let mut s = caps.structure(0).expect("parsed above").to_owned();
|
||||
s.set_name("audio/x-raw");
|
||||
|
||||
Some(gst::Caps::from(s))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for idx in 0..self.args.audio_streams {
|
||||
let audiosrc = gst::ElementFactory::make("audiotestsrc")
|
||||
.property("is-live", true)
|
||||
|
@ -226,11 +247,47 @@ impl App {
|
|||
.context("Creating audiotestsrc")?;
|
||||
self.pipeline().add(&audiosrc).context("Adding audiosrc")?;
|
||||
|
||||
audiosrc
|
||||
.link_pads(None, &webrtcsink, Some("audio_%u"))
|
||||
.context("Linking audiosrc")?;
|
||||
if let Some(ref raw_caps) = raw_audio_caps {
|
||||
audiosrc
|
||||
.link_pads_filtered(None, &webrtcsink, Some("audio_%u"), raw_caps)
|
||||
.context("Linking audiosrc")?;
|
||||
} else {
|
||||
audiosrc
|
||||
.link_pads(None, &webrtcsink, Some("audio_%u"))
|
||||
.context("Linking audiosrc")?;
|
||||
}
|
||||
}
|
||||
|
||||
let raw_video_caps = {
|
||||
let mut raw_video_caps = if let Some(ref video_caps) = self.args.video_caps {
|
||||
let caps = gst::Caps::from_str(video_caps).context("Parsing video caps")?;
|
||||
webrtcsink.set_property("video-caps", &caps);
|
||||
|
||||
// Reuse the first user defined caps for the raw caps
|
||||
let mut s = caps.structure(0).expect("parsed above").to_owned();
|
||||
s.set_name("video/x-raw");
|
||||
|
||||
gst::Caps::from(s)
|
||||
} else {
|
||||
gst::Caps::new_empty_simple("video/x-raw")
|
||||
};
|
||||
|
||||
// If no width / height are specified, set something big enough
|
||||
let caps_mut = raw_video_caps.make_mut();
|
||||
let s = caps_mut.structure_mut(0).expect("created above");
|
||||
match (s.get::<i32>("width").ok(), s.get::<i32>("height").ok()) {
|
||||
(None, None) => {
|
||||
s.set("width", 800i32);
|
||||
s.set("height", 600i32);
|
||||
}
|
||||
(Some(width), None) => s.set("height", 3 * width / 4),
|
||||
(None, Some(height)) => s.set("width", 4 * height / 3),
|
||||
_ => (),
|
||||
}
|
||||
|
||||
raw_video_caps
|
||||
};
|
||||
|
||||
for idx in 0..self.args.video_streams {
|
||||
let videosrc = gst::ElementFactory::make("videotestsrc")
|
||||
.property("is-live", true)
|
||||
|
@ -247,13 +304,7 @@ impl App {
|
|||
.expect("adding video elements");
|
||||
|
||||
videosrc
|
||||
.link_filtered(
|
||||
&video_overlay,
|
||||
&gst::Caps::builder("video/x-raw")
|
||||
.field("width", 800i32)
|
||||
.field("height", 600i32)
|
||||
.build(),
|
||||
)
|
||||
.link_filtered(&video_overlay, &raw_video_caps)
|
||||
.context("Linking videosrc to timeoverlay")?;
|
||||
|
||||
video_overlay
|
||||
|
@ -261,10 +312,6 @@ impl App {
|
|||
.context("Linking video overlay")?;
|
||||
}
|
||||
|
||||
if let Some(ref video_caps) = self.args.video_caps {
|
||||
webrtcsink.set_property("video-caps", &gst::Caps::builder(video_caps).build());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
@ -427,7 +427,7 @@ impl Clone for DecodingInfo {
|
|||
|
||||
#[derive(Clone, Debug)]
|
||||
struct EncodingInfo {
|
||||
encoder: gst::ElementFactory,
|
||||
encoder: Option<gst::ElementFactory>,
|
||||
payloader: gst::ElementFactory,
|
||||
output_filter: Option<gst::Caps>,
|
||||
}
|
||||
|
@ -449,16 +449,27 @@ impl Codec {
|
|||
stream_type: gst::StreamType,
|
||||
caps: &gst::Caps,
|
||||
decoders: &glib::List<gst::ElementFactory>,
|
||||
depayloaders: &glib::List<gst::ElementFactory>,
|
||||
encoders: &glib::List<gst::ElementFactory>,
|
||||
payloaders: &glib::List<gst::ElementFactory>,
|
||||
) -> Self {
|
||||
let has_decoder = Self::has_decoder_for_caps(caps, decoders);
|
||||
let has_depayloader = Self::has_depayloader_for_codec(name, depayloaders);
|
||||
|
||||
let decoding_info = if has_depayloader && has_decoder {
|
||||
Some(DecodingInfo {
|
||||
has_decoder: AtomicBool::new(has_decoder),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let encoder = Self::get_encoder_for_caps(caps, encoders);
|
||||
let payloader = Self::get_payloader_for_codec(name, payloaders);
|
||||
|
||||
let encoding_info = if let (Some(encoder), Some(payloader)) = (encoder, payloader) {
|
||||
Some(EncodingInfo {
|
||||
encoder,
|
||||
encoder: Some(encoder),
|
||||
payloader,
|
||||
output_filter: None,
|
||||
})
|
||||
|
@ -471,11 +482,39 @@ impl Codec {
|
|||
stream_type,
|
||||
name: name.into(),
|
||||
payload_type: None,
|
||||
decoding_info,
|
||||
encoding_info,
|
||||
}
|
||||
}
|
||||
|
||||
decoding_info: Some(DecodingInfo {
|
||||
has_decoder: AtomicBool::new(has_decoder),
|
||||
}),
|
||||
pub fn new_raw(
|
||||
name: &str,
|
||||
stream_type: gst::StreamType,
|
||||
caps: &gst::Caps,
|
||||
depayloaders: &glib::List<gst::ElementFactory>,
|
||||
payloaders: &glib::List<gst::ElementFactory>,
|
||||
) -> Self {
|
||||
let decoding_info = if Self::has_depayloader_for_codec(name, depayloaders) {
|
||||
Some(DecodingInfo {
|
||||
has_decoder: AtomicBool::new(false),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let payloader = Self::get_payloader_for_codec(name, payloaders);
|
||||
let encoding_info = payloader.map(|payloader| EncodingInfo {
|
||||
encoder: None,
|
||||
payloader,
|
||||
output_filter: None,
|
||||
});
|
||||
|
||||
Self {
|
||||
caps: caps.clone(),
|
||||
stream_type,
|
||||
name: name.into(),
|
||||
payload_type: None,
|
||||
decoding_info,
|
||||
encoding_info,
|
||||
}
|
||||
}
|
||||
|
@ -488,34 +527,16 @@ impl Codec {
|
|||
self.payload_type = Some(pt);
|
||||
}
|
||||
|
||||
pub fn new_decoding(
|
||||
name: &str,
|
||||
stream_type: gst::StreamType,
|
||||
caps: &gst::Caps,
|
||||
decoders: &glib::List<gst::ElementFactory>,
|
||||
) -> Self {
|
||||
let has_decoder = Self::has_decoder_for_caps(caps, decoders);
|
||||
|
||||
Self {
|
||||
caps: caps.clone(),
|
||||
stream_type,
|
||||
name: name.into(),
|
||||
payload_type: None,
|
||||
|
||||
decoding_info: Some(DecodingInfo {
|
||||
has_decoder: AtomicBool::new(has_decoder),
|
||||
}),
|
||||
|
||||
encoding_info: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_decoder(&self) -> bool {
|
||||
pub fn can_be_received(&self) -> bool {
|
||||
if self.decoding_info.is_none() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let decoder_info = self.decoding_info.as_ref().unwrap();
|
||||
if is_raw_caps(&self.caps) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if decoder_info.has_decoder.load(Ordering::SeqCst) {
|
||||
true
|
||||
} else if Self::has_decoder_for_caps(
|
||||
|
@ -599,6 +620,40 @@ impl Codec {
|
|||
})
|
||||
}
|
||||
|
||||
fn has_depayloader_for_codec(
|
||||
codec: &str,
|
||||
depayloaders: &glib::List<gst::ElementFactory>,
|
||||
) -> bool {
|
||||
depayloaders.iter().any(|factory| {
|
||||
factory.static_pad_templates().iter().any(|template| {
|
||||
let template_caps = template.caps();
|
||||
|
||||
if template.direction() != gst::PadDirection::Sink || template_caps.is_any() {
|
||||
return false;
|
||||
}
|
||||
|
||||
template_caps.iter().any(|s| {
|
||||
s.has_field("encoding-name")
|
||||
&& s.get::<gst::List>("encoding-name").map_or_else(
|
||||
|_| {
|
||||
if let Ok(encoding_name) = s.get::<&str>("encoding-name") {
|
||||
encoding_name == codec
|
||||
} else {
|
||||
false
|
||||
}
|
||||
},
|
||||
|encoding_names| {
|
||||
encoding_names.iter().any(|v| {
|
||||
v.get::<&str>()
|
||||
.map_or(false, |encoding_name| encoding_name == codec)
|
||||
})
|
||||
},
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_video(&self) -> bool {
|
||||
matches!(self.stream_type, gst::StreamType::VIDEO)
|
||||
}
|
||||
|
@ -608,11 +663,13 @@ impl Codec {
|
|||
}
|
||||
|
||||
pub fn build_encoder(&self) -> Option<Result<gst::Element, Error>> {
|
||||
self.encoding_info.as_ref().map(|info| {
|
||||
info.encoder
|
||||
.create()
|
||||
.build()
|
||||
.with_context(|| format!("Creating encoder {}", info.encoder.name()))
|
||||
self.encoding_info.as_ref().and_then(|info| {
|
||||
info.encoder.as_ref().map(|encoder| {
|
||||
encoder
|
||||
.create()
|
||||
.build()
|
||||
.with_context(|| format!("Creating encoder {}", encoder.name()))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -655,13 +712,17 @@ impl Codec {
|
|||
}
|
||||
|
||||
pub fn encoder_factory(&self) -> Option<gst::ElementFactory> {
|
||||
self.encoding_info.as_ref().map(|info| info.encoder.clone())
|
||||
self.encoding_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.encoder.clone())
|
||||
}
|
||||
|
||||
pub fn encoder_name(&self) -> Option<String> {
|
||||
self.encoding_info
|
||||
.as_ref()
|
||||
.map(|info| info.encoder.name().to_string())
|
||||
self.encoding_info.as_ref().and_then(|info| {
|
||||
info.encoder
|
||||
.as_ref()
|
||||
.map(|encoder| encoder.name().to_string())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set_output_filter(&mut self, caps: gst::Caps) {
|
||||
|
@ -713,6 +774,36 @@ impl Codec {
|
|||
|
||||
pub static AUDIO_CAPS: Lazy<gst::Caps> = Lazy::new(|| gst::Caps::new_empty_simple("audio/x-raw"));
|
||||
pub static OPUS_CAPS: Lazy<gst::Caps> = Lazy::new(|| gst::Caps::new_empty_simple("audio/x-opus"));
|
||||
pub static L24_CAPS: Lazy<gst::Caps> = Lazy::new(|| {
|
||||
gst::Caps::builder_full()
|
||||
.structure(
|
||||
gst::Structure::builder("audio/x-raw")
|
||||
.field("format", gst_audio::AudioFormat::S24be.to_str())
|
||||
.field("layout", glib::gstr!("interleaved"))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
});
|
||||
pub static L16_CAPS: Lazy<gst::Caps> = Lazy::new(|| {
|
||||
gst::Caps::builder_full()
|
||||
.structure(
|
||||
gst::Structure::builder("audio/x-raw")
|
||||
.field("format", gst_audio::AudioFormat::S16be.to_str())
|
||||
.field("layout", glib::gstr!("interleaved"))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
});
|
||||
pub static L8_CAPS: Lazy<gst::Caps> = Lazy::new(|| {
|
||||
gst::Caps::builder_full()
|
||||
.structure(
|
||||
gst::Structure::builder("audio/x-raw")
|
||||
.field("format", gst_audio::AudioFormat::U8.to_str())
|
||||
.field("layout", glib::gstr!("interleaved"))
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
});
|
||||
|
||||
pub static VIDEO_CAPS: Lazy<gst::Caps> = Lazy::new(|| {
|
||||
gst::Caps::builder_full_with_any_features()
|
||||
|
@ -723,6 +814,32 @@ pub static VP8_CAPS: Lazy<gst::Caps> = Lazy::new(|| gst::Caps::new_empty_simple(
|
|||
pub static VP9_CAPS: Lazy<gst::Caps> = Lazy::new(|| gst::Caps::new_empty_simple("video/x-vp9"));
|
||||
pub static H264_CAPS: Lazy<gst::Caps> = Lazy::new(|| gst::Caps::new_empty_simple("video/x-h264"));
|
||||
pub static H265_CAPS: Lazy<gst::Caps> = Lazy::new(|| gst::Caps::new_empty_simple("video/x-h265"));
|
||||
pub static VRAW_CAPS: Lazy<gst::Caps> = Lazy::new(|| {
|
||||
gst::Caps::builder_full()
|
||||
.structure(
|
||||
gst::Structure::builder("video/x-raw")
|
||||
.field(
|
||||
"format",
|
||||
gst::List::new(
|
||||
[
|
||||
gst_video::VideoFormat::Rgb,
|
||||
gst_video::VideoFormat::Rgba,
|
||||
gst_video::VideoFormat::Bgr,
|
||||
gst_video::VideoFormat::Bgra,
|
||||
gst_video::VideoFormat::Ayuv,
|
||||
gst_video::VideoFormat::Uyvy,
|
||||
gst_video::VideoFormat::I420,
|
||||
gst_video::VideoFormat::Y41b,
|
||||
gst_video::VideoFormat::Uyvp,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|f| f.to_str()),
|
||||
),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
});
|
||||
|
||||
pub static RTP_CAPS: Lazy<gst::Caps> =
|
||||
Lazy::new(|| gst::Caps::new_empty_simple("application/x-rtp"));
|
||||
|
@ -750,7 +867,7 @@ impl Codecs {
|
|||
Self(codecs.values().cloned().collect())
|
||||
}
|
||||
|
||||
pub fn find_for_encoded_caps(&self, caps: &gst::Caps) -> Option<Codec> {
|
||||
pub fn find_for_payloadable_caps(&self, caps: &gst::Caps) -> Option<Codec> {
|
||||
self.iter()
|
||||
.find(|codec| codec.caps.can_intersect(caps) && codec.encoding_info.is_some())
|
||||
.cloned()
|
||||
|
@ -763,6 +880,11 @@ static CODECS: Lazy<Codecs> = Lazy::new(|| {
|
|||
gst::Rank::MARGINAL,
|
||||
);
|
||||
|
||||
let depayloaders = gst::ElementFactory::factories_with_type(
|
||||
gst::ElementFactoryType::DEPAYLOADER,
|
||||
gst::Rank::MARGINAL,
|
||||
);
|
||||
|
||||
let encoders = gst::ElementFactory::factories_with_type(
|
||||
gst::ElementFactoryType::ENCODER,
|
||||
gst::Rank::MARGINAL,
|
||||
|
@ -779,14 +901,37 @@ static CODECS: Lazy<Codecs> = Lazy::new(|| {
|
|||
gst::StreamType::AUDIO,
|
||||
&OPUS_CAPS,
|
||||
&decoders,
|
||||
&depayloaders,
|
||||
&encoders,
|
||||
&payloaders,
|
||||
),
|
||||
Codec::new_raw(
|
||||
"L24",
|
||||
gst::StreamType::AUDIO,
|
||||
&L24_CAPS,
|
||||
&depayloaders,
|
||||
&payloaders,
|
||||
),
|
||||
Codec::new_raw(
|
||||
"L16",
|
||||
gst::StreamType::AUDIO,
|
||||
&L16_CAPS,
|
||||
&depayloaders,
|
||||
&payloaders,
|
||||
),
|
||||
Codec::new_raw(
|
||||
"L8",
|
||||
gst::StreamType::AUDIO,
|
||||
&L8_CAPS,
|
||||
&depayloaders,
|
||||
&payloaders,
|
||||
),
|
||||
Codec::new(
|
||||
"VP8",
|
||||
gst::StreamType::VIDEO,
|
||||
&VP8_CAPS,
|
||||
&decoders,
|
||||
&depayloaders,
|
||||
&encoders,
|
||||
&payloaders,
|
||||
),
|
||||
|
@ -795,6 +940,7 @@ static CODECS: Lazy<Codecs> = Lazy::new(|| {
|
|||
gst::StreamType::VIDEO,
|
||||
&H264_CAPS,
|
||||
&decoders,
|
||||
&depayloaders,
|
||||
&encoders,
|
||||
&payloaders,
|
||||
),
|
||||
|
@ -803,6 +949,7 @@ static CODECS: Lazy<Codecs> = Lazy::new(|| {
|
|||
gst::StreamType::VIDEO,
|
||||
&VP9_CAPS,
|
||||
&decoders,
|
||||
&depayloaders,
|
||||
&encoders,
|
||||
&payloaders,
|
||||
),
|
||||
|
@ -811,9 +958,17 @@ static CODECS: Lazy<Codecs> = Lazy::new(|| {
|
|||
gst::StreamType::VIDEO,
|
||||
&H265_CAPS,
|
||||
&decoders,
|
||||
&depayloaders,
|
||||
&encoders,
|
||||
&payloaders,
|
||||
),
|
||||
Codec::new_raw(
|
||||
"RAW",
|
||||
gst::StreamType::VIDEO,
|
||||
&VRAW_CAPS,
|
||||
&depayloaders,
|
||||
&payloaders,
|
||||
),
|
||||
])
|
||||
});
|
||||
|
||||
|
@ -899,7 +1054,6 @@ impl Codecs {
|
|||
}
|
||||
|
||||
pub fn is_raw_caps(caps: &gst::Caps) -> bool {
|
||||
assert!(caps.is_fixed());
|
||||
["video/x-raw", "audio/x-raw"].contains(&caps.structure(0).unwrap().name().as_str())
|
||||
}
|
||||
|
||||
|
|
|
@ -845,7 +845,7 @@ impl PayloadChainBuilder {
|
|||
codec = self.codec,
|
||||
);
|
||||
|
||||
let needs_encoding = is_raw_caps(&self.input_caps);
|
||||
let needs_encoding = !is_raw_caps(&self.codec.caps) && is_raw_caps(&self.input_caps);
|
||||
let mut elements: Vec<gst::Element> = Vec::new();
|
||||
|
||||
let (raw_filter, encoder) = if needs_encoding {
|
||||
|
@ -3503,13 +3503,40 @@ impl BaseWebRTCSink {
|
|||
output_caps: gst::Caps,
|
||||
codecs: &Codecs,
|
||||
) -> Result<(), Error> {
|
||||
let futs = if let Some(codec) = codecs.find_for_encoded_caps(&discovery_info.caps) {
|
||||
let futs = if is_raw_caps(&discovery_info.caps) {
|
||||
let sink_caps = discovery_info.caps.clone();
|
||||
|
||||
let is_video = match sink_caps.structure(0).unwrap().name().as_str() {
|
||||
"video/x-raw" => true,
|
||||
"audio/x-raw" => false,
|
||||
_ => anyhow::bail!("expected audio or video raw caps: {sink_caps}"),
|
||||
};
|
||||
|
||||
codecs
|
||||
.iter()
|
||||
.filter(|codec| {
|
||||
codec.is_video() == is_video
|
||||
&& (!is_raw_caps(&codec.caps) || codec.caps.can_intersect(&sink_caps))
|
||||
})
|
||||
.map(|codec| {
|
||||
BaseWebRTCSink::run_discovery_pipeline(
|
||||
element,
|
||||
&name,
|
||||
&discovery_info,
|
||||
codec.clone(),
|
||||
sink_caps.clone(),
|
||||
&output_caps,
|
||||
ExtensionConfigurationType::Auto,
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
} else if let Some(codec) = codecs.find_for_payloadable_caps(&discovery_info.caps) {
|
||||
let mut caps = discovery_info.caps.clone();
|
||||
|
||||
gst::info!(
|
||||
CAT,
|
||||
obj: element,
|
||||
"Stream is already encoded with codec {}, still need to payload it",
|
||||
"Stream already conforms to {}, still need to payload it",
|
||||
codec.name
|
||||
);
|
||||
|
||||
|
@ -3525,29 +3552,7 @@ impl BaseWebRTCSink {
|
|||
ExtensionConfigurationType::Auto,
|
||||
)]
|
||||
} else {
|
||||
let sink_caps = discovery_info.caps.clone();
|
||||
|
||||
let is_video = match sink_caps.structure(0).unwrap().name().as_str() {
|
||||
"video/x-raw" => true,
|
||||
"audio/x-raw" => false,
|
||||
_ => anyhow::bail!("Unsupported caps: {}", discovery_info.caps),
|
||||
};
|
||||
|
||||
codecs
|
||||
.iter()
|
||||
.filter(|codec| codec.is_video() == is_video)
|
||||
.map(|codec| {
|
||||
BaseWebRTCSink::run_discovery_pipeline(
|
||||
element,
|
||||
&name,
|
||||
&discovery_info,
|
||||
codec.clone(),
|
||||
sink_caps.clone(),
|
||||
&output_caps,
|
||||
ExtensionConfigurationType::Auto,
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
anyhow::bail!("Unsupported caps: {}", discovery_info.caps);
|
||||
};
|
||||
|
||||
let mut payloader_caps = gst::Caps::new_empty();
|
||||
|
@ -4258,6 +4263,8 @@ impl GstObjectImpl for BaseWebRTCSink {}
|
|||
impl ElementImpl for BaseWebRTCSink {
|
||||
fn pad_templates() -> &'static [gst::PadTemplate] {
|
||||
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
|
||||
// Ignore specific raw caps from Codecs: they are covered by video/x-raw & audio/x-raw
|
||||
|
||||
let mut caps_builder = gst::Caps::builder_full()
|
||||
.structure(gst::Structure::builder("video/x-raw").build())
|
||||
.structure_with_features(
|
||||
|
@ -4277,7 +4284,10 @@ impl ElementImpl for BaseWebRTCSink {
|
|||
gst::CapsFeatures::new([D3D11_MEMORY_FEATURE]),
|
||||
);
|
||||
|
||||
for codec in Codecs::video_codecs() {
|
||||
for codec in Codecs::video_codecs()
|
||||
.iter()
|
||||
.filter(|codec| !is_raw_caps(&codec.caps))
|
||||
{
|
||||
caps_builder = caps_builder.structure(codec.caps.structure(0).unwrap().to_owned());
|
||||
}
|
||||
|
||||
|
@ -4292,7 +4302,10 @@ impl ElementImpl for BaseWebRTCSink {
|
|||
|
||||
let mut caps_builder =
|
||||
gst::Caps::builder_full().structure(gst::Structure::builder("audio/x-raw").build());
|
||||
for codec in Codecs::audio_codecs() {
|
||||
for codec in Codecs::audio_codecs()
|
||||
.iter()
|
||||
.filter(|codec| !is_raw_caps(&codec.caps))
|
||||
{
|
||||
caps_builder = caps_builder.structure(codec.caps.structure(0).unwrap().to_owned());
|
||||
}
|
||||
let audio_pad_template = gst::PadTemplate::with_gtype(
|
||||
|
|
|
@ -274,11 +274,11 @@ impl Default for Settings {
|
|||
meta: Default::default(),
|
||||
audio_codecs: Codecs::audio_codecs()
|
||||
.into_iter()
|
||||
.filter(|codec| codec.has_decoder())
|
||||
.filter(|codec| codec.can_be_received())
|
||||
.collect(),
|
||||
video_codecs: Codecs::video_codecs()
|
||||
.into_iter()
|
||||
.filter(|codec| codec.has_decoder())
|
||||
.filter(|codec| codec.can_be_received())
|
||||
.collect(),
|
||||
enable_data_channel_navigation: DEFAULT_ENABLE_DATA_CHANNEL_NAVIGATION,
|
||||
do_retransmission: DEFAULT_DO_RETRANSMISSION,
|
||||
|
@ -1027,11 +1027,18 @@ impl BaseWebRTCSrc {
|
|||
impl ElementImpl for BaseWebRTCSrc {
|
||||
fn pad_templates() -> &'static [gst::PadTemplate] {
|
||||
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
|
||||
use crate::utils::is_raw_caps;
|
||||
|
||||
// Ignore specific raw caps from Codecs: they are covered by VIDEO_CAPS & AUDIO_CAPS
|
||||
|
||||
let mut video_caps_builder = gst::Caps::builder_full()
|
||||
.structure_with_any_features(VIDEO_CAPS.structure(0).unwrap().to_owned())
|
||||
.structure(RTP_CAPS.structure(0).unwrap().to_owned());
|
||||
|
||||
for codec in Codecs::video_codecs() {
|
||||
for codec in Codecs::video_codecs()
|
||||
.iter()
|
||||
.filter(|codec| !is_raw_caps(&codec.caps))
|
||||
{
|
||||
video_caps_builder =
|
||||
video_caps_builder.structure(codec.caps.structure(0).unwrap().to_owned());
|
||||
}
|
||||
|
@ -1040,7 +1047,10 @@ impl ElementImpl for BaseWebRTCSrc {
|
|||
.structure_with_any_features(AUDIO_CAPS.structure(0).unwrap().to_owned())
|
||||
.structure(RTP_CAPS.structure(0).unwrap().to_owned());
|
||||
|
||||
for codec in Codecs::audio_codecs() {
|
||||
for codec in Codecs::audio_codecs()
|
||||
.iter()
|
||||
.filter(|codec| !is_raw_caps(&codec.caps))
|
||||
{
|
||||
audio_caps_builder =
|
||||
audio_caps_builder.structure(codec.caps.structure(0).unwrap().to_owned());
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue