From f563f8334be687cd954744dffce555807caee82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 14 Oct 2023 11:23:56 +0300 Subject: [PATCH] rtp: Add PCMU/PCMA RTP payloader / depayloader elements These come with new generic RTP payloader, RTP raw-ish audio payloader and RTP depayloader base classes. Part-of: --- Cargo.lock | 89 +- docs/plugins/gst_plugins_cache.json | 539 ++++++- net/rtp/Cargo.toml | 11 +- net/rtp/src/audio_discont.rs | 193 +++ net/rtp/src/baseaudiopay/imp.rs | 518 +++++++ net/rtp/src/baseaudiopay/mod.rs | 40 + net/rtp/src/basedepay/imp.rs | 1937 ++++++++++++++++++++++++ net/rtp/src/basedepay/mod.rs | 543 +++++++ net/rtp/src/basepay/imp.rs | 2115 +++++++++++++++++++++++++++ net/rtp/src/basepay/mod.rs | 494 +++++++ net/rtp/src/lib.rs | 26 + net/rtp/src/pcmau/depay/imp.rs | 286 ++++ net/rtp/src/pcmau/depay/mod.rs | 59 + net/rtp/src/pcmau/mod.rs | 14 + net/rtp/src/pcmau/pay/imp.rs | 294 ++++ net/rtp/src/pcmau/pay/mod.rs | 59 + net/rtp/src/pcmau/tests.rs | 239 +++ net/rtp/src/tests.rs | 515 +++++++ 18 files changed, 7930 insertions(+), 41 deletions(-) create mode 100644 net/rtp/src/audio_discont.rs create mode 100644 net/rtp/src/baseaudiopay/imp.rs create mode 100644 net/rtp/src/baseaudiopay/mod.rs create mode 100644 net/rtp/src/basedepay/imp.rs create mode 100644 net/rtp/src/basedepay/mod.rs create mode 100644 net/rtp/src/basepay/imp.rs create mode 100644 net/rtp/src/basepay/mod.rs create mode 100644 net/rtp/src/pcmau/depay/imp.rs create mode 100644 net/rtp/src/pcmau/depay/mod.rs create mode 100644 net/rtp/src/pcmau/mod.rs create mode 100644 net/rtp/src/pcmau/pay/imp.rs create mode 100644 net/rtp/src/pcmau/pay/mod.rs create mode 100644 net/rtp/src/pcmau/tests.rs create mode 100644 net/rtp/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 634f837d..a2ee5be9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2011,7 +2011,7 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "gio" version = "0.20.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#03c5a850037c582a0a5fca07f8e1cd1cf924c4fc" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#90fc2033d4824e1c80a575d03b3c164e9901faf8" dependencies = [ "futures-channel", "futures-core", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "gio-sys" version = "0.20.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#03c5a850037c582a0a5fca07f8e1cd1cf924c4fc" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#90fc2033d4824e1c80a575d03b3c164e9901faf8" dependencies = [ "glib-sys", "gobject-sys", @@ -2040,7 +2040,7 @@ dependencies = [ [[package]] name = "glib" version = "0.20.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#03c5a850037c582a0a5fca07f8e1cd1cf924c4fc" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#90fc2033d4824e1c80a575d03b3c164e9901faf8" dependencies = [ "bitflags 2.4.2", "futures-channel", @@ -2061,7 +2061,7 @@ dependencies = [ [[package]] name = "glib-macros" version = "0.20.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#03c5a850037c582a0a5fca07f8e1cd1cf924c4fc" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#90fc2033d4824e1c80a575d03b3c164e9901faf8" dependencies = [ "heck", "proc-macro-crate", @@ -2073,7 +2073,7 @@ dependencies = [ [[package]] name = "glib-sys" version = "0.20.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#03c5a850037c582a0a5fca07f8e1cd1cf924c4fc" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#90fc2033d4824e1c80a575d03b3c164e9901faf8" dependencies = [ "libc", "system-deps", @@ -2088,7 +2088,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "gobject-sys" version = "0.20.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#03c5a850037c582a0a5fca07f8e1cd1cf924c4fc" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=master#90fc2033d4824e1c80a575d03b3c164e9901faf8" dependencies = [ "glib-sys", "libc", @@ -2630,13 +2630,18 @@ dependencies = [ name = "gst-plugin-rtp" version = "0.13.0-alpha.1" dependencies = [ + "atomic_refcell", "bitstream-io", "chrono", "gst-plugin-version-helper", "gstreamer", + "gstreamer-app", "gstreamer-check", "gstreamer-rtp", "once_cell", + "rand", + "rtp-types", + "smallvec", ] [[package]] @@ -2954,7 +2959,7 @@ dependencies = [ [[package]] name = "gstreamer" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "cfg-if", "futures-channel", @@ -2980,7 +2985,7 @@ dependencies = [ [[package]] name = "gstreamer-app" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "futures-core", "futures-sink", @@ -2994,7 +2999,7 @@ dependencies = [ [[package]] name = "gstreamer-app-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-base-sys", @@ -3006,7 +3011,7 @@ dependencies = [ [[package]] name = "gstreamer-audio" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "cfg-if", "glib", @@ -3021,7 +3026,7 @@ dependencies = [ [[package]] name = "gstreamer-audio-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3034,7 +3039,7 @@ dependencies = [ [[package]] name = "gstreamer-base" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "atomic_refcell", "cfg-if", @@ -3047,7 +3052,7 @@ dependencies = [ [[package]] name = "gstreamer-base-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3059,7 +3064,7 @@ dependencies = [ [[package]] name = "gstreamer-check" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3069,7 +3074,7 @@ dependencies = [ [[package]] name = "gstreamer-check-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3081,7 +3086,7 @@ dependencies = [ [[package]] name = "gstreamer-gl" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3095,7 +3100,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-egl" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3107,7 +3112,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-egl-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-gl-sys", @@ -3118,7 +3123,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3132,7 +3137,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-wayland" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3144,7 +3149,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-wayland-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-gl-sys", @@ -3155,7 +3160,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-x11" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3167,7 +3172,7 @@ dependencies = [ [[package]] name = "gstreamer-gl-x11-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-gl-sys", @@ -3178,7 +3183,7 @@ dependencies = [ [[package]] name = "gstreamer-net" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "gio", "glib", @@ -3189,7 +3194,7 @@ dependencies = [ [[package]] name = "gstreamer-net-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "gio-sys", "glib-sys", @@ -3201,7 +3206,7 @@ dependencies = [ [[package]] name = "gstreamer-pbutils" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3215,7 +3220,7 @@ dependencies = [ [[package]] name = "gstreamer-pbutils-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3229,7 +3234,7 @@ dependencies = [ [[package]] name = "gstreamer-rtp" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3240,7 +3245,7 @@ dependencies = [ [[package]] name = "gstreamer-rtp-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-base-sys", @@ -3252,7 +3257,7 @@ dependencies = [ [[package]] name = "gstreamer-sdp" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3262,7 +3267,7 @@ dependencies = [ [[package]] name = "gstreamer-sdp-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-sys", @@ -3273,7 +3278,7 @@ dependencies = [ [[package]] name = "gstreamer-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3284,7 +3289,7 @@ dependencies = [ [[package]] name = "gstreamer-utils" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "gstreamer", "gstreamer-app", @@ -3296,7 +3301,7 @@ dependencies = [ [[package]] name = "gstreamer-video" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "cfg-if", "futures-channel", @@ -3313,7 +3318,7 @@ dependencies = [ [[package]] name = "gstreamer-video-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gobject-sys", @@ -3326,7 +3331,7 @@ dependencies = [ [[package]] name = "gstreamer-webrtc" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib", "gstreamer", @@ -3338,7 +3343,7 @@ dependencies = [ [[package]] name = "gstreamer-webrtc-sys" version = "0.23.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#b10f395c2cf0dc6d1af35068c0960ea5b839158d" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#28fe70f479fcb2f22971fc29a3b81e1cef097578" dependencies = [ "glib-sys", "gstreamer-sdp-sys", @@ -5390,6 +5395,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "rtp-types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01b38bb7fd9425628876786934ade84ec5cb63905804f073583e6554d33f9af" +dependencies = [ + "smallvec", + "thiserror", +] + [[package]] name = "rtsp-types" version = "0.1.1" diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index d87d4877..f3837d0d 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -6311,11 +6311,548 @@ } }, "rank": "none" + }, + "rtppcmadepay2": { + "author": "Sebastian Dröge ", + "description": "Depayload A-law from RTP packets (RFC 3551)", + "hierarchy": [ + "GstRtpPcmaDepay2", + "GstRtpPcmauDepay2", + "GstRtpBaseDepay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Depayloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "application/x-rtp:\n media: audio\n payload: 8\n clock-rate: 8000\napplication/x-rtp:\n media: audio\n clock-rate: [ 1, 2147483647 ]\n encoding-name: PCMA\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "audio/x-alaw:\n channels: 1\n rate: [ 1, 2147483647 ]\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + }, + "rtppcmapay2": { + "author": "Sebastian Dröge ", + "description": "Payload A-law Audio into RTP packets (RFC 3551)", + "hierarchy": [ + "GstRtpPcmaPay2", + "GstRtpPcmauPay2", + "GstRtpBaseAudioPay2", + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Payloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "audio/x-alaw:\n channels: 1\n rate: [ 1, 2147483647 ]\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "application/x-rtp:\n media: audio\n payload: 8\n clock-rate: 8000\napplication/x-rtp:\n media: audio\n payload: [ 96, 127 ]\n encoding-name: PCMA\n clock-rate: [ 1, 2147483647 ]\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + }, + "rtppcmudepay2": { + "author": "Sebastian Dröge ", + "description": "Depayload µ-law from RTP packets (RFC 3551)", + "hierarchy": [ + "GstRtpPcmuDepay2", + "GstRtpPcmauDepay2", + "GstRtpBaseDepay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Depayloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "application/x-rtp:\n media: audio\n payload: 0\n clock-rate: 8000\napplication/x-rtp:\n media: audio\n clock-rate: [ 1, 2147483647 ]\n encoding-name: PCMU\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "audio/x-mulaw:\n channels: 1\n rate: [ 1, 2147483647 ]\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + }, + "rtppcmupay2": { + "author": "Sebastian Dröge ", + "description": "Payload µ-law Audio into RTP packets (RFC 3551)", + "hierarchy": [ + "GstRtpPcmuPay2", + "GstRtpPcmauPay2", + "GstRtpBaseAudioPay2", + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Payloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "audio/x-mulaw:\n channels: 1\n rate: [ 1, 2147483647 ]\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "application/x-rtp:\n media: audio\n payload: 0\n clock-rate: 8000\napplication/x-rtp:\n media: audio\n payload: [ 96, 127 ]\n encoding-name: PCMU\n clock-rate: [ 1, 2147483647 ]\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" } }, "filename": "gstrsrtp", "license": "MPL-2.0", - "other-types": {}, + "other-types": { + "GstRtpBaseAudioPay2": { + "hierarchy": [ + "GstRtpBaseAudioPay2", + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "alignment-threshold": { + "blurb": "Timestamp alignment threshold in nanoseconds", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "40000000", + "max": "18446744073709551615", + "min": "0", + "mutable": "playing", + "readable": true, + "type": "guint64", + "writable": true + }, + "discont-wait": { + "blurb": "Window of time in nanoseconds to wait before creating a discontinuity", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "1000000000", + "max": "18446744073709551615", + "min": "0", + "mutable": "playing", + "readable": true, + "type": "guint64", + "writable": true + }, + "max-ptime": { + "blurb": "Maximum duration of the packet data in ns (-1 = unlimited up to MTU)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "18446744073709551615", + "max": "9223372036854775807", + "min": "-1", + "mutable": "playing", + "readable": true, + "type": "gint64", + "writable": true + }, + "min-ptime": { + "blurb": "Minimum duration of the packet data in ns (can't go above MTU)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "9223372036854775807", + "min": "0", + "mutable": "playing", + "readable": true, + "type": "gint64", + "writable": true + }, + "ptime-multiple": { + "blurb": "Force buffers to be multiples of this duration in ns (0 disables)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "9223372036854775807", + "min": "0", + "mutable": "playing", + "readable": true, + "type": "gint64", + "writable": true + } + } + }, + "GstRtpBaseDepay2": { + "hierarchy": [ + "GstRtpBaseDepay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "auto-header-extensions": { + "blurb": "Whether RTP header extensions should be automatically enabled, if an implementation is available", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true + }, + "extensions": { + "blurb": "List of enabled RTP header extensions", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "mutable": "null", + "readable": true, + "type": "GstValueArray", + "writable": false + }, + "max-reorder": { + "blurb": "Maximum seqnum reorder before assuming sender has restarted", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "100", + "max": "32767", + "min": "0", + "mutable": "playing", + "readable": true, + "type": "guint", + "writable": true + }, + "source-info": { + "blurb": "Add RTP source information as buffer metadata", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "playing", + "readable": true, + "type": "gboolean", + "writable": true + }, + "stats": { + "blurb": "Various statistics", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "application/x-rtp-depayload-stats;", + "mutable": "null", + "readable": true, + "type": "GstStructure", + "writable": false + } + }, + "signals": { + "add-extension": { + "action": true, + "args": [ + { + "name": "arg0", + "type": "GstRTPHeaderExtension" + } + ], + "return-type": "void", + "when": "last" + }, + "clear-extensions": { + "action": true, + "args": [], + "return-type": "void", + "when": "last" + }, + "request-extension": { + "args": [ + { + "name": "arg0", + "type": "guint" + }, + { + "name": "arg1", + "type": "gchararray" + } + ], + "return-type": "GstRTPHeaderExtension", + "when": "last" + } + } + }, + "GstRtpBasePay2": { + "hierarchy": [ + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "auto-header-extensions": { + "blurb": "Whether RTP header extensions should be automatically enabled, if an implementation is available", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true + }, + "extensions": { + "blurb": "List of enabled RTP header extensions", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "mutable": "null", + "readable": true, + "type": "GstValueArray", + "writable": false + }, + "mtu": { + "blurb": "Maximum size of one RTP packet", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "1400", + "max": "-1", + "min": "28", + "mutable": "playing", + "readable": true, + "type": "guint", + "writable": true + }, + "onvif-no-rate-control": { + "blurb": "Enable ONVIF Rate-Control=no timestamping mode", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true + }, + "pt": { + "blurb": "Payload type of the packets", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "8", + "max": "127", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint", + "writable": true + }, + "scale-rtptime": { + "blurb": "Whether the RTP timestamp should be scaled with the rate (speed)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true + }, + "seqnum": { + "blurb": "RTP sequence number of the last packet", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "65535", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": false + }, + "seqnum-offset": { + "blurb": "Offset that is added to all RTP sequence numbers (-1 == random)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "65535", + "min": "-1", + "mutable": "ready", + "readable": true, + "type": "gint", + "writable": true + }, + "source-info": { + "blurb": "Add RTP source information as buffer metadata", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "playing", + "readable": true, + "type": "gboolean", + "writable": true + }, + "ssrc": { + "blurb": "SSRC of the packets (-1 == random)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "18446744073709551615", + "max": "4294967295", + "min": "-1", + "mutable": "ready", + "readable": true, + "type": "gint64", + "writable": true + }, + "stats": { + "blurb": "Various statistics", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "application/x-rtp-payload-stats;", + "mutable": "null", + "readable": true, + "type": "GstStructure", + "writable": false + }, + "timestamp": { + "blurb": "RTP timestamp of the last packet", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "65535", + "min": "0", + "mutable": "null", + "readable": true, + "type": "guint", + "writable": false + }, + "timestamp-offset": { + "blurb": "Offset that is added to all RTP timestamps (-1 == random)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "18446744073709551615", + "max": "4294967295", + "min": "-1", + "mutable": "ready", + "readable": true, + "type": "gint64", + "writable": true + } + }, + "signals": { + "add-extension": { + "action": true, + "args": [ + { + "name": "arg0", + "type": "GstRTPHeaderExtension" + } + ], + "return-type": "void", + "when": "last" + }, + "clear-extensions": { + "action": true, + "args": [], + "return-type": "void", + "when": "last" + }, + "request-extension": { + "args": [ + { + "name": "arg0", + "type": "guint" + }, + { + "name": "arg1", + "type": "gchararray" + } + ], + "return-type": "GstRTPHeaderExtension", + "when": "last" + } + } + }, + "GstRtpPcmauDepay2": { + "hierarchy": [ + "GstRtpPcmauDepay2", + "GstRtpBaseDepay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object" + }, + "GstRtpPcmauPay2": { + "hierarchy": [ + "GstRtpPcmauPay2", + "GstRtpBaseAudioPay2", + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object" + } + }, "package": "gst-plugin-rtp", "source": "gst-plugin-rtp", "tracers": {}, diff --git a/net/rtp/Cargo.toml b/net/rtp/Cargo.toml index 07e886f4..7361a4ea 100644 --- a/net/rtp/Cargo.toml +++ b/net/rtp/Cargo.toml @@ -9,14 +9,19 @@ description = "GStreamer Rust RTP Plugin" rust-version.workspace = true [dependencies] -bitstream-io = "2.0" -gst = { workspace = true, features = ["v1_20"] } -gst-rtp = { workspace = true, features = ["v1_20"]} +atomic_refcell = "0.1" +bitstream-io = "2.1" chrono = { version = "0.4", default-features = false } +gst = { workspace = true, features = ["v1_20"] } +gst-rtp = { workspace = true, features = ["v1_20"] } once_cell.workspace = true +rand = { version = "0.8", default-features = false, features = ["std", "std_rng" ] } +rtp-types = { version = "0.1" } +smallvec = { version = "1.11", features = ["union", "write", "const_generics", "const_new"] } [dev-dependencies] gst-check = { workspace = true, features = ["v1_20"] } +gst-app = { workspace = true, features = ["v1_20"] } [build-dependencies] gst-plugin-version-helper.workspace = true diff --git a/net/rtp/src/audio_discont.rs b/net/rtp/src/audio_discont.rs new file mode 100644 index 00000000..8681b348 --- /dev/null +++ b/net/rtp/src/audio_discont.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: MPL-2.0 + +use gst::{ + glib::{self, prelude::*}, + prelude::*, +}; + +#[derive(Debug, Default)] +pub struct AudioDiscont { + /// If last processing detected a discontinuity. + discont_pending: bool, + /// Base PTS to which the offsets below are relative. + base_pts: Option, + /// Next output sample offset, i.e. offset of the first sample of the queued buffers. + /// + /// This is only set once the packet with the base PTS is output. + next_out_offset: Option, + /// Next expected input sample offset. + next_in_offset: Option, + /// PTS of the last buffer that was above the alignment threshold. + /// + /// This is reset whenever the next buffer is actually below the alignment threshold again. + /// FIXME: Should this be running time? + discont_time: Option, + /// Last known sample rate. + last_rate: Option, +} + +impl AudioDiscont { + pub fn process_input( + &mut self, + settings: &AudioDiscontConfiguration, + discont: bool, + rate: u32, + pts: gst::ClockTime, + num_samples: usize, + ) -> bool { + if self.discont_pending { + return true; + } + + if self.last_rate.map_or(false, |last_rate| last_rate != rate) { + self.discont_pending = true; + } + self.last_rate = Some(rate); + + if discont { + self.discont_pending = true; + return true; + } + + // If we have no base PTS yet, this is the first buffer and there's a discont + let Some(base_pts) = self.base_pts else { + self.discont_pending = true; + return true; + }; + + // Never detect a discont if alignment threshold is not set + let Some(alignment_threshold) = settings.alignment_threshold else { + return false; + }; + + let expected_pts = base_pts + + gst::ClockTime::from_nseconds( + self.next_in_offset + .unwrap_or(0) + .mul_div_ceil(gst::ClockTime::SECOND.nseconds(), rate as u64) + .unwrap(), + ); + + let mut discont = false; + + let diff = pts.into_positive() - expected_pts.into_positive(); + if diff.abs() >= alignment_threshold { + let mut resync = false; + if settings.discont_wait.is_zero() { + resync = true; + } else if let Some(discont_time) = self.discont_time { + if (discont_time.into_positive() - pts.into_positive()).abs() + >= settings.discont_wait.into_positive() + { + resync = true; + } + } else if (expected_pts.into_positive() - pts.into_positive()).abs() + >= settings.discont_wait.into_positive() + { + resync = true; + } else { + self.discont_time = Some(expected_pts); + } + + if resync { + discont = true; + } + } else { + self.discont_time = None; + } + + self.next_in_offset = Some(self.next_in_offset.unwrap_or(0) + num_samples as u64); + + if discont { + self.discont_pending = true; + } + + discont + } + + pub fn resync(&mut self, base_pts: gst::ClockTime, num_samples: usize) { + self.discont_pending = false; + self.base_pts = Some(base_pts); + self.next_in_offset = Some(num_samples as u64); + self.next_out_offset = None; + self.discont_time = None; + self.last_rate = None; + } + + pub fn reset(&mut self) { + *self = AudioDiscont::default(); + } + + pub fn base_pts(&self) -> Option { + self.base_pts + } + + pub fn next_output_offset(&self) -> Option { + self.next_out_offset + } + + pub fn process_output(&mut self, num_samples: usize) { + self.next_out_offset = Some(self.next_out_offset.unwrap_or(0) + num_samples as u64); + } +} + +#[derive(Debug, Copy, Clone)] +pub struct AudioDiscontConfiguration { + pub alignment_threshold: Option, + pub discont_wait: gst::ClockTime, +} + +impl Default for AudioDiscontConfiguration { + fn default() -> Self { + AudioDiscontConfiguration { + alignment_threshold: Some(gst::ClockTime::from_mseconds(40)), + discont_wait: gst::ClockTime::from_seconds(1), + } + } +} + +impl AudioDiscontConfiguration { + pub fn create_pspecs() -> Vec { + vec![ + glib::ParamSpecUInt64::builder("alignment-threshold") + .nick("Alignment Threshold") + .blurb("Timestamp alignment threshold in nanoseconds") + .default_value( + Self::default() + .alignment_threshold + .map(gst::ClockTime::nseconds) + .unwrap_or(u64::MAX), + ) + .mutable_playing() + .build(), + glib::ParamSpecUInt64::builder("discont-wait") + .nick("Discont Wait") + .blurb("Window of time in nanoseconds to wait before creating a discontinuity") + .default_value(Self::default().discont_wait.nseconds()) + .mutable_playing() + .build(), + ] + } + + pub fn set_property(&mut self, value: &glib::Value, pspec: &glib::ParamSpec) -> bool { + match pspec.name() { + "alignment-threshold" => { + self.alignment_threshold = value.get().unwrap(); + true + } + "discont-wait" => { + self.discont_wait = value.get().unwrap(); + true + } + _ => false, + } + } + + pub fn property(&self, pspec: &glib::ParamSpec) -> Option { + match pspec.name() { + "alignment-threshold" => Some(self.alignment_threshold.to_value()), + "discont-wait" => Some(self.discont_wait.to_value()), + _ => None, + } + } +} diff --git a/net/rtp/src/baseaudiopay/imp.rs b/net/rtp/src/baseaudiopay/imp.rs new file mode 100644 index 00000000..ba386c1f --- /dev/null +++ b/net/rtp/src/baseaudiopay/imp.rs @@ -0,0 +1,518 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use std::{cmp, collections::VecDeque, sync::Mutex}; + +use atomic_refcell::AtomicRefCell; +use gst::{glib, prelude::*, subclass::prelude::*}; + +use once_cell::sync::Lazy; + +use crate::{ + audio_discont::{AudioDiscont, AudioDiscontConfiguration}, + basepay::{PacketToBufferRelation, RtpBasePay2Ext, RtpBasePay2ImplExt, TimestampOffset}, +}; + +#[derive(Clone)] +struct Settings { + max_ptime: Option, + min_ptime: gst::ClockTime, + ptime_multiple: gst::ClockTime, + audio_discont: AudioDiscontConfiguration, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + max_ptime: None, + min_ptime: gst::ClockTime::ZERO, + ptime_multiple: gst::ClockTime::ZERO, + audio_discont: AudioDiscontConfiguration::default(), + } + } +} + +struct QueuedBuffer { + /// ID of the buffer. + id: u64, + /// The mapped buffer itself. + buffer: gst::MappedBuffer, + /// Offset into the buffer that was not consumed yet. + offset: usize, +} + +#[derive(Default)] +struct State { + /// Currently configured clock rate. + clock_rate: Option, + /// Number of bytes per frame. + bpf: Option, + + /// Desired "packet time", i.e. packet duration, from the caps, if set. + ptime: Option, + max_ptime: Option, + + /// Currently queued buffers. + queued_buffers: VecDeque, + /// Currently queued number of bytes. + queued_bytes: usize, + + audio_discont: AudioDiscont, +} + +#[derive(Default)] +pub struct RtpBaseAudioPay2 { + settings: Mutex, + state: AtomicRefCell, +} + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpbaseaudiopay2", + gst::DebugColorFlags::empty(), + Some("Base RTP Audio Payloader"), + ) +}); + +#[glib::object_subclass] +impl ObjectSubclass for RtpBaseAudioPay2 { + const ABSTRACT: bool = true; + const NAME: &'static str = "GstRtpBaseAudioPay2"; + type Type = super::RtpBaseAudioPay2; + type ParentType = crate::basepay::RtpBasePay2; +} + +impl ObjectImpl for RtpBaseAudioPay2 { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + let mut properties = vec![ + // Using same type/semantics as C payloaders + glib::ParamSpecInt64::builder("max-ptime") + .nick("Maximum Packet Time") + .blurb("Maximum duration of the packet data in ns (-1 = unlimited up to MTU)") + .default_value( + Settings::default() + .max_ptime + .map(gst::ClockTime::nseconds) + .map(|x| x as i64) + .unwrap_or(-1), + ) + .minimum(-1) + .maximum(i64::MAX) + .mutable_playing() + .build(), + // Using same type/semantics as C payloaders + glib::ParamSpecInt64::builder("min-ptime") + .nick("Minimum Packet Time") + .blurb("Minimum duration of the packet data in ns (can't go above MTU)") + .default_value(Settings::default().min_ptime.nseconds() as i64) + .minimum(0) + .maximum(i64::MAX) + .mutable_playing() + .build(), + // Using same type/semantics as C payloaders + glib::ParamSpecInt64::builder("ptime-multiple") + .nick("Packet Time Multiple") + .blurb("Force buffers to be multiples of this duration in ns (0 disables)") + .default_value(Settings::default().ptime_multiple.nseconds() as i64) + .minimum(0) + .maximum(i64::MAX) + .mutable_playing() + .build(), + ]; + + properties.extend_from_slice(&AudioDiscontConfiguration::create_pspecs()); + + properties + }); + + PROPERTIES.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + if self + .settings + .lock() + .unwrap() + .audio_discont + .set_property(value, pspec) + { + return; + } + + match pspec.name() { + "max-ptime" => { + let v = value.get::().unwrap(); + self.settings.lock().unwrap().max_ptime = + (v != -1).then_some(gst::ClockTime::from_nseconds(v as u64)); + } + "min-ptime" => { + let v = gst::ClockTime::from_nseconds(value.get::().unwrap() as u64); + + let mut settings = self.settings.lock().unwrap(); + let changed = settings.min_ptime != v; + settings.min_ptime = v; + drop(settings); + + if changed { + let _ = self + .obj() + .post_message(gst::message::Latency::builder().src(&*self.obj()).build()); + } + } + "ptime-multiple" => { + self.settings.lock().unwrap().ptime_multiple = + gst::ClockTime::from_nseconds(value.get::().unwrap() as u64); + } + _ => unimplemented!(), + }; + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + if let Some(value) = self.settings.lock().unwrap().audio_discont.property(pspec) { + return value; + } + + match pspec.name() { + "max-ptime" => (self + .settings + .lock() + .unwrap() + .max_ptime + .map(gst::ClockTime::nseconds) + .map(|x| x as i64) + .unwrap_or(-1)) + .to_value(), + "min-ptime" => (self.settings.lock().unwrap().min_ptime.nseconds() as i64).to_value(), + "ptime-multiple" => { + (self.settings.lock().unwrap().ptime_multiple.nseconds() as i64).to_value() + } + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for RtpBaseAudioPay2 {} + +impl ElementImpl for RtpBaseAudioPay2 {} + +impl crate::basepay::RtpBasePay2Impl for RtpBaseAudioPay2 { + const ALLOWED_META_TAGS: &'static [&'static str] = &["audio"]; + + fn start(&self) -> Result<(), gst::ErrorMessage> { + *self.state.borrow_mut() = State::default(); + Ok(()) + } + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + *self.state.borrow_mut() = State::default(); + Ok(()) + } + + fn negotiate(&self, mut src_caps: gst::Caps) { + // Fixate here as a first step + src_caps.fixate(); + + let s = src_caps.structure(0).unwrap(); + + // Negotiate ptime/maxptime with downstream and use them in combination with the + // properties. See https://datatracker.ietf.org/doc/html/rfc4566#section-6 + let ptime = s + .get::("ptime") + .ok() + .map(u64::from) + .map(gst::ClockTime::from_mseconds); + + let max_ptime = s + .get::("maxptime") + .ok() + .map(u64::from) + .map(gst::ClockTime::from_mseconds); + + let clock_rate = match s.get::("clock-rate") { + Ok(clock_rate) if clock_rate > 0 => clock_rate as u32, + _ => { + panic!("RTP caps {src_caps:?} without 'clock-rate'"); + } + }; + + self.parent_negotiate(src_caps); + + // Draining happened above if the clock rate has changed + let mut state = self.state.borrow_mut(); + state.ptime = ptime; + state.max_ptime = max_ptime; + state.clock_rate = Some(clock_rate); + drop(state); + } + + fn drain(&self) -> Result { + let settings = self.settings.lock().unwrap().clone(); + let mut state = self.state.borrow_mut(); + self.drain_packets(&settings, &mut state, true) + } + + fn flush(&self) { + let mut state = self.state.borrow_mut(); + state.queued_buffers.clear(); + state.queued_bytes = 0; + state.audio_discont.reset(); + } + + fn handle_buffer( + &self, + buffer: &gst::Buffer, + id: u64, + ) -> Result { + let buffer = buffer.clone().into_mapped_buffer_readable().map_err(|_| { + gst::error!(CAT, imp: self, "Can't map buffer readable"); + gst::FlowError::Error + })?; + let pts = buffer.buffer().pts().unwrap(); + + let settings = self.settings.lock().unwrap().clone(); + let mut state = self.state.borrow_mut(); + + let Some(bpf) = state.bpf else { + return Err(gst::FlowError::NotNegotiated); + }; + let Some(clock_rate) = state.clock_rate else { + return Err(gst::FlowError::NotNegotiated); + }; + let num_samples = buffer.size() / bpf; + + let discont = state.audio_discont.process_input( + &settings.audio_discont, + buffer.buffer().flags().contains(gst::BufferFlags::DISCONT), + clock_rate, + pts, + num_samples, + ); + + if discont { + if state.audio_discont.base_pts().is_some() { + gst::debug!(CAT, imp: self, "Draining because of discontinuity"); + self.drain_packets(&settings, &mut state, true)?; + } + + state.audio_discont.resync(pts, num_samples); + } + + state.queued_bytes += buffer.buffer().size(); + state.queued_buffers.push_back(QueuedBuffer { + id, + buffer, + offset: 0, + }); + + self.drain_packets(&settings, &mut state, false) + } + + #[allow(clippy::single_match)] + fn src_query(&self, query: &mut gst::QueryRef) -> bool { + let res = self.parent_src_query(query); + if !res { + return false; + } + + match query.view_mut() { + gst::QueryViewMut::Latency(query) => { + let (is_live, mut min, mut max) = query.result(); + let min_ptime = self.settings.lock().unwrap().min_ptime; + min += min_ptime; + max.opt_add_assign(min_ptime); + query.set(is_live, min, max); + } + _ => (), + } + + true + } +} + +impl RtpBaseAudioPay2 { + /// Returns the minimum, maximum and chunk/multiple packet sizes + fn calculate_packet_sizes( + &self, + settings: &Settings, + state: &State, + clock_rate: u32, + bpf: usize, + ) -> (usize, usize, usize) { + let min_pframes = settings + .min_ptime + .nseconds() + .mul_div_ceil(clock_rate as u64, gst::ClockTime::SECOND.nseconds()) + .unwrap() as u32; + let max_ptime = match (settings.max_ptime, state.max_ptime) { + (Some(max_ptime), Some(caps_max_ptime)) => Some(cmp::min(max_ptime, caps_max_ptime)), + (None, Some(max_ptime)) => Some(max_ptime), + (Some(max_ptime), None) => Some(max_ptime), + _ => None, + }; + let max_pframes = max_ptime.map(|max_ptime| { + max_ptime + .nseconds() + .mul_div_ceil(clock_rate as u64, gst::ClockTime::SECOND.nseconds()) + .unwrap() as u32 + }); + let pframes_multiple = cmp::max( + 1, + settings + .ptime_multiple + .nseconds() + .mul_div_ceil(clock_rate as u64, gst::ClockTime::SECOND.nseconds()) + .unwrap() as u32, + ); + + gst::trace!( + CAT, + imp: self, + "min ptime {} (frames: {}), max ptime {} (frames: {}), ptime multiple {} (frames {})", + settings.min_ptime, + min_pframes, + settings.max_ptime.display(), + max_pframes.unwrap_or(0), + settings.ptime_multiple, + pframes_multiple, + ); + + let psize_multiple = pframes_multiple as usize * bpf; + + let mut max_packet_size = self.obj().max_payload_size() as usize; + max_packet_size -= max_packet_size % psize_multiple; + if let Some(max_pframes) = max_pframes { + max_packet_size = cmp::min(max_pframes as usize * bpf, max_packet_size) + } + let mut min_packet_size = cmp::min( + cmp::max(min_pframes as usize * bpf, psize_multiple), + max_packet_size, + ); + + if let Some(ptime) = state.ptime { + let pframes = ptime + .nseconds() + .mul_div_ceil(clock_rate as u64, gst::ClockTime::SECOND.nseconds()) + .unwrap() as u32; + + let psize = pframes as usize * bpf; + min_packet_size = cmp::max(min_packet_size, psize); + max_packet_size = cmp::min(max_packet_size, psize); + } + + (min_packet_size, max_packet_size, psize_multiple) + } + + fn drain_packets( + &self, + settings: &Settings, + state: &mut State, + force: bool, + ) -> Result { + // Always set when caps are set + let Some(clock_rate) = state.clock_rate else { + return Ok(gst::FlowSuccess::Ok); + }; + let bpf = state.bpf.unwrap(); + + let (min_packet_size, max_packet_size, psize_multiple) = + self.calculate_packet_sizes(settings, state, clock_rate, bpf); + + gst::trace!( + CAT, + imp: self, + "Currently {} bytes queued, min packet size {min_packet_size}, max packet size {max_packet_size}, force {force}", + state.queued_bytes, + ); + + while state.queued_bytes >= min_packet_size || (force && state.queued_bytes > 0) { + let packet_size = { + let mut packet_size = cmp::min(max_packet_size, state.queued_bytes); + packet_size -= packet_size % psize_multiple; + packet_size + }; + + gst::trace!( + CAT, + imp: self, + "Creating packet of size {packet_size} ({} frames), marker {}", + packet_size / bpf, + state.audio_discont.next_output_offset().is_none(), + ); + + // Set marker bit on the first packet after a discontinuity + let mut packet_builder = rtp_types::RtpPacketBuilder::new() + .marker_bit(state.audio_discont.next_output_offset().is_none()); + + let front = state.queued_buffers.front().unwrap(); + let start_id = front.id; + let mut end_id = front.id; + + // Fill payload from all relevant buffers and collect start/end ids that apply. + let mut remaining_packet_size = packet_size; + for buffer in state.queued_buffers.iter() { + let this_buffer_payload_size = + cmp::min(buffer.buffer.size() - buffer.offset, remaining_packet_size); + + end_id = buffer.id; + packet_builder = packet_builder + .payload(&buffer.buffer[buffer.offset..][..this_buffer_payload_size]); + + remaining_packet_size -= this_buffer_payload_size; + if remaining_packet_size == 0 { + break; + } + } + + // Then create the packet. + self.obj().queue_packet( + PacketToBufferRelation::IdsWithOffset { + ids: start_id..=end_id, + timestamp_offset: { + if let Some(next_out_offset) = state.audio_discont.next_output_offset() { + TimestampOffset::Rtp(next_out_offset) + } else { + TimestampOffset::Pts(gst::ClockTime::ZERO) + } + }, + }, + packet_builder, + )?; + + // And finally dequeue or update all currently queued buffers. + let mut remaining_packet_size = packet_size; + while remaining_packet_size > 0 { + let buffer = state.queued_buffers.front_mut().unwrap(); + + if buffer.buffer.size() - buffer.offset > remaining_packet_size { + buffer.offset += remaining_packet_size; + remaining_packet_size = 0; + } else { + remaining_packet_size -= buffer.buffer.size() - buffer.offset; + let _ = state.queued_buffers.pop_front(); + } + } + state.queued_bytes -= packet_size; + state.audio_discont.process_output(packet_size / bpf); + } + + gst::trace!(CAT, imp: self, "Currently {} bytes / {} frames queued", state.queued_bytes, state.queued_bytes / bpf); + + Ok(gst::FlowSuccess::Ok) + } +} + +/// Wrapper functions for public API. +#[allow(dead_code)] +impl RtpBaseAudioPay2 { + pub(super) fn set_bpf(&self, bpf: usize) { + let mut state = self.state.borrow_mut(); + state.bpf = Some(bpf); + } +} diff --git a/net/rtp/src/baseaudiopay/mod.rs b/net/rtp/src/baseaudiopay/mod.rs new file mode 100644 index 00000000..75e943dc --- /dev/null +++ b/net/rtp/src/baseaudiopay/mod.rs @@ -0,0 +1,40 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::{glib, prelude::*, subclass::prelude::*}; + +use crate::basepay::RtpBasePay2Impl; + +pub mod imp; + +glib::wrapper! { + pub struct RtpBaseAudioPay2(ObjectSubclass) + @extends crate::basepay::RtpBasePay2, gst::Element, gst::Object; +} + +/// Trait containing extension methods for `RtpBaseAudioPay2`. +pub trait RtpBaseAudioPay2Ext: IsA { + /// Sets the number of bytes per frame. + /// + /// Should always be called together with `RtpBasePay2Ext::set_src_caps()`. + fn set_bpf(&self, bpf: usize) { + self.upcast_ref::().imp().set_bpf(bpf) + } +} + +impl> RtpBaseAudioPay2Ext for O {} + +/// Trait to implement in `RtpBaseAudioPay2` subclasses. +pub trait RtpBaseAudioPay2Impl: RtpBasePay2Impl {} + +unsafe impl IsSubclassable for RtpBaseAudioPay2 { + fn class_init(class: &mut glib::Class) { + Self::parent_class_init::(class); + } +} diff --git a/net/rtp/src/basedepay/imp.rs b/net/rtp/src/basedepay/imp.rs new file mode 100644 index 00000000..1c4d7674 --- /dev/null +++ b/net/rtp/src/basedepay/imp.rs @@ -0,0 +1,1937 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use atomic_refcell::AtomicRefCell; +use gst::{glib, prelude::*, subclass::prelude::*}; +use gst_rtp::prelude::*; + +use once_cell::sync::Lazy; + +use std::{ + collections::{BTreeMap, VecDeque}, + ops::{Bound, RangeBounds}, + sync::Mutex, +}; + +use super::{PacketToBufferRelation, TimestampOffset}; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpbasedepay2", + gst::DebugColorFlags::empty(), + Some("RTP Base Depayloader 2"), + ) +}); + +#[derive(Clone, Debug)] +struct Settings { + max_reorder: u32, + source_info: bool, + auto_header_extensions: bool, +} + +impl Default for Settings { + fn default() -> Self { + Self { + max_reorder: 100, + source_info: false, + auto_header_extensions: true, + } + } +} + +/// Metadata of a pending packet for tracking purposes. +struct PendingPacket { + /// Extended sequence number of the packet. + ext_seqnum: u64, + /// Extended RTP timestamp of the packet. + ext_timestamp: u64, + /// SSRC of the packet. + ssrc: u32, + /// CSRC of the packet. + csrc: [u32; rtp_types::RtpPacket::MAX_N_CSRCS], + n_csrc: u8, + /// Packet contains header extensions. + have_header_extensions: bool, + /// Buffer containing the packet. + buffer: gst::Buffer, +} + +/// A pending buffer that still has to be sent downstream. +struct PendingBuffer { + /// If no metadata is set then this buffer must be pushed together with + /// the next buffer where it is actually known, or at EOS with the + /// last known one. + metadata_set: bool, + buffer: gst::Buffer, +} + +/// Information about the current stream that is handled. +struct CurrentStream { + /// SSRC of the last received packet. + ssrc: u32, + /// Payload type of the last received packet. + pt: u8, + /// Extended sequence number of the last received packet. + ext_seqnum: u64, + /// Extended RTP time of the last received packet. + ext_rtptime: u64, +} + +struct State { + segment: Option<(gst::Seqnum, gst::FormattedSegment)>, + /// Set when a new segment event should be sent downstream before the next buffer. + pending_segment: bool, + + sink_caps: Option, + src_caps: Option, + + /// Set when the sinkpad caps are known. + clock_rate: Option, + + // NPT start from old-style RTSP caps. + npt_start: Option, + // NPT stop from old-style RTSP caps. + npt_stop: Option, + // Play scale from old-style RTSP caps. + play_speed: f64, + // Play speed from old-style RTSP caps. + play_scale: f64, + // Clock base from old-style RTSP caps. + clock_base: Option, + // Initial PTS / ext_rtptime after a caps event where NPT start was set. + npt_start_times: Option<(Option, u64)>, + + /// Set when the next outgoing buffer should have the discont flag set. + discont_pending: bool, + + /// Current stream configuration. Set on first packet and whenever it changes. + current_stream: Option, + + /// Last extended seqnum that was used for a buffer. + last_used_ext_seqnum: Option, + + /// PTS of the last outgoing buffer. + last_pts: Option, + /// DTS of the last outgoing buffer. + last_dts: Option, + + /// Pending input packets that were not completely used up for outgoing buffers yet. + pending_packets: VecDeque, + + /// Pending buffers that have to be pushed downstream. Some of them might not have a PTS yet, + /// i.e. were created without corresponding seqnums. + pending_buffers: VecDeque, +} + +impl Default for State { + fn default() -> Self { + Self { + segment: None, + pending_segment: false, + + sink_caps: None, + src_caps: None, + + clock_rate: None, + + npt_start: None, + npt_stop: None, + play_scale: 1.0, + play_speed: 1.0, + clock_base: None, + npt_start_times: None, + + discont_pending: true, + + current_stream: None, + + last_used_ext_seqnum: None, + + last_pts: None, + last_dts: None, + + pending_packets: VecDeque::new(), + pending_buffers: VecDeque::new(), + } + } +} + +#[derive(Clone, Debug)] +struct Stats { + ssrc: u32, + clock_rate: u32, + running_time_dts: Option, + running_time_pts: Option, + seqnum: u16, + timestamp: u32, + npt_start: Option, + npt_stop: Option, + play_speed: f64, + play_scale: f64, +} + +pub struct RtpBaseDepay2 { + sink_pad: gst::Pad, + src_pad: gst::Pad, + state: AtomicRefCell, + settings: Mutex, + stats: Mutex>, + extensions: Mutex>, +} + +/// Wrapper around `gst::Caps` that implements `Ord` around the structure name. +struct CapsOrd(gst::Caps); + +impl CapsOrd { + fn new(caps: gst::Caps) -> Self { + assert!(caps.is_fixed()); + CapsOrd(caps) + } +} + +impl PartialEq for CapsOrd { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == std::cmp::Ordering::Equal + } +} + +impl Eq for CapsOrd {} + +impl PartialOrd for CapsOrd { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for CapsOrd { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0 + .structure(0) + .unwrap() + .name() + .cmp(other.0.structure(0).unwrap().name()) + } +} + +/// Wrappers for public methods and associated helper functions. +#[allow(dead_code)] +impl RtpBaseDepay2 { + pub(super) fn set_src_caps(&self, src_caps: &gst::Caps) { + gst::debug!(CAT, imp: self, "Setting caps {src_caps:?}"); + + let mut state = self.state.borrow_mut(); + if Some(src_caps) == state.src_caps.as_ref() { + gst::debug!(CAT, imp: self, "Setting same caps {src_caps:?} again"); + return; + } + + let seqnum = if let Some((seqnum, _)) = state.segment { + seqnum + } else { + gst::Seqnum::next() + }; + state.src_caps = Some(src_caps.clone()); + let segment_event = self.retrieve_pending_segment_event(&mut state); + drop(state); + + // Rely on the actual flow return later when pushing a buffer downstream. The bool + // return from pushing the event does not allow to distinguish between different kinds of + // errors. + let _ = self + .src_pad + .push_event(gst::event::Caps::builder(src_caps).seqnum(seqnum).build()); + + if let Some(segment_event) = segment_event { + let _ = self.src_pad.push_event(segment_event); + } + } + + pub(super) fn drop_packets(&self, ext_seqnum: impl RangeBounds) { + gst::trace!(CAT, imp: self, "Dropping packets up to ext seqnum {:?}", ext_seqnum.end_bound()); + + let mut state = self.state.borrow_mut(); + state.discont_pending = true; + + let end = match ext_seqnum.end_bound() { + Bound::Included(end) => *end, + Bound::Excluded(end) if *end == 0 => return, + Bound::Excluded(end) => *end - 1, + Bound::Unbounded => { + state.pending_buffers.clear(); + return; + } + }; + + if let Some(back) = state.pending_packets.back() { + if back.ext_seqnum <= end { + state.pending_packets.clear(); + return; + } + } else { + return; + } + + while state + .pending_packets + .front() + .map_or(false, |p| p.ext_seqnum <= end) + { + let _ = state.pending_packets.pop_front(); + } + } + + fn retrieve_pts_dts( + &self, + state: &State, + ext_seqnum: u64, + ) -> (Option, Option) { + let mut pts = None; + let mut dts = None; + + for front in state + .pending_packets + .iter() + .take_while(|p| p.ext_seqnum <= ext_seqnum) + { + // Remember first PTS/DTS + if pts.is_none() || dts.is_none() { + pts = front.buffer.pts(); + dts = front.buffer.dts(); + } + + if pts.is_some() || dts.is_some() { + break; + } + } + + (pts, dts) + } + + fn copy_metas( + &self, + settings: &Settings, + state: &mut State, + ext_seqnum: u64, + buffer: &mut gst::BufferRef, + ) -> bool { + let mut extension_wants_caps_update = false; + let extensions = self.extensions.lock().unwrap(); + + let mut ssrc = None; + let mut csrc = [0u32; rtp_types::RtpPacket::MAX_N_CSRCS]; + let mut n_csrc = 0; + + // Copy over metas and other metadata from the packets that made up this buffer + let obj = self.obj(); + for front in state + .pending_packets + .iter() + .take_while(|p| p.ext_seqnum <= ext_seqnum) + { + // Filter out reference timestamp metas that have the same timestamps + let mut reference_timestamp_metas = BTreeMap::new(); + + front.buffer.foreach_meta(|meta| { + use std::ops::ControlFlow::*; + + // Do not copy metas that are memory specific + if meta.has_tag::() + || meta.has_tag::() + { + return Continue(()); + } + + // Actual filtering of reference timestamp metas with same timestamp + if let Some(meta) = meta.downcast_ref::() { + let mut same = false; + reference_timestamp_metas + .entry(CapsOrd::new(meta.reference_owned())) + .and_modify(|timestamp| { + same = *timestamp == meta.timestamp(); + *timestamp = meta.timestamp(); + }) + .or_insert(meta.timestamp()); + + if same { + return Continue(()); + } + } + + (obj.class().as_ref().transform_meta)(&obj, &front.buffer, &meta, buffer); + + Continue(()) + }); + + if !extensions.is_empty() && front.have_header_extensions { + let map = front.buffer.map_readable().unwrap(); + let packet = rtp_types::RtpPacket::parse(&map).unwrap(); + let (extension_pattern, mut extensions_data) = packet.extension().unwrap(); + + let extension_flags = match extension_pattern { + 0xBEDE => gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE, + x if x >> 4 == 0x100 => gst_rtp::RTPHeaderExtensionFlags::TWO_BYTE, + _ => { + gst::trace!(CAT, imp: self, "Unknown extension pattern {extension_pattern:04X}"); + continue; + } + }; + + while !extensions_data.is_empty() { + let (id, len) = if extension_flags == gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE + { + let b = extensions_data[0]; + extensions_data = &extensions_data[1..]; + + let id = b >> 4; + let len = (b & 0x0f) + 1; + + // Padding + if id == 0 { + continue; + } + if id == 15 { + // Special ID + break; + } + + (id, len as usize) + } else { + let id = extensions_data[0]; + extensions_data = &extensions_data[1..]; + + // Padding + if id == 0 { + continue; + } + + if extensions_data.is_empty() { + break; + } + + let len = extensions_data[1]; + extensions_data = &extensions_data[1..]; + + (id, len as usize) + }; + + if extensions_data.len() < len { + break; + } + + gst::trace!(CAT, imp: self, "Handling RTP header extension with id {id} and length {len}"); + + let (extension_data, remainder) = extensions_data.split_at(len); + extensions_data = remainder; + + let Some(extension) = extensions.get(&id) else { + continue; + }; + + if !extension.read(extension_flags, extension_data, buffer) { + gst::warning!(CAT, imp: self, "Failed reading RTP header extension with id {id} and length {len}"); + continue; + } + + extension_wants_caps_update |= extension.wants_update_non_rtp_src_caps(); + } + } + + if settings.source_info { + if ssrc.is_none() { + ssrc = Some(front.ssrc); + } + + for c in &front.csrc[..front.n_csrc as usize] { + if !csrc[..n_csrc].contains(c) && n_csrc < rtp_types::RtpPacket::MAX_N_CSRCS { + csrc[n_csrc] = *c; + n_csrc += 1; + } + } + } + } + + if settings.source_info && ssrc.is_some() { + gst_rtp::RTPSourceMeta::add(buffer, ssrc, &csrc[..n_csrc]); + } + + extension_wants_caps_update + } + + pub(super) fn queue_buffer( + &self, + packet_to_buffer_relation: PacketToBufferRelation, + mut buffer: gst::Buffer, + ) -> Result { + let settings = self.settings.lock().unwrap().clone(); + + { + // Reset some buffer metadata, it shouldn't have been set + let buffer_ref = buffer.make_mut(); + buffer_ref.set_pts(None); + buffer_ref.set_dts(None); + } + + gst::trace!(CAT, imp: self, "Queueing buffer {buffer:?} for seqnum range {packet_to_buffer_relation:?}"); + + let mut state = self.state.borrow_mut(); + if state.src_caps.is_none() { + gst::error!(CAT, imp: self, "No source pad caps negotiated yet"); + return Err(gst::FlowError::NotNegotiated); + } + + if matches!(packet_to_buffer_relation, PacketToBufferRelation::OutOfBand) { + gst::trace!(CAT, imp: self, "Keeping buffer without associated seqnums until next buffer or EOS"); + + state.pending_buffers.push_back(PendingBuffer { + metadata_set: false, + buffer, + }); + return Ok(gst::FlowSuccess::Ok); + }; + + let (seqnums, timestamp_offset) = match packet_to_buffer_relation { + PacketToBufferRelation::Seqnums(seqnums) => (seqnums.clone(), None), + PacketToBufferRelation::SeqnumsWithOffset { + seqnums: ids, + timestamp_offset, + } => (ids.clone(), Some(timestamp_offset)), + PacketToBufferRelation::OutOfBand => unreachable!(), + }; + + if seqnums.is_empty() { + gst::error!(CAT, imp: self, "Empty packet ext seqnum range provided"); + return Err(gst::FlowError::Error); + } + + let seqnum_start = *seqnums.start(); + let seqnum_end = *seqnums.end(); + + // Drop all older seqnums + while state + .pending_packets + .front() + .map_or(false, |p| p.ext_seqnum < seqnum_start) + { + let p = state.pending_packets.pop_front().unwrap(); + gst::trace!(CAT, imp: self, "Dropping packet with extended seqnum {}", p.ext_seqnum); + } + + if state.pending_packets.is_empty() { + gst::error!(CAT, imp: self, "Queueing buffers for future ext seqnums not allowed"); + return Err(gst::FlowError::Error); + }; + + if seqnum_end > state.pending_packets.back().unwrap().ext_seqnum { + gst::error!(CAT, imp: self, "Queueing buffers for future ext seqnums not allowed"); + return Err(gst::FlowError::Error); + } + + let mut discont_pending = state.discont_pending; + state.discont_pending = false; + + let mut pts = state + .pending_packets + .iter() + .take_while(|p| p.ext_seqnum <= seqnum_end) + .find_map(|p| p.buffer.pts()); + let mut dts = None; + + if let Some(timestamp_offset) = timestamp_offset { + match timestamp_offset { + TimestampOffset::Pts(pts_offset) => { + if let Some(ts) = pts { + let pts_signed = ts.into_positive() + pts_offset; + if let Some(ts) = pts_signed.positive() { + pts = Some(ts); + } else { + gst::warning!(CAT, imp: self, "Negative PTS {} calculated, not supported", pts_signed); + } + } + } + TimestampOffset::PtsAndDts(pts_offset, dts_offset) => { + if let Some(ts) = pts { + let pts_signed = ts.into_positive() + pts_offset; + if let Some(ts) = pts_signed.positive() { + pts = Some(ts); + } else { + gst::warning!(CAT, imp: self, "Negative PTS {} calculated, not supported", pts_signed); + } + } + + if let Some(pts) = pts { + let dts_signed = pts.into_positive() + dts_offset; + if let Some(ts) = dts_signed.positive() { + dts = Some(ts); + } else { + gst::warning!(CAT, imp: self, "Negative DTS {} calculated, not supported", dts_signed); + } + } + } + } + } else if state + .last_used_ext_seqnum + .map_or(false, |last_used_ext_seqnum| { + seqnum_end <= last_used_ext_seqnum + }) + { + // If we have no timestamp offset and this is not the first time a buffer for this + // packet is queued then unset the PTS. It's not going to be correct. + pts = None; + } + + // Decorate all previously queued buffers metadata + for buffer in state + .pending_buffers + .iter_mut() + .skip_while(|b| b.metadata_set) + { + assert!(!buffer.metadata_set); + buffer.metadata_set = true; + let buffer = &mut buffer.buffer; + let buffer_ref = buffer.make_mut(); + buffer_ref.set_pts(pts); + buffer_ref.set_dts(dts); + if discont_pending { + buffer_ref.set_flags(gst::BufferFlags::DISCONT); + discont_pending = false; + } + } + + // Decorate the new buffer + { + let buffer_ref = buffer.make_mut(); + buffer_ref.set_pts(pts); + buffer_ref.set_dts(dts); + let needs_caps_update = self.copy_metas(&settings, &mut state, seqnum_end, buffer_ref); + + if discont_pending { + buffer_ref.set_flags(gst::BufferFlags::DISCONT); + } + + // If a caps update is needed then let's do that *before* outputting this buffer. + if needs_caps_update { + let old_src_caps = state.src_caps.as_ref().unwrap(); + let mut src_caps = old_src_caps.copy(); + + { + let src_caps = src_caps.get_mut().unwrap(); + let extensions = self.extensions.lock().unwrap(); + for extension in extensions.values() { + if !extension.update_non_rtp_src_caps(src_caps) { + gst::error!(CAT, imp: self, "RTP header extension {} could not update caps", extension.name()); + return Err(gst::FlowError::Error); + } + } + } + + if &src_caps != old_src_caps { + let seqnum = state.segment.as_ref().unwrap().0; + drop(state); + self.finish_pending_buffers()?; + + let _ = self + .src_pad + .push_event(gst::event::Caps::builder(&src_caps).seqnum(seqnum).build()); + + state = self.state.borrow_mut(); + state.src_caps = Some(src_caps); + } + } + } + + state.pending_buffers.push_back(PendingBuffer { + metadata_set: true, + buffer, + }); + + state.last_used_ext_seqnum = Some(seqnum_end); + if pts.is_some() { + state.last_pts = pts; + state.last_dts = dts; + } + + Ok(gst::FlowSuccess::Ok) + } + + pub(super) fn finish_pending_buffers(&self) -> Result { + let mut state = self.state.borrow_mut(); + + // As long as there are buffers that can be finished, take all with the same PTS (or no + // PTS) and put them into a buffer list. + while state.pending_buffers.iter().any(|b| b.metadata_set) { + let pts = state.pending_buffers.front().unwrap().buffer.pts(); + + // First count number of pending packets so we can allocate a buffer list with the correct + // size instead of re-allocating for every added buffer, or simply push a single buffer + // downstream instead of allocating a buffer list just for one buffer. + let num_buffers = state + .pending_buffers + .iter() + .take_while(|b| { + b.metadata_set && (b.buffer.pts() == pts || b.buffer.pts().is_none()) + }) + .count(); + + assert_ne!(num_buffers, 0); + + gst::trace!(CAT, imp: self, "Flushing {num_buffers} buffers"); + + let segment_event = self.retrieve_pending_segment_event(&mut state); + + enum BufferOrList { + Buffer(gst::Buffer), + List(gst::BufferList), + } + + let buffers = if num_buffers == 1 { + let buffer = state.pending_buffers.pop_front().unwrap().buffer; + gst::trace!(CAT, imp: self, "Finishing buffer {buffer:?}"); + BufferOrList::Buffer(buffer) + } else { + let mut list = gst::BufferList::new_sized(num_buffers); + { + let list = list.get_mut().unwrap(); + while state.pending_buffers.front().map_or(false, |b| { + b.metadata_set && (b.buffer.pts() == pts || b.buffer.pts().is_none()) + }) { + let buffer = state.pending_buffers.pop_front().unwrap().buffer; + gst::trace!(CAT, imp: self, "Finishing buffer {buffer:?}"); + list.add(buffer); + } + } + + BufferOrList::List(list) + }; + + drop(state); + + if let Some(segment_event) = segment_event { + let _ = self.src_pad.push_event(segment_event); + } + + let res = match buffers { + BufferOrList::Buffer(buffer) => self.src_pad.push(buffer), + BufferOrList::List(list) => self.src_pad.push_list(list), + }; + + if let Err(err) = res { + if ![gst::FlowError::Flushing, gst::FlowError::Eos].contains(&err) { + gst::warning!(CAT, imp: self, "Failed pushing buffers: {err:?}"); + } else { + gst::debug!(CAT, imp: self, "Failed pushing buffers: {err:?}"); + } + } + + state = self.state.borrow_mut(); + } + + Ok(gst::FlowSuccess::Ok) + } + + pub(super) fn sink_pad(&self) -> &gst::Pad { + &self.sink_pad + } + + pub(super) fn src_pad(&self) -> &gst::Pad { + &self.src_pad + } +} + +/// Default virtual method implementations. +impl RtpBaseDepay2 { + fn start_default(&self) -> Result<(), gst::ErrorMessage> { + Ok(()) + } + + fn stop_default(&self) -> Result<(), gst::ErrorMessage> { + Ok(()) + } + + fn set_sink_caps_default(&self, _caps: &gst::Caps) -> bool { + true + } + + fn handle_packet_default( + &self, + _packet: &super::Packet, + ) -> Result { + unimplemented!() + } + + fn drain_default(&self) -> Result { + Ok(gst::FlowSuccess::Ok) + } + + fn flush_default(&self) {} + + fn sink_event_default(&self, event: gst::Event) -> Result { + match event.view() { + gst::EventView::Caps(event) => { + let caps = event.caps_owned(); + + return self.set_sink_caps(caps); + } + gst::EventView::Segment(event) => { + let segment = event.segment(); + let seqnum = event.seqnum(); + + let state = self.state.borrow(); + if state.sink_caps.is_none() { + gst::warning!(CAT, imp: self, "Received segment before caps"); + return Err(gst::FlowError::NotNegotiated); + } + + let same_segment = state + .segment + .as_ref() + .map(|(old_seqnum, old_segment)| { + (&seqnum, segment) == (old_seqnum, old_segment.upcast_ref()) + }) + .unwrap_or(false); + // Only drain if the segment changed and we actually had one before + let drain = !same_segment && state.segment.is_some(); + drop(state); + + if !same_segment { + gst::debug!(CAT, imp: self, "Received segment {segment:?}"); + + if drain { + if let Err(err) = self.drain() { + // Just continue here. The depayloader is now flushed and can proceed below, + // and if there was a serious error downstream it will happen on the next + // buffer pushed downstream + gst::debug!(CAT, imp: self, "Draining failed: {err:?}"); + } + } + + let mut state = self.state.borrow_mut(); + if let Some(segment) = segment.downcast_ref::() { + state.segment = Some((event.seqnum(), segment.clone())); + state.pending_segment = true; + } else { + gst::error!(CAT, imp: self, "Segments in non-TIME format are not supported"); + state.segment = None; + // FIXME: Forget everything here? + return Err(gst::FlowError::Error); + } + + return Ok(gst::FlowSuccess::Ok); + } + } + gst::EventView::Eos(_) => { + if let Err(err) = self.drain() { + gst::debug!(CAT, imp: self, "Draining on EOS failed: {err:?}"); + } + } + gst::EventView::FlushStop(_) => { + self.flush(); + + let mut state = self.state.borrow_mut(); + state.segment = None; + state.pending_segment = false; + state.current_stream = None; + state.last_pts = None; + state.last_dts = None; + drop(state); + + *self.stats.lock().unwrap() = None; + } + _ => (), + } + + gst::debug!(CAT, imp: self, "Forwarding event: {event:?}"); + if self.src_pad.push_event(event) { + Ok(gst::FlowSuccess::Ok) + } else if self.src_pad.pad_flags().contains(gst::PadFlags::FLUSHING) { + Err(gst::FlowError::Flushing) + } else { + Err(gst::FlowError::Error) + } + } + + fn src_event_default(&self, event: gst::Event) -> Result { + if gst::Pad::event_default(&self.src_pad, Some(&*self.obj()), event) { + Ok(gst::FlowSuccess::Ok) + } else if self.sink_pad.pad_flags().contains(gst::PadFlags::FLUSHING) { + Err(gst::FlowError::Flushing) + } else { + Err(gst::FlowError::Error) + } + } + + fn sink_query_default(&self, query: &mut gst::QueryRef) -> bool { + gst::Pad::query_default(&self.sink_pad, Some(&*self.obj()), query) + } + + fn src_query_default(&self, query: &mut gst::QueryRef) -> bool { + gst::Pad::query_default(&self.src_pad, Some(&*self.obj()), query) + } + + fn transform_meta_default( + &self, + _in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ) { + let tags = meta.tags(); + + if tags.len() > 1 { + gst::trace!( + CAT, + imp: self, + "Not copying meta {}: has multiple tags {tags:?}", + meta.api(), + ); + return; + } + + let allowed_tags = self.obj().class().as_ref().allowed_meta_tags; + if tags.len() == 1 { + let meta_tag = &tags[0]; + + if !allowed_tags.iter().copied().any(|tag| tag == *meta_tag) { + gst::trace!( + CAT, + imp: self, + "Not copying meta {}: tag '{meta_tag}' not allowed", + meta.api(), + ); + return; + } + } + + gst::trace!(CAT, imp: self, "Copying meta {}", meta.api()); + + if let Err(err) = meta.transform(out_buf, &gst::meta::MetaTransformCopy::new(false, ..)) { + gst::trace!(CAT, imp: self, "Could not copy meta {}: {err}", meta.api()); + } + } + + fn add_extension(&self, ext: &gst_rtp::RTPHeaderExtension) { + assert_ne!(ext.id(), 0); + + let mut extensions = self.extensions.lock().unwrap(); + extensions.insert(ext.id() as u8, ext.clone()); + // FIXME: Needs caps update! + drop(extensions); + + self.obj().notify("extensions"); + } + + fn clear_extensions(&self) { + let mut extensions = self.extensions.lock().unwrap(); + extensions.clear(); + // FIXME: Needs caps update! + drop(extensions); + + self.obj().notify("extensions"); + } + + fn request_extension(&self, ext_id: u32, uri: &str) -> Option { + let settings = self.settings.lock().unwrap(); + if !settings.auto_header_extensions { + return None; + } + drop(settings); + + let Some(ext) = gst_rtp::RTPHeaderExtension::create_from_uri(uri) else { + gst::debug!( + CAT, + imp: self, + "Didn't find any extension implementing URI {uri}", + ); + return None; + }; + + gst::debug!( + CAT, + imp: self, + "Automatically enabling extension {} for URI {uri}", + ext.name(), + ); + + ext.set_id(ext_id); + + Some(ext) + } +} + +/// Pad functions and associated helper functions. +impl RtpBaseDepay2 { + fn sink_chain( + &self, + _pad: &gst::Pad, + buffer: gst::Buffer, + ) -> Result { + gst::trace!(CAT, imp: self, "Received buffer {buffer:?}"); + + let settings = self.settings.lock().unwrap().clone(); + self.handle_buffer(&settings, buffer) + } + + fn sink_chain_list( + &self, + _pad: &gst::Pad, + list: gst::BufferList, + ) -> Result { + gst::trace!(CAT, imp: self, "Received buffer list {list:?}"); + + let settings = self.settings.lock().unwrap().clone(); + for buffer in list.iter_owned() { + self.handle_buffer(&settings, buffer)?; + } + + Ok(gst::FlowSuccess::Ok) + } + + fn sink_event( + &self, + pad: &gst::Pad, + event: gst::Event, + ) -> Result { + gst::debug!(CAT, obj: pad, "Received event: {event:?}"); + + let obj = self.obj(); + (obj.class().as_ref().sink_event)(&obj, event) + } + + fn src_event( + &self, + pad: &gst::Pad, + event: gst::Event, + ) -> Result { + gst::debug!(CAT, obj: pad, "Received event: {event:?}"); + + let obj = self.obj(); + (obj.class().as_ref().src_event)(&obj, event) + } + + fn sink_query(&self, pad: &gst::Pad, query: &mut gst::QueryRef) -> bool { + gst::trace!(CAT, obj: pad, "Received query: {query:?}"); + + let obj = self.obj(); + (obj.class().as_ref().sink_query)(&obj, query) + } + + fn src_query(&self, pad: &gst::Pad, query: &mut gst::QueryRef) -> bool { + gst::trace!(CAT, obj: pad, "Received query: {query:?}"); + + let obj = self.obj(); + (obj.class().as_ref().src_query)(&obj, query) + } + + fn retrieve_pending_segment_event(&self, state: &mut State) -> Option { + // If a segment event was pending, take it now and push it out later. + if !state.pending_segment { + return None; + } + + state.src_caps.as_ref()?; + + let (seqnum, segment) = state.segment.as_ref().unwrap(); + let mut segment = segment.clone(); + + // If npt-start is set, all the other values are also valid. + if let Some(npt_start) = state.npt_start { + let Some((mut npt_start_pts, npt_start_ext_rtptime)) = state.npt_start_times else { + return None; + }; + + let clock_rate = state.clock_rate.unwrap(); + + let mut start = segment.start().unwrap(); + + if let Some((clock_base, pts)) = Option::zip(state.clock_base, npt_start_pts) { + let ext_clock_base = clock_base as u64 + (1 << 32); + + let gap = npt_start_ext_rtptime + .saturating_sub(ext_clock_base) + .mul_div_floor(*gst::ClockTime::SECOND, clock_rate as u64) + .map(gst::ClockTime::from_nseconds); + + // Account for lost packets + if let Some(gap) = gap { + if pts > gap { + gst::debug!( + CAT, + imp: self, + "Found gap of {}, adjusting start: {} = {} - {}", + gap.display(), + (pts - gap).display(), + pts.display(), + gap.display(), + ); + start = pts - gap; + } + } + } + + let mut stop = segment.stop(); + if let Some(npt_stop) = state.npt_stop { + stop = Some(start + npt_stop.saturating_sub(npt_start)); + } + + if npt_start_pts.is_none() { + npt_start_pts = Some(start); + } + + let running_time = segment.to_running_time(start); + + segment.reset(); + segment.set_rate(state.play_speed); + segment.set_applied_rate(state.play_scale); + segment.set_start(start); + segment.set_stop(stop); + segment.set_time(npt_start); + segment.set_position(npt_start_pts); + segment.set_base(running_time); + } + + let segment_event = gst::event::Segment::builder(&segment) + .seqnum(*seqnum) + .build(); + state.pending_segment = false; + gst::debug!( + CAT, + imp: self, + "Created segment event {segment:?} with seqnum {seqnum:?}", + ); + + Some(segment_event) + } + + fn handle_buffer( + &self, + settings: &Settings, + buffer: gst::Buffer, + ) -> Result { + let mut state = self.state.borrow_mut(); + + if state.sink_caps.is_none() { + gst::error!(CAT, imp: self, "No sink pad caps"); + gst::element_imp_error!( + self, + gst::CoreError::Negotiation, + ("No RTP caps were negotiated"), + [ + "Input buffers need to have RTP caps set on them. This is usually \ + achieved by setting the 'caps' property of the upstream source \ + element (often udpsrc or appsrc), or by putting a capsfilter \ + element before the depayloader and setting the 'caps' property \ + on that. Also see https://gitlab.freedesktop.org/gstreamer/gstreamer/-/ \ + blob/main/subprojects/gst-plugins-good/gst/rtp/README", + ] + ); + return Err(gst::FlowError::NotNegotiated); + } + // Always set if the caps are set. + let clock_rate = state.clock_rate.unwrap(); + + // Check for too many pending packets + if let Some((front, back)) = + Option::zip(state.pending_packets.front(), state.pending_packets.back()) + { + let rtp_diff = back + .ext_timestamp + .saturating_sub(front.ext_timestamp) + .mul_div_floor(*gst::ClockTime::SECOND, clock_rate as u64) + .map(gst::ClockTime::from_nseconds); + let pts_diff = back.buffer.pts().opt_saturating_sub(front.buffer.pts()); + + if let Some(rtp_diff) = rtp_diff { + if rtp_diff > gst::ClockTime::SECOND { + gst::warning!(CAT, imp: self, "More than {rtp_diff} of RTP time queued, probably a bug in the subclass"); + } + } + + if let Some(pts_diff) = pts_diff { + if pts_diff > gst::ClockTime::SECOND { + gst::warning!(CAT, imp: self, "More than {pts_diff} of PTS time queued, probably a bug in the subclass"); + } + } + } + + let buffer = match buffer.into_mapped_buffer_readable() { + Ok(buffer) => buffer, + Err(_) => { + gst::error!(CAT, imp: self, "Failed to map buffer"); + gst::element_imp_error!(self, gst::StreamError::Failed, ["Failed to map buffer"]); + return Err(gst::FlowError::Error); + } + }; + + // Parse RTP packet and extract the relevant fields we will need later. + let packet = match rtp_types::RtpPacket::parse(&buffer) { + Ok(packet) => packet, + Err(err) => { + gst::warning!(CAT, imp: self, "Failed to parse RTP packet: {err:?}"); + return Ok(gst::FlowSuccess::Ok); + } + }; + + let seqnum = packet.sequence_number(); + let ssrc = packet.ssrc(); + let pt = packet.payload_type(); + let rtptime = packet.timestamp(); + let marker = packet.marker_bit(); + let have_header_extensions = packet.extension_len() != 0; + let csrc = { + let mut csrc = packet.csrc(); + std::array::from_fn::(|_idx| { + csrc.next().unwrap_or_default() + }) + }; + let n_csrc = packet.n_csrcs(); + + let payload_range = { + let payload = packet.payload(); + if payload.is_empty() { + 0..0 + } else { + let offset = packet.payload_offset(); + offset..(offset + payload.len()) + } + }; + + // FIXME: Allow subclasses to decide not to flush on DISCONTs and handle the situation + // themselves, e.g. to be able to continue reconstructing frames despite missing data. + if buffer.buffer().flags().contains(gst::BufferFlags::DISCONT) { + gst::info!(CAT, imp: self, "Discont received"); + state.current_stream = None; + } + + // Check if there was a stream discontinuity, e.g. based on the SSRC changing or the seqnum + // having a gap compared to the previous packet. + if let Some(ref mut current_stream) = state.current_stream { + if current_stream.ssrc != ssrc { + gst::info!(CAT, imp: self, "Stream SSRC changed from {:08x} to {:08x}", current_stream.ssrc, ssrc); + state.current_stream = None; + } else if current_stream.pt != pt { + gst::info!(CAT, imp: self, "Stream payload type changed from {} to {}", current_stream.pt, pt); + state.current_stream = None; + } else { + let expected_seqnum = (current_stream.ext_seqnum + 1) & 0xffff; + if expected_seqnum != seqnum as u64 { + gst::info!(CAT, imp: self, "Got seqnum {seqnum} but expected {expected_seqnum}"); + + let mut ext_seqnum = seqnum as u64 + (current_stream.ext_seqnum & !0xffff); + + if ext_seqnum < current_stream.ext_seqnum { + let diff = current_stream.ext_seqnum - ext_seqnum; + if diff > 0x7fff { + ext_seqnum += 1 << 16; + } + } else { + let diff = ext_seqnum - current_stream.ext_seqnum; + if diff > 0x7fff { + ext_seqnum -= 1 << 16; + } + } + let diff = if ext_seqnum > current_stream.ext_seqnum { + (ext_seqnum - current_stream.ext_seqnum) as i32 + } else { + -((current_stream.ext_seqnum - ext_seqnum) as i32) + }; + + if diff > 0 { + gst::info!(CAT, imp: self, "{diff} missing packets or sender restart"); + state.current_stream = None; + } else if diff >= -(settings.max_reorder as i32) { + gst::info!(CAT, imp: self, "Got old packet, dropping"); + return Ok(gst::FlowSuccess::Ok); + } else { + gst::info!(CAT, imp: self, "Sender restart"); + state.current_stream = None; + } + } else { + // Calculate extended RTP time + let ext_rtptime = { + let mut ext_rtptime = + rtptime as u64 + (current_stream.ext_rtptime & !0xffff_ffff); + + // Check for wraparound + if ext_rtptime < current_stream.ext_rtptime { + let diff = current_stream.ext_rtptime - ext_rtptime; + if diff > 0x7fff_ffff { + ext_rtptime += 1 << 32; + } + } else { + let diff = ext_rtptime - current_stream.ext_rtptime; + if diff > 0x7fff_ffff { + ext_rtptime -= 1 << 32; + } + } + + ext_rtptime + }; + + current_stream.ext_seqnum += 1; + current_stream.ext_rtptime = ext_rtptime; + gst::trace!( + CAT, + imp: self, + "Handling packet with extended seqnum {} and extended RTP time {}", + current_stream.ext_seqnum, + current_stream.ext_rtptime, + ); + } + } + } + + // Drain if there was any discontinuity. + let discont = state.current_stream.is_none(); + if discont && !state.pending_packets.is_empty() { + drop(state); + gst::info!(CAT, imp: self, "Got discontinuity, draining"); + + if let Err(err) = self.drain() { + if ![gst::FlowError::Flushing, gst::FlowError::Eos].contains(&err) { + gst::warning!(CAT, imp: self, "Error while draining: {err:?}"); + } else { + gst::debug!(CAT, imp: self, "Error while draining: {err:?}"); + } + return Err(err); + } + state = self.state.borrow_mut(); + } + + // Update current stream state at this point. + let ext_seqnum; + let ext_rtptime; + if let Some(ref current_stream) = state.current_stream { + ext_seqnum = current_stream.ext_seqnum; + ext_rtptime = current_stream.ext_rtptime; + } else { + ext_seqnum = seqnum as u64 + (1 << 16); + ext_rtptime = rtptime as u64 + (1 << 32); + + gst::info!( + CAT, + imp: self, + "Starting stream with SSRC {ssrc:08x} at extended seqnum {ext_seqnum} with extended RTP time {ext_rtptime}", + ); + + state.current_stream = Some(CurrentStream { + ssrc, + pt, + ext_seqnum, + ext_rtptime, + }) + } + + // Remember the initial PTS/rtp_time mapping if old-style RTSP caps are used. + if state.npt_start.is_some() && state.npt_start_times.is_none() { + state.npt_start_times = Some((buffer.buffer().pts(), ext_rtptime)); + } + + // If a segment event was pending, take it now and push it out later. + let segment_event = self.retrieve_pending_segment_event(&mut state); + + // Remember current packet. + state.pending_packets.push_back(PendingPacket { + ext_seqnum, + ext_timestamp: ext_rtptime, + ssrc, + csrc, + n_csrc, + have_header_extensions, + buffer: buffer.buffer_owned(), + }); + + // Remember if the next buffer that has to be pushed out should have the DISCONT flag set. + state.discont_pending |= discont; + + // Update stats with the current packet. + *self.stats.lock().unwrap() = { + let (_seqnum, segment) = state.segment.as_ref().unwrap(); + + let running_time_pts = segment.to_running_time(buffer.buffer().pts()); + let running_time_dts = segment.to_running_time(buffer.buffer().dts()); + Some(Stats { + ssrc, + clock_rate, + running_time_dts, + running_time_pts, + seqnum, + timestamp: rtptime, + npt_start: state.npt_start, + npt_stop: state.npt_stop, + play_speed: state.play_speed, + play_scale: state.play_scale, + }) + }; + drop(state); + + if let Some(segment_event) = segment_event { + let _ = self.src_pad.push_event(segment_event); + } + + // Now finally handle the packet. + let obj = self.obj(); + let mut res = (obj.class().as_ref().handle_packet)( + &obj, + &super::Packet { + buffer, + discont, + ext_seqnum, + ext_timestamp: ext_rtptime, + marker, + payload_range, + }, + ); + if let Err(err) = res { + gst::error!(CAT, imp: self, "Failed handling packet: {err:?}"); + } else { + res = self.finish_pending_buffers(); + + if let Err(err) = res { + gst::debug!(CAT, imp: self, "Failed finishing pending buffers: {err:?}"); + } + } + + // Now drop all pending packets that were used for producing buffers + // except for the very last one as that might still be used. + let mut state = self.state.borrow_mut(); + if let Some(last_used_ext_seqnum) = state.last_used_ext_seqnum { + while state + .pending_packets + .front() + .map_or(false, |b| b.ext_seqnum < last_used_ext_seqnum) + { + let _ = state.pending_packets.pop_front(); + } + } + + res + } +} + +/// Other methods. +impl RtpBaseDepay2 { + fn update_extensions_from_sink_caps( + &self, + sink_caps: &gst::Caps, + ) -> Result { + let s = sink_caps.structure(0).unwrap(); + + // Check which header extensions to enable/disable based on the caps. + let mut caps_extensions = BTreeMap::new(); + for (k, v) in s.iter() { + let Some(ext_id) = k.strip_prefix("extmap-") else { + continue; + }; + let Ok(ext_id) = ext_id.parse::() else { + gst::error!(CAT, imp: self, "Can't parse RTP header extension id from caps {sink_caps:?}"); + return Err(gst::FlowError::NotNegotiated); + }; + + let uri = if let Ok(uri) = v.get::() { + uri + } else if let Ok(arr) = v.get::() { + if let Some(uri) = arr.get(1).and_then(|v| v.get::().ok()) { + uri + } else { + gst::error!(CAT, imp: self, "Couldn't get URI for RTP header extension id {ext_id} from caps {sink_caps:?}"); + return Err(gst::FlowError::NotNegotiated); + } + } else { + gst::error!(CAT, imp: self, "Couldn't get URI for RTP header extension id {ext_id} from caps {sink_caps:?}"); + return Err(gst::FlowError::NotNegotiated); + }; + + caps_extensions.insert(ext_id, uri); + } + + let mut extensions = self.extensions.lock().unwrap(); + let mut extensions_changed = false; + for (ext_id, uri) in caps_extensions { + if let Some(extension) = extensions.get(&ext_id) { + if extension.uri().as_deref() == Some(&uri) { + // Same extension, nothing to do here, we will update it with the new caps + // later + continue; + } + gst::debug!( + CAT, + imp: self, + "Extension ID {ext_id} changed from {:?} to {uri}", + extension.uri(), + ); + extensions_changed |= true; + extensions.remove(&ext_id); + } + + gst::debug!(CAT, imp: self, "Requesting extension {uri} for ID {ext_id}"); + let ext = self + .obj() + .emit_by_name::>( + "request-extension", + &[&(ext_id as u32), &uri], + ); + + let Some(ext) = ext else { + gst::warning!(CAT, imp: self, "Couldn't create extension for {uri} with ID {ext_id}"); + continue; + }; + + if ext.id() != ext_id as u32 { + gst::warning!(CAT, imp: self, "Created extension has wrong ID"); + continue; + } + + extensions.insert(ext_id, ext); + extensions_changed |= true; + } + + let mut to_remove = vec![]; + for (ext_id, ext) in extensions.iter() { + if !ext.set_attributes_from_caps(sink_caps) { + gst::warning!(CAT, imp: self, "Failed to configure extension {ext_id} from caps {sink_caps}"); + to_remove.push(*ext_id); + } + } + for ext_id in to_remove { + extensions.remove(&ext_id); + extensions_changed |= true; + } + drop(extensions); + + if extensions_changed { + self.obj().notify("extensions"); + } + + Ok(gst::FlowSuccess::Ok) + } + + fn set_sink_caps(&self, sink_caps: gst::Caps) -> Result { + gst::debug!(CAT, imp: self, "Received caps {sink_caps:?}"); + + let s = sink_caps.structure(0).unwrap(); + + if !s.has_name("application/x-rtp") { + gst::error!(CAT, imp: self, "Non-RTP caps {sink_caps:?} not supported"); + return Err(gst::FlowError::NotNegotiated); + } + + let clock_rate = match s.get::("clock-rate") { + Ok(clock_rate) if clock_rate > 0 => clock_rate, + _ => { + gst::error!(CAT, imp: self, "RTP caps {sink_caps:?} without 'clock-rate'"); + return Err(gst::FlowError::NotNegotiated); + } + }; + + let npt_start = s.get::("npt-start").ok(); + let npt_stop = s.get::("npt-stop").ok(); + let play_scale = s.get::("play-scale").unwrap_or(1.0); + let play_speed = s.get::("play-speed").unwrap_or(1.0); + let clock_base = s.get::("clock-base").ok(); + let onvif_mode = s.get::("onvif-mode").unwrap_or_default(); + + self.update_extensions_from_sink_caps(&sink_caps)?; + + let state = self.state.borrow_mut(); + let same_caps = state + .sink_caps + .as_ref() + .map(|old_caps| &sink_caps == old_caps) + .unwrap_or(false); + drop(state); + + if !same_caps { + let obj = self.obj(); + + if let Err(err) = self.drain() { + // Just continue here. The depayloader is now flushed and can proceed below, + // and if there was a serious error downstream it will happen on the next + // buffer pushed downstream + gst::debug!(CAT, imp: self, "Draining failed: {err:?}"); + } + + let set_sink_caps_res = if (obj.class().as_ref().set_sink_caps)(&obj, &sink_caps) { + gst::debug!(CAT, imp: self, "Caps {sink_caps:?} accepted"); + Ok(gst::FlowSuccess::Ok) + } else { + gst::warning!(CAT, imp: self, "Caps {sink_caps:?} not accepted"); + Err(gst::FlowError::NotNegotiated) + }; + + let mut state = self.state.borrow_mut(); + if let Err(err) = set_sink_caps_res { + state.sink_caps = None; + // FIXME: Forget everything here? + return Err(err); + } + + state.sink_caps = Some(sink_caps); + state.clock_rate = Some(clock_rate as u32); + + // If npt-start is set, all the other values are also valid. + // + // In ONVIF mode these are all ignored and instead upstream is supposed to provide the + // correct segment already. + if npt_start.is_some() && !onvif_mode { + state.npt_start = npt_start; + state.npt_stop = npt_stop; + state.play_speed = play_speed; + state.play_scale = play_scale; + state.clock_base = clock_base; + state.npt_start_times = None; + state.pending_segment = true; + } else { + if state.npt_start.is_some() { + state.pending_segment = true; + } + state.npt_start = None; + state.npt_stop = None; + state.play_speed = 1.0; + state.play_scale = 1.0; + state.clock_base = None; + state.npt_start_times = None; + } + + drop(state); + } + + Ok(gst::FlowSuccess::Ok) + } + + fn drain(&self) -> Result { + let obj = self.obj(); + if let Err(err) = (obj.class().as_ref().drain)(&obj) { + if ![gst::FlowError::Flushing, gst::FlowError::Eos].contains(&err) { + gst::warning!(CAT, imp: self, "Draining failed: {err:?}"); + } else { + gst::debug!(CAT, imp: self, "Draining failed: {err:?}"); + } + self.flush(); + return Err(err); + } + + let mut state = self.state.borrow_mut(); + state.pending_packets.clear(); + + // Update all pending buffers without metadata with the last buffer's PTS/DTS + let last_pts = state.last_pts; + let last_dts = state.last_dts; + let mut discont_pending = state.discont_pending; + state.discont_pending = false; + + for buffer in state + .pending_buffers + .iter_mut() + .skip_while(|b| b.metadata_set) + { + assert!(!buffer.metadata_set); + buffer.metadata_set = true; + let buffer = &mut buffer.buffer; + let buffer_ref = buffer.make_mut(); + buffer_ref.set_pts(last_pts); + buffer_ref.set_dts(last_dts); + if discont_pending { + buffer_ref.set_flags(gst::BufferFlags::DISCONT); + discont_pending = false; + } + } + drop(state); + + // Forward all buffers + let res = self.finish_pending_buffers(); + + self.flush(); + + res + } + + fn flush(&self) { + let obj = self.obj(); + (obj.class().as_ref().flush)(&obj); + + let mut state = self.state.borrow_mut(); + state.pending_packets.clear(); + state.pending_buffers.clear(); + state.discont_pending = true; + } + + fn create_stats(&self) -> gst::Structure { + let stats = self.stats.lock().unwrap().clone(); + if let Some(stats) = stats { + gst::Structure::builder("application/x-rtp-depayload-stats") + .field("ssrc", stats.ssrc) + .field("clock-rate", stats.clock_rate) + .field("running-time-dts", stats.running_time_dts) + .field("running-time-pts", stats.running_time_pts) + .field("seqnum", stats.seqnum as u32) + .field("timestamp", stats.timestamp) + .field("npt-start", stats.npt_start) + .field("npt-stop", stats.npt_stop) + .field("play-speed", stats.play_speed) + .field("play-scale", stats.play_scale) + .build() + } else { + gst::Structure::builder("application/x-rtp-depayload-stats").build() + } + } +} + +#[glib::object_subclass] +impl ObjectSubclass for RtpBaseDepay2 { + const NAME: &'static str = "GstRtpBaseDepay2"; + const ABSTRACT: bool = true; + type Type = super::RtpBaseDepay2; + type ParentType = gst::Element; + type Class = super::Class; + + fn class_init(class: &mut Self::Class) { + class.start = |obj| obj.imp().start_default(); + class.stop = |obj| obj.imp().stop_default(); + class.set_sink_caps = |obj, caps| obj.imp().set_sink_caps_default(caps); + class.handle_packet = |obj, packet| obj.imp().handle_packet_default(packet); + class.drain = |obj| obj.imp().drain_default(); + class.flush = |obj| obj.imp().flush_default(); + class.sink_event = |obj, event| obj.imp().sink_event_default(event); + class.src_event = |obj, event| obj.imp().src_event_default(event); + class.sink_query = |obj, query| obj.imp().sink_query_default(query); + class.src_query = |obj, query| obj.imp().src_query_default(query); + class.transform_meta = + |obj, in_buf, meta, out_buf| obj.imp().transform_meta_default(in_buf, meta, out_buf); + class.allowed_meta_tags = &[]; + } + + fn with_class(class: &Self::Class) -> Self { + let templ = class + .pad_template("sink") + .expect("Subclass did not provide a \"sink\" pad template"); + let sink_pad = gst::Pad::builder_from_template(&templ) + .chain_function(|pad, parent, buffer| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.sink_chain(pad, buffer), + ) + }) + .chain_list_function(|pad, parent, list| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.sink_chain_list(pad, list), + ) + }) + .event_full_function(|pad, parent, event| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.sink_event(pad, event), + ) + }) + .query_function(|pad, parent, query| { + Self::catch_panic_pad_function(parent, || false, |imp| imp.sink_query(pad, query)) + }) + .build(); + + let templ = class + .pad_template("src") + .expect("Subclass did not provide a \"src\" pad template"); + let src_pad = gst::Pad::builder_from_template(&templ) + .event_full_function(|pad, parent, event| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.src_event(pad, event), + ) + }) + .query_function(|pad, parent, query| { + Self::catch_panic_pad_function(parent, || false, |imp| imp.src_query(pad, query)) + }) + .flags(gst::PadFlags::FIXED_CAPS) + .build(); + + Self { + src_pad, + sink_pad, + state: AtomicRefCell::default(), + settings: Mutex::default(), + stats: Mutex::default(), + extensions: Mutex::default(), + } + } +} + +impl ObjectImpl for RtpBaseDepay2 { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![ + glib::ParamSpecBoxed::builder::("stats") + .nick("Statistics") + .blurb("Various statistics") + .read_only() + .build(), + glib::ParamSpecUInt::builder("max-reorder") + .nick("Maximum Reorder") + .blurb("Maximum seqnum reorder before assuming sender has restarted") + .default_value(Settings::default().max_reorder) + .minimum(0) + .maximum(0x7fff) + .mutable_playing() + .build(), + glib::ParamSpecBoolean::builder("source-info") + .nick("RTP Source Info") + .blurb("Add RTP source information as buffer metadata") + .default_value(Settings::default().source_info) + .mutable_playing() + .build(), + glib::ParamSpecBoolean::builder("auto-header-extensions") + .nick("Automatic RTP Header Extensions") + .blurb("Whether RTP header extensions should be automatically enabled, if an implementation is available") + .default_value(Settings::default().auto_header_extensions) + .mutable_ready() + .build(), + gst::ParamSpecArray::builder("extensions") + .nick("RTP Header Extensions") + .blurb("List of enabled RTP header extensions") + .element_spec(&glib::ParamSpecObject::builder::("extension") + .nick("RTP Header Extension") + .blurb("Enabled RTP header extension") + .read_only() + .build() + ) + .read_only() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn signals() -> &'static [glib::subclass::Signal] { + static SIGNALS: Lazy> = Lazy::new(|| { + vec![ + glib::subclass::Signal::builder("add-extension") + .action() + .param_types([gst_rtp::RTPHeaderExtension::static_type()]) + .class_handler(|_token, args| { + let s = args[0].get::().unwrap(); + let ext = args[1].get::<&gst_rtp::RTPHeaderExtension>().unwrap(); + s.imp().add_extension(ext); + + None + }) + .build(), + glib::subclass::Signal::builder("request-extension") + .param_types([u32::static_type(), String::static_type()]) + .return_type::() + .accumulator(|_hint, acc, val| { + if matches!(val.get::>(), Ok(Some(_))) { + *acc = val.clone(); + false + } else { + true + } + }) + .class_handler(|_token, args| { + let s = args[0].get::().unwrap(); + let ext_id = args[1].get::().unwrap(); + let uri = args[2].get::<&str>().unwrap(); + let ext = s.imp().request_extension(ext_id, uri); + + Some(ext.to_value()) + }) + .build(), + glib::subclass::Signal::builder("clear-extensions") + .action() + .class_handler(|_token, args| { + let s = args[0].get::().unwrap(); + s.imp().clear_extensions(); + + None + }) + .build(), + ] + }); + + SIGNALS.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "max-reorder" => { + self.settings.lock().unwrap().max_reorder = value.get().unwrap(); + } + "source-info" => { + self.settings.lock().unwrap().source_info = value.get().unwrap(); + } + "auto-header-extensions" => { + self.settings.lock().unwrap().auto_header_extensions = value.get().unwrap(); + } + _ => unimplemented!(), + }; + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "stats" => self.create_stats().to_value(), + "max-reorder" => self.settings.lock().unwrap().max_reorder.to_value(), + "source-info" => self.settings.lock().unwrap().source_info.to_value(), + "auto-header-extensions" => self + .settings + .lock() + .unwrap() + .auto_header_extensions + .to_value(), + "extensions" => gst::Array::new(self.extensions.lock().unwrap().values()).to_value(), + _ => unimplemented!(), + } + } + + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.obj(); + obj.add_pad(&self.sink_pad).unwrap(); + obj.add_pad(&self.src_pad).unwrap(); + } +} + +impl GstObjectImpl for RtpBaseDepay2 {} + +impl ElementImpl for RtpBaseDepay2 { + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result { + gst::debug!(CAT, imp: self, "Changing state: {transition}"); + + if transition == gst::StateChange::ReadyToPaused { + *self.state.borrow_mut() = State::default(); + *self.stats.lock().unwrap() = None; + + let obj = self.obj(); + (obj.class().as_ref().start)(&obj).map_err(|err_msg| { + self.post_error_message(err_msg); + gst::StateChangeError + })?; + } + + let ret = self.parent_change_state(transition)?; + + if transition == gst::StateChange::PausedToReady { + let obj = self.obj(); + (obj.class().as_ref().stop)(&obj).map_err(|err_msg| { + self.post_error_message(err_msg); + gst::StateChangeError + })?; + + *self.state.borrow_mut() = State::default(); + *self.stats.lock().unwrap() = None; + } + + Ok(ret) + } +} diff --git a/net/rtp/src/basedepay/mod.rs b/net/rtp/src/basedepay/mod.rs new file mode 100644 index 00000000..b345b523 --- /dev/null +++ b/net/rtp/src/basedepay/mod.rs @@ -0,0 +1,543 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use std::ops::{Range, RangeBounds, RangeInclusive}; + +use gst::{glib, prelude::*, subclass::prelude::*}; + +pub mod imp; + +glib::wrapper! { + pub struct RtpBaseDepay2(ObjectSubclass) + @extends gst::Element, gst::Object; +} + +/// Trait containing extension methods for `RtpBaseDepay2`. +pub trait RtpBaseDepay2Ext: IsA { + /// Sends a caps event with the given caps downstream before the next output buffer. + fn set_src_caps(&self, src_caps: &gst::Caps) { + assert!(src_caps.is_fixed()); + self.upcast_ref::() + .imp() + .set_src_caps(src_caps); + } + + /// Drop the packets of the given packet range. + /// + /// This should be called when packets are dropped because they either don't make up any output + /// buffer or because they're corrupted in some way. + /// + /// All pending packets up to the end of the range are dropped, i.e. the start of the range is + /// irrelevant. + /// + /// It is not necessary to call this as part of `drain()` or `flush()` as all still pending packets are + /// considered dropped afterwards. + /// + /// The next buffer that is finished will automatically get the `DISCONT` flag set. + // FIXME: Allow subclasses to drop explicit packet ranges while keeping older packets around to + // allow for continuing to reconstruct a frame despite broken packets in the middle. + fn drop_packets(&self, ext_seqnum: impl RangeBounds) { + self.upcast_ref::() + .imp() + .drop_packets(ext_seqnum) + } + + /// Drop a single packet + /// + /// Convenience wrapper for calling drop_packets() for a single packet. + fn drop_packet(&self, packet: &Packet) { + self.drop_packets(packet.ext_seqnum()..=packet.ext_seqnum()) + } + + /// Queue a buffer made from a given range of packet seqnums and timestamp offsets. + /// + /// All buffers that are queued during one call of `handle_packet()` are collected in a + /// single buffer list and forwarded once `handle_packet()` has returned with a successful flow + /// return or `finish_pending_buffers()` was called. + /// + /// All pending packets for which buffers were queued are released once `handle_packets()` + /// returned except for the last one. This means that it is possible for subclasses to queue a + /// buffer and output a remaining chunk of that buffer together with data from the next buffer. + /// + /// If passing `OutOfBand` then the buffer is assumed to be produced using some other data, + /// e.g. from the caps, and not associated with any packets. In that case it will be pushed + /// right before the next buffer with the timestamp of that buffer, or at EOS with the + /// timestamp of the previous buffer. + /// + /// In all other cases a seqnum range is provided and needs to be valid. + /// + /// If the seqnum range doesn't start with the first pending packet then all packets up to the + /// first one given in the range are considered dropped. + /// + /// Together with the seqnum range it is possible to provide a timestamp offset relative to + /// which the outgoing buffers's timestamp should be set. + /// + /// The timestamp offset is provided in nanoseconds relative to the PTS of the packet that the + /// first seqnum refers to. This mode is mostly useful for subclasses that consume a packet + /// with multiple frames and send out one buffer per frame. + /// + /// Both a PTS and DTS offset can be provided. If no DTS offset is provided then no DTS will be + /// set at all. If no PTS offset is provided then the first buffer for a given start seqnum + /// will get the PTS of the corresponding packet and all following buffers that start with the + /// same seqnum will get no PTS set. + /// + /// Note that the DTS offset is relative to the final PTS and as such should not include the + /// PTS offset. + fn queue_buffer( + &self, + packet_to_buffer_relation: PacketToBufferRelation, + buffer: gst::Buffer, + ) -> Result { + self.upcast_ref::() + .imp() + .queue_buffer(packet_to_buffer_relation, buffer) + } + + /// Finish currently pending buffers and push them downstream in a single buffer list. + fn finish_pending_buffers(&self) -> Result { + self.upcast_ref::() + .imp() + .finish_pending_buffers() + } + + /// Returns a reference to the sink pad. + fn sink_pad(&self) -> &gst::Pad { + self.upcast_ref::().imp().sink_pad() + } + + /// Returns a reference to the src pad. + fn src_pad(&self) -> &gst::Pad { + self.upcast_ref::().imp().src_pad() + } +} + +impl> RtpBaseDepay2Ext for O {} + +/// Trait to implement in `RtpBaseDepay2` subclasses. +pub trait RtpBaseDepay2Impl: ElementImpl { + /// By default only metas without any tags are copied. Adding tags here will also copy the + /// metas that *only* have exactly one of these tags. + /// + /// If more complex copying of metas is needed then [`RtpBaseDepay2Impl::transform_meta`] has + /// to be implemented. + const ALLOWED_META_TAGS: &'static [&'static str] = &[]; + + /// Called when streaming starts (READY -> PAUSED state change) + /// + /// Optional, can be used to initialise streaming state. + fn start(&self) -> Result<(), gst::ErrorMessage> { + self.parent_start() + } + + /// Called after streaming has stopped (PAUSED -> READY state change) + /// + /// Optional, can be used to clean up streaming state. + fn stop(&self) -> Result<(), gst::ErrorMessage> { + self.parent_stop() + } + + /// Called when new caps are received on the sink pad. + /// + /// Can be used to configure the caps on the src pad or to configure caps-specific state. + /// If draining is necessary because of the caps change then the subclass will have to do that. + /// + /// Optional, by default does nothing. + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + self.parent_set_sink_caps(caps) + } + + /// Called whenever a new packet is available. + fn handle_packet(&self, packet: &Packet) -> Result { + self.parent_handle_packet(packet) + } + + /// Called whenever a discontinuity or EOS is observed. + /// + /// The subclass should output any pending buffers it can output at this point. + /// + /// This will be followed by a call to [`Self::flush`]. + /// + /// Optional, by default drops all still pending packets and forwards all still pending buffers + /// with the last known timestamp. + fn drain(&self) -> Result { + self.parent_drain() + } + + /// Called on `FlushStop` or whenever all pending data should simply be discarded. + /// + /// The subclass should reset its internal state as necessary. + /// + /// Optional. + fn flush(&self) { + self.parent_flush() + } + + /// Called whenever a new event arrives on the sink pad. + /// + /// Optional, by default does the standard event handling of the base class. + fn sink_event(&self, event: gst::Event) -> Result { + self.parent_sink_event(event) + } + + /// Called whenever a new event arrives on the src pad. + /// + /// Optional, by default does the standard event handling of the base class. + fn src_event(&self, event: gst::Event) -> Result { + self.parent_src_event(event) + } + + /// Called whenever a new query arrives on the sink pad. + /// + /// Optional, by default does the standard query handling of the base class. + fn sink_query(&self, query: &mut gst::QueryRef) -> bool { + self.parent_sink_query(query) + } + + /// Called whenever a new query arrives on the src pad. + /// + /// Optional, by default does the standard query handling of the base class. + fn src_query(&self, query: &mut gst::QueryRef) -> bool { + self.parent_src_query(query) + } + + /// Called whenever a meta from an input buffer has to be copied to the output buffer. + /// + /// Optional, by default simply copies over all metas. + fn transform_meta( + &self, + in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ) { + self.parent_transform_meta(in_buf, meta, out_buf); + } +} + +/// Trait containing extension methods for `RtpBaseDepay2Impl`, specifically methods for chaining +/// up to the parent implementation of virtual methods. +pub trait RtpBaseDepay2ImplExt: RtpBaseDepay2Impl { + fn parent_set_sink_caps(&self, caps: &gst::Caps) -> bool { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.set_sink_caps)(self.obj().unsafe_cast_ref(), caps) + } + } + + fn parent_start(&self) -> Result<(), gst::ErrorMessage> { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.start)(self.obj().unsafe_cast_ref()) + } + } + + fn parent_stop(&self) -> Result<(), gst::ErrorMessage> { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.stop)(self.obj().unsafe_cast_ref()) + } + } + + fn parent_handle_packet(&self, packet: &Packet) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.handle_packet)(self.obj().unsafe_cast_ref(), packet) + } + } + + fn parent_drain(&self) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.drain)(self.obj().unsafe_cast_ref()) + } + } + + fn parent_flush(&self) { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.flush)(self.obj().unsafe_cast_ref()); + } + } + + fn parent_sink_event(&self, event: gst::Event) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.sink_event)(self.obj().unsafe_cast_ref(), event) + } + } + + fn parent_src_event(&self, event: gst::Event) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.src_event)(self.obj().unsafe_cast_ref(), event) + } + } + + fn parent_sink_query(&self, query: &mut gst::QueryRef) -> bool { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.sink_query)(self.obj().unsafe_cast_ref(), query) + } + } + + fn parent_src_query(&self, query: &mut gst::QueryRef) -> bool { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.src_query)(self.obj().unsafe_cast_ref(), query) + } + } + + fn parent_transform_meta( + &self, + in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ) { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.transform_meta)(self.obj().unsafe_cast_ref(), in_buf, meta, out_buf) + } + } +} + +impl RtpBaseDepay2ImplExt for T {} + +#[derive(Debug)] +pub struct Packet { + buffer: gst::MappedBuffer, + + discont: bool, + ext_seqnum: u64, + ext_timestamp: u64, + marker: bool, + payload_range: Range, +} + +impl Packet { + pub fn ext_seqnum(&self) -> u64 { + self.ext_seqnum + } + + pub fn ext_timestamp(&self) -> u64 { + self.ext_timestamp + } + + pub fn marker_bit(&self) -> bool { + self.marker + } + + pub fn discont(&self) -> bool { + self.discont + } + + pub fn pts(&self) -> Option { + self.buffer.buffer().pts() + } + + pub fn payload(&self) -> &[u8] { + &self.buffer[self.payload_range.clone()] + } + + pub fn payload_buffer(&self) -> gst::Buffer { + self.buffer + .buffer() + .copy_region( + gst::BufferCopyFlags::MEMORY, + self.payload_range.start..self.payload_range.end, + ) + .expect("Failed copying buffer") + } + + /// Note: This function will panic if the range is out of bounds. + pub fn payload_subbuffer(&self, range: impl RangeBounds) -> gst::Buffer { + let range_start = match range.start_bound() { + std::ops::Bound::Included(&start) => start, + std::ops::Bound::Excluded(&start) => start.checked_add(1).unwrap(), + std::ops::Bound::Unbounded => 0, + } + .checked_add(self.payload_range.start) + .unwrap(); + + let range_end = match range.end_bound() { + std::ops::Bound::Included(&range_end) => range_end + .checked_add(self.payload_range.start) + .and_then(|v| v.checked_add(1)) + .unwrap(), + std::ops::Bound::Excluded(&range_end) => { + range_end.checked_add(self.payload_range.start).unwrap() + } + std::ops::Bound::Unbounded => self.payload_range.end, + }; + + self.buffer + .buffer() + .copy_region(gst::BufferCopyFlags::MEMORY, range_start..range_end) + .expect("Failed to create subbuffer") + } + + /// Note: For offset with unspecified length just use `payload_subbuffer(off..)`. + /// Note: This function will panic if the offset or length are out of bounds. + pub fn payload_subbuffer_from_offset_with_length( + &self, + start: usize, + length: usize, + ) -> gst::Buffer { + self.payload_subbuffer(start..start + length) + } +} + +/// Class struct for `RtpBaseDepay2`. +#[repr(C)] +pub struct Class { + parent: gst::ffi::GstElementClass, + + start: fn(&RtpBaseDepay2) -> Result<(), gst::ErrorMessage>, + stop: fn(&RtpBaseDepay2) -> Result<(), gst::ErrorMessage>, + + set_sink_caps: fn(&RtpBaseDepay2, caps: &gst::Caps) -> bool, + handle_packet: fn(&RtpBaseDepay2, packet: &Packet) -> Result, + drain: fn(&RtpBaseDepay2) -> Result, + flush: fn(&RtpBaseDepay2), + + sink_event: fn(&RtpBaseDepay2, event: gst::Event) -> Result, + src_event: fn(&RtpBaseDepay2, event: gst::Event) -> Result, + + sink_query: fn(&RtpBaseDepay2, query: &mut gst::QueryRef) -> bool, + src_query: fn(&RtpBaseDepay2, query: &mut gst::QueryRef) -> bool, + + transform_meta: fn( + &RtpBaseDepay2, + in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ), + + allowed_meta_tags: &'static [&'static str], +} + +unsafe impl ClassStruct for Class { + type Type = imp::RtpBaseDepay2; +} + +impl std::ops::Deref for Class { + type Target = glib::Class<<::Type as ObjectSubclass>::ParentType>; + + fn deref(&self) -> &Self::Target { + unsafe { &*(&self.parent as *const _ as *const _) } + } +} + +unsafe impl IsSubclassable for RtpBaseDepay2 { + fn class_init(class: &mut glib::Class) { + Self::parent_class_init::(class); + + let class = class.as_mut(); + + class.start = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.start() + }; + + class.stop = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.stop() + }; + + class.set_sink_caps = |obj, caps| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.set_sink_caps(caps) + }; + + class.handle_packet = |obj, packet| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.handle_packet(packet) + }; + + class.drain = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.drain() + }; + + class.flush = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.flush() + }; + + class.sink_event = |obj, event| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.sink_event(event) + }; + + class.src_event = |obj, event| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.src_event(event) + }; + + class.sink_query = |obj, query| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.sink_query(query) + }; + + class.src_query = |obj, query| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.src_query(query) + }; + + class.transform_meta = |obj, in_buf, meta, out_buf| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.transform_meta(in_buf, meta, out_buf) + }; + + class.allowed_meta_tags = T::ALLOWED_META_TAGS; + } +} + +/// Timestamp offset between this buffer and the reference packet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code)] +pub enum TimestampOffset { + /// Offset in nanoseconds relative to the first seqnum this buffer belongs to. + Pts(gst::Signed), + /// Offset in nanoseconds relative to the first seqnum this buffer belongs to. + PtsAndDts(gst::Signed, gst::Signed), +} + +/// Relation between queued buffer and input packet seqnums. +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum PacketToBufferRelation { + Seqnums(RangeInclusive), + SeqnumsWithOffset { + seqnums: RangeInclusive, + timestamp_offset: TimestampOffset, + }, + OutOfBand, +} + +impl<'a> From<&'a Packet> for PacketToBufferRelation { + fn from(packet: &'a Packet) -> Self { + PacketToBufferRelation::Seqnums(packet.ext_seqnum()..=packet.ext_seqnum()) + } +} + +impl From for PacketToBufferRelation { + fn from(ext_seqnum: u64) -> Self { + PacketToBufferRelation::Seqnums(ext_seqnum..=ext_seqnum) + } +} diff --git a/net/rtp/src/basepay/imp.rs b/net/rtp/src/basepay/imp.rs new file mode 100644 index 00000000..123c6e7e --- /dev/null +++ b/net/rtp/src/basepay/imp.rs @@ -0,0 +1,2115 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use atomic_refcell::AtomicRefCell; +use gst::{glib, prelude::*, subclass::prelude::*}; +use gst_rtp::prelude::*; + +use once_cell::sync::Lazy; +use smallvec::SmallVec; + +use std::{ + collections::{BTreeMap, VecDeque}, + num::Wrapping, + ops::{Bound, RangeBounds}, + sync::{ + atomic::{self, AtomicU64}, + Mutex, + }, +}; + +use super::PacketToBufferRelation; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpbasepay2", + gst::DebugColorFlags::empty(), + Some("RTP Base Payloader 2"), + ) +}); + +#[derive(Clone, Debug)] +struct Settings { + mtu: u32, + pt: u8, + ssrc: Option, + timestamp_offset: Option, + seqnum_offset: Option, + onvif_no_rate_control: bool, + scale_rtptime: bool, + source_info: bool, + auto_header_extensions: bool, +} + +impl Default for Settings { + fn default() -> Self { + Self { + // TODO: Should we use a different default here? libwebrtc uses 1200, for example + mtu: 1400, + pt: 96, + ssrc: None, + timestamp_offset: None, + seqnum_offset: None, + onvif_no_rate_control: false, + scale_rtptime: true, + source_info: false, + auto_header_extensions: true, + } + } +} + +#[derive(Debug)] +struct Stream { + pt: u8, + ssrc: u32, + timestamp_offset: u32, + #[allow(dead_code)] + seqnum_offset: u16, + /// Set if onvif_no_rate_control || !scale_rtptime + use_stream_time: bool, +} + +/// Metadata of a pending buffer for tracking purposes. +struct PendingBuffer { + /// Buffer id of this buffer. + id: u64, + buffer: gst::Buffer, +} + +/// A pending packet that still has to be sent downstream. +struct PendingPacket { + /// If no PTS is set then this packet also has no timestamp yet + /// and must be pushed together with the next packet where + /// it is actually known, or at EOS with the last known one. + buffer: gst::Buffer, +} + +struct State { + segment: Option<(gst::Seqnum, gst::FormattedSegment)>, + /// Set when a new segment event should be sent downstream before the next buffer. + pending_segment: bool, + + sink_caps: Option, + src_caps: Option, + negotiated_src_caps: Option, + + /// Set when the srcpad caps are known. + clock_rate: Option, + /// Stream configuration. Set in Ready->Paused. + stream: Option, + + /// Set when the next outgoing buffer should have the discont flag set. + discont_pending: bool, + + /// Buffer id of the next buffer. + current_buffer_id: u64, + /// Last buffer id that was used for a packet. + last_used_buffer_id: u64, + + /// Last seqnum that was put on a packet. This is initialized with stream.seqnum_offset. + last_seqnum: Wrapping, + + /// Last RTP timestamp that was put on a packet. This is initialized with stream.timestamp_offset. + last_timestamp: Wrapping, + /// Last PTS that was put on an outgoing buffer. + last_pts: Option, + + /// Last PTS / RTP time mapping. + /// + /// This is reset when draining and used if the subclass provides RTP timestamp offsets + /// to get the base mapping. + last_pts_rtp_mapping: Option<(gst::ClockTime, u32)>, + + /// Pending input buffers that were not completely used up for outgoing packets yet. + pending_buffers: VecDeque, + /// Pending packets that have to be pushed downstream. Some of them might not have a PTS or RTP + /// timestamp yet. + pending_packets: VecDeque, +} + +impl Default for State { + fn default() -> Self { + Self { + segment: None, + pending_segment: false, + + sink_caps: None, + src_caps: None, + negotiated_src_caps: None, + + clock_rate: None, + stream: None, + + discont_pending: true, + + current_buffer_id: 0, + last_used_buffer_id: 0, + + last_seqnum: Wrapping(0), + + last_timestamp: Wrapping(0), + last_pts: None, + + last_pts_rtp_mapping: None, + + pending_buffers: VecDeque::new(), + pending_packets: VecDeque::new(), + } + } +} + +#[derive(Clone, Debug)] +struct Stats { + ssrc: u32, + pt: u8, + // Only known once the caps are known + clock_rate: Option, + running_time: Option, + seqnum: u16, + timestamp: u32, + seqnum_offset: u16, + timestamp_offset: u32, +} + +pub struct RtpBasePay2 { + sink_pad: gst::Pad, + src_pad: gst::Pad, + state: AtomicRefCell, + settings: Mutex, + stats: Mutex>, + + /// If set to a different value than `u64::MAX` then there + /// was an SSRC collision and we should switch to the provided + /// SSRC here. + ssrc_collision: AtomicU64, + /// Currently configured header extensions + extensions: Mutex>, +} + +/// Wrappers for public methods and associated helper functions. +#[allow(dead_code)] +impl RtpBasePay2 { + pub(super) fn mtu(&self) -> u32 { + self.settings.lock().unwrap().mtu + } + + pub(super) fn max_payload_size(&self) -> u32 { + let settings = self.settings.lock().unwrap(); + + // FIXME: This does not consider the space needed for header extensions. Doing so would + // require knowing the buffer id range so we can calculate the maximum here. + settings + .mtu + .saturating_sub(if settings.source_info { + rtp_types::RtpPacket::MAX_N_CSRCS as u32 * 4 + } else { + 0 + }) + .saturating_sub(rtp_types::RtpPacket::MIN_RTP_PACKET_LEN as u32) + } + + pub(super) fn set_src_caps(&self, src_caps: &gst::Caps) { + gst::debug!(CAT, imp: self, "Setting src caps {src_caps:?}"); + + let s = src_caps.structure(0).unwrap(); + assert!( + s.has_name("application/x-rtp"), + "Non-RTP caps {src_caps:?} not supported" + ); + + let mut state = self.state.borrow_mut(); + state.src_caps = Some(src_caps.clone()); + drop(state); + + self.negotiate(); + } + + fn negotiate(&self) { + let state = self.state.borrow_mut(); + let Some(ref src_caps) = state.src_caps else { + gst::debug!(CAT, imp: self, "No src caps set yet, can't negotiate"); + return; + }; + let mut src_caps = src_caps.clone(); + drop(state); + + gst::debug!(CAT, imp: self, "Configured src caps: {src_caps:?}"); + + let peer_caps = self.src_pad.peer_query_caps(Some(&src_caps)); + if !peer_caps.is_empty() { + gst::debug!(CAT, imp: self, "Peer caps: {peer_caps:?}"); + src_caps = peer_caps; + } else { + gst::debug!(CAT, imp: self, "Empty peer caps"); + } + gst::debug!(CAT, imp: self, "Negotiating with caps {src_caps:?}"); + + src_caps.make_mut(); + let obj = self.obj(); + (obj.class().as_ref().negotiate)(&obj, src_caps); + } + + pub(super) fn drop_buffers(&self, ids: impl RangeBounds) { + gst::trace!(CAT, imp: self, "Dropping buffers up to {:?}", ids.end_bound()); + + let mut state = self.state.borrow_mut(); + let end = match ids.end_bound() { + Bound::Included(end) => *end, + Bound::Excluded(end) if *end == 0 => return, + Bound::Excluded(end) => *end - 1, + Bound::Unbounded => { + state.pending_buffers.clear(); + return; + } + }; + + if let Some(back) = state.pending_buffers.back() { + if back.id <= end { + state.pending_buffers.clear(); + return; + } + } else { + return; + } + + while state.pending_buffers.front().map_or(false, |b| b.id <= end) { + let _ = state.pending_buffers.pop_front(); + } + } + + pub(super) fn queue_packet( + &self, + packet_to_buffer_relation: PacketToBufferRelation, + mut packet: rtp_types::RtpPacketBuilder<&[u8], &[u8]>, + ) -> Result { + gst::trace!( + CAT, + imp: self, + "Queueing packet for {packet_to_buffer_relation:?}", + ); + + let settings = self.settings.lock().unwrap().clone(); + + let mut state = self.state.borrow_mut(); + if state.negotiated_src_caps.is_none() { + gst::error!(CAT, imp: self, "No source pad caps negotiated yet"); + return Err(gst::FlowError::NotNegotiated); + } + + state.last_seqnum += 1; + let stream = state.stream.as_ref().unwrap(); + let seqnum = state.last_seqnum.0; + gst::trace!(CAT, imp: self, "Using seqnum {seqnum}"); + packet = packet + .payload_type(stream.pt) + .ssrc(stream.ssrc) + .sequence_number(seqnum); + let use_stream_time = stream.use_stream_time; + + // Queue up packet for later if it doesn't have any associated buffers that could be used + // for figuring out the timestamp. + if matches!(packet_to_buffer_relation, PacketToBufferRelation::OutOfBand) { + gst::trace!(CAT, imp: self, "Keeping packet without associated buffer until next packet or EOS"); + + // FIXME: Use more optimal packet writing API once available + // https://github.com/ystreet/rtp-types/issues/4 + // https://github.com/ystreet/rtp-types/issues/5 + // TODO: Maybe use an MTU-sized buffer pool? + let packet_buffer = packet.write_vec().map_err(|err| { + gst::error!(CAT, imp: self, "Can't write packet: {err}"); + gst::FlowError::Error + })?; + let packet_len = packet_buffer.len(); + + if (settings.mtu as usize) < packet_len { + gst::warning!(CAT, imp: self, "Generated packet has bigger size {packet_len} than MTU {}", settings.mtu); + } + + gst::trace!(CAT, imp: self, "Queueing packet of size {packet_len}"); + + let marker = { + let packet = rtp_types::RtpPacket::parse(&packet_buffer).unwrap(); + packet.marker_bit() + }; + + let mut buffer = gst::Buffer::from_mut_slice(packet_buffer); + { + let buffer = buffer.get_mut().unwrap(); + + if marker { + buffer.set_flags(gst::BufferFlags::MARKER); + } + + if state.discont_pending { + buffer.set_flags(gst::BufferFlags::DISCONT); + state.discont_pending = false; + } + } + + state.pending_packets.push_back(PendingPacket { buffer }); + + return Ok(gst::FlowSuccess::Ok); + }; + + let (ids, timestamp_offset) = match packet_to_buffer_relation { + PacketToBufferRelation::Ids(ids) => (ids.clone(), None), + PacketToBufferRelation::IdsWithOffset { + ids, + timestamp_offset, + } => (ids.clone(), Some(timestamp_offset)), + PacketToBufferRelation::OutOfBand => unreachable!(), + }; + + if ids.is_empty() { + gst::error!(CAT, imp: self, "Empty buffer id range provided"); + return Err(gst::FlowError::Error); + } + + let id_start = *ids.start(); + let id_end = *ids.end(); + + // Drop all older buffers + while state + .pending_buffers + .front() + .map_or(false, |b| b.id < id_start) + { + let b = state.pending_buffers.pop_front().unwrap(); + gst::trace!(CAT, imp: self, "Dropping buffer with id {}", b.id); + } + + let Some(front) = state.pending_buffers.front() else { + gst::error!(CAT, imp: self, "Queueing packet for future buffer ids not allowed"); + return Err(gst::FlowError::Error); + }; + + if id_end > state.pending_buffers.back().unwrap().id { + gst::error!(CAT, imp: self, "Queueing packet for future buffer ids not allowed"); + return Err(gst::FlowError::Error); + } + + // Set when caps are set + let clock_rate = state.clock_rate.unwrap(); + + // Clip PTS to the segment boundaries + let segment = state.segment.as_ref().unwrap().1.clone(); + let pts = front.buffer.pts().unwrap(); + let pts = { + let start = segment.start().expect("no segment start"); + let stop = segment.stop(); + + if pts <= start { + start + } else if let Some(stop) = stop { + if pts >= stop { + stop + } else { + pts + } + } else { + pts + } + }; + + let rtptime_base = if use_stream_time { + segment.to_stream_time(pts).unwrap() + } else { + segment.to_running_time(pts).unwrap() + }; + + let packet_pts; + let packet_rtptime; + if let Some(timestamp_offset) = timestamp_offset { + match timestamp_offset { + crate::basepay::TimestampOffset::Pts(pts_diff) => { + if let Some(stop) = segment.stop() { + if pts + pts_diff > stop { + packet_pts = stop + } else { + packet_pts = pts + pts_diff + } + } else { + packet_pts = pts + pts_diff + } + + let packet_rtptime_base = if use_stream_time { + segment.to_stream_time(packet_pts).unwrap() + } else { + segment.to_running_time(packet_pts).unwrap() + }; + + packet_rtptime = (*packet_rtptime_base + .mul_div_ceil(clock_rate as u64, *gst::ClockTime::SECOND) + .unwrap() + & 0xffff_ffff) as u32; + + state.last_pts_rtp_mapping = Some((packet_pts, packet_rtptime)); + } + crate::basepay::TimestampOffset::Rtp(rtp_diff) => { + let Some((base_pts, base_rtptime)) = state.last_pts_rtp_mapping else { + gst::error!(CAT, imp: self, "Have no base PTS / RTP time mapping"); + return Err(gst::FlowError::Error); + }; + + packet_pts = base_pts + + gst::ClockTime::from_nseconds( + rtp_diff + .mul_div_ceil(*gst::ClockTime::SECOND, clock_rate as u64) + .unwrap(), + ); + + let rate = if use_stream_time { + segment.applied_rate() + } else { + 1.0 / segment.rate() + }; + + let rtp_diff = if rate != 0.0 { + (rtp_diff as f64 * rate) as u64 + } else { + rtp_diff + }; + + packet_rtptime = base_rtptime + (rtp_diff & 0xffff_ffff) as u32; + } + } + } else { + packet_pts = pts; + packet_rtptime = (*rtptime_base + .mul_div_ceil(clock_rate as u64, *gst::ClockTime::SECOND) + .unwrap() + & 0xffff_ffff) as u32; + + state.last_pts_rtp_mapping = Some((packet_pts, packet_rtptime)); + } + + // Check if any discont is pending here + let mut discont_pending = state.discont_pending + || (state.last_used_buffer_id < id_end + && state + .pending_buffers + .iter() + .skip_while(|b| b.id <= state.last_used_buffer_id) + .take_while(|b| b.id <= id_end) + .any(|b| b.buffer.flags().contains(gst::BufferFlags::DISCONT))); + state.discont_pending = false; + + let stream = state.stream.as_ref().unwrap(); + let packet_rtptime = stream.timestamp_offset.wrapping_add(packet_rtptime); + + // update pts and rtp timestamp of all pending packets that have none yet + for packet in state + .pending_packets + .iter_mut() + .skip_while(|p| p.buffer.pts().is_some()) + { + assert!(packet.buffer.pts().is_none()); + let buffer = packet.buffer.get_mut().unwrap(); + buffer.set_pts(pts); + + if discont_pending { + buffer.set_flags(gst::BufferFlags::DISCONT); + discont_pending = false; + } + + let mut map = buffer.map_writable().unwrap(); + let mut packet = rtp_types::RtpPacketMut::parse(&mut map).unwrap(); + packet.set_timestamp(packet_rtptime); + } + + packet = packet.timestamp(packet_rtptime); + + if settings.source_info + /* || header_extensions */ + { + let mut csrc = [0u32; rtp_types::RtpPacket::MAX_N_CSRCS]; + let mut n_csrc = 0; + + // Copy over metas and other metadata from the packets that made up this buffer + for front in state.pending_buffers.iter().take_while(|b| b.id <= id_end) { + if n_csrc < rtp_types::RtpPacket::MAX_N_CSRCS { + if let Some(meta) = front.buffer.meta::() { + if let Some(ssrc) = meta.ssrc() { + if !csrc[..n_csrc].contains(&ssrc) { + csrc[n_csrc] = ssrc; + n_csrc += 1; + } + } + + for c in meta.csrc() { + if n_csrc >= rtp_types::RtpPacket::MAX_N_CSRCS { + break; + } + + if !csrc[..n_csrc].contains(c) { + csrc[n_csrc] = *c; + n_csrc += 1; + } + } + } + } + } + + if n_csrc > 0 { + for c in &csrc[..n_csrc] { + packet = packet.add_csrc(*c); + } + } + } + + // Calculate size and flags for header extensions + + // FIXME: We're only considering the first buffer that makes up this packet for RTP header + // extensions. + let extension_input_buffer = state + .pending_buffers + .iter() + .take_while(|b| b.id <= id_end) + .next() + .expect("no input buffer for this packet"); + + let extensions = self.extensions.lock().unwrap(); + let mut extension_flags = + gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE | gst_rtp::RTPHeaderExtensionFlags::TWO_BYTE; + let mut extension_size = 0; + for extension in extensions.values() { + extension_flags &= extension.supported_flags(); + + let max_size = extension.max_size(&extension_input_buffer.buffer); + let ext_id = extension.id(); + if max_size > 16 || ext_id > 14 { + extension_flags -= gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE; + } + if max_size > 255 || ext_id > 255 { + extension_flags -= gst_rtp::RTPHeaderExtensionFlags::TWO_BYTE; + } + extension_size += max_size; + } + + let extension_pattern = + if extension_flags.contains(gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE) { + extension_flags = gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE; + extension_size += extensions.len(); + 0xBEDE + } else if extension_flags.contains(gst_rtp::RTPHeaderExtensionFlags::TWO_BYTE) { + extension_flags = gst_rtp::RTPHeaderExtensionFlags::TWO_BYTE; + extension_size += 2 * extensions.len(); + 0x1000 + } else { + extension_flags = gst_rtp::RTPHeaderExtensionFlags::empty(); + extension_size = 0; + 0 + }; + + // Round up to a multiple of 4 bytes + extension_size = ((extension_size + 3) / 4) * 4; + + // If there are extensions, write an empty extension area of the required size. If this is + // not filled then it would be considered as padding inside the extension because of the + // zeroes, which are not a valid extension ID. + let mut extension_data = SmallVec::<[u8; 256]>::with_capacity(extension_size); + extension_data.resize(extension_size, 0); + + let mut packet = packet; + + if !extension_data.is_empty() && extension_pattern != 0 { + packet = packet.extension(extension_pattern, extension_data.as_slice()); + } + + // Queue up packet and timestamp all others without timestamp that are before it. + // FIXME: Use more optimal packet writing API once available + // https://github.com/ystreet/rtp-types/issues/4 + // https://github.com/ystreet/rtp-types/issues/5 + // TODO: Maybe use an MTU-sized buffer pool? + let packet_buffer = packet.write_vec().map_err(|err| { + gst::error!(CAT, imp: self, "Can't write packet: {err}"); + gst::FlowError::Error + })?; + let packet_len = packet_buffer.len(); + + // FIXME: See comment in `max_payload_size()`. We currently don't provide a way to the + // subclass to know how much space will be used up by extensions. + if (settings.mtu as usize) < packet_len - extension_size { + gst::warning!(CAT, imp: self, "Generated packet has bigger size {packet_len} than MTU {}", settings.mtu); + } + + gst::trace!( + CAT, + imp: self, + "Queueing packet of size {packet_len} with RTP timestamp {packet_rtptime} and PTS {packet_pts}", + ); + + let marker = { + let packet = rtp_types::RtpPacket::parse(&packet_buffer).unwrap(); + packet.marker_bit() + }; + + let mut buffer = gst::Buffer::from_mut_slice(packet_buffer); + + { + let buffer = buffer.get_mut().unwrap(); + + buffer.set_pts(packet_pts); + if marker { + buffer.set_flags(gst::BufferFlags::MARKER); + } + + if discont_pending { + buffer.set_flags(gst::BufferFlags::DISCONT); + } + + // Copy over metas and other metadata from the packets that made up this buffer + let obj = self.obj(); + for front in state.pending_buffers.iter().take_while(|b| b.id <= id_end) { + use std::ops::ControlFlow::*; + + front.buffer.foreach_meta(|meta| { + // Do not copy metas that are memory specific + if meta.has_tag::() + || meta.has_tag::() + { + return Continue(()); + } + + (obj.class().as_ref().transform_meta)(&obj, &front.buffer, &meta, buffer); + + Continue(()) + }); + } + } + + // Finally write extensions into the otherwise complete packet + if extension_size != 0 && extension_pattern != 0 { + let buffer = buffer.get_mut().unwrap(); + // FIXME Get a mutable reference to the output buffer via a pointer to + // work around bug in the C API. + // See https://gitlab.freedesktop.org/gstreamer/gstreamer-rs/-/issues/375 + let buffer_ptr = buffer.as_mut_ptr(); + + let mut map = buffer.map_writable().unwrap(); + let mut packet = rtp_types::RtpPacketMut::parse(&mut map).unwrap(); + + let mut extension_data = packet.extension_mut().unwrap(); + + for (ext_id, extension) in extensions.iter() { + let offset = if extension_flags == gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE { + 1 + } else { + 2 + }; + + if extension_data.len() < offset { + gst::error!(CAT, imp: self, "No space left for writing RTP header extension {ext_id}"); + break; + } + + if let Ok(written) = extension.write( + &extension_input_buffer.buffer, + extension_flags, + unsafe { gst::BufferRef::from_mut_ptr(buffer_ptr) }, + &mut extension_data[offset..], + ) { + // Nothing written, can just continue + if written == 0 { + continue; + } + + if extension_flags == gst_rtp::RTPHeaderExtensionFlags::ONE_BYTE { + assert!(written <= 16); + extension_data[0] = (*ext_id << 4) | (written as u8 - 1); + } else { + assert!(written <= 255); + extension_data[0] = *ext_id; + extension_data[1] = written as u8; + } + + extension_data = &mut extension_data[offset + written..]; + } else { + gst::error!(CAT, imp: self, "Writing RTP header extension {ext_id} failed"); + } + } + } + + drop(extensions); + + state.pending_packets.push_back(PendingPacket { buffer }); + + state.last_used_buffer_id = id_end; + state.last_timestamp = Wrapping(packet_rtptime); + state.last_pts = Some(pts); + + // Update stats + if let Some(stats) = self.stats.lock().unwrap().as_mut() { + stats.running_time = segment.to_running_time(packet_pts); + stats.seqnum = seqnum; + stats.timestamp = packet_rtptime; + } + + Ok(gst::FlowSuccess::Ok) + } + + pub(super) fn finish_pending_packets(&self) -> Result { + let mut state = self.state.borrow_mut(); + + // As long as there are packets that can be finished, take all with the same PTS and put + // them into a buffer list. + while state + .pending_packets + .iter() + .any(|p| p.buffer.pts().is_some()) + { + let pts = state.pending_packets.front().unwrap().buffer.pts().unwrap(); + + // First count number of pending packets so we can allocate a buffer list with the correct + // size instead of re-allocating for every added buffer, or simply push a single buffer + // downstream instead of allocating a buffer list just for one buffer. + let num_buffers = state + .pending_packets + .iter() + .take_while(|p| p.buffer.pts() == Some(pts)) + .count(); + + assert_ne!(num_buffers, 0); + + gst::trace!(CAT, imp: self, "Flushing {num_buffers} packets"); + + let segment_event = self.retrieve_pending_segment_event(&mut state); + + enum BufferOrList { + Buffer(gst::Buffer), + List(gst::BufferList), + } + + let packets = if num_buffers == 1 { + let buffer = state.pending_packets.pop_front().unwrap().buffer; + gst::trace!(CAT, imp: self, "Finishing buffer {buffer:?}"); + BufferOrList::Buffer(buffer) + } else { + let mut list = gst::BufferList::new_sized(num_buffers); + { + let list = list.get_mut().unwrap(); + while state + .pending_packets + .front() + .map_or(false, |p| p.buffer.pts() == Some(pts)) + { + let buffer = state.pending_packets.pop_front().unwrap().buffer; + gst::trace!(CAT, imp: self, "Finishing buffer {buffer:?}"); + list.add(buffer); + } + } + + BufferOrList::List(list) + }; + + drop(state); + + if let Some(segment_event) = segment_event { + let _ = self.src_pad.push_event(segment_event); + } + + let res = match packets { + BufferOrList::Buffer(buffer) => self.src_pad.push(buffer), + BufferOrList::List(list) => self.src_pad.push_list(list), + }; + + if let Err(err) = res { + if ![gst::FlowError::Flushing, gst::FlowError::Eos].contains(&err) { + gst::warning!(CAT, imp: self, "Failed pushing packets: {err:?}"); + } else { + gst::debug!(CAT, imp: self, "Failed pushing packets: {err:?}"); + } + return Err(err); + } + + state = self.state.borrow_mut(); + } + + Ok(gst::FlowSuccess::Ok) + } + + pub(super) fn sink_pad(&self) -> &gst::Pad { + &self.sink_pad + } + + pub(super) fn src_pad(&self) -> &gst::Pad { + &self.src_pad + } +} + +/// Default virtual method implementations. +impl RtpBasePay2 { + fn start_default(&self) -> Result<(), gst::ErrorMessage> { + Ok(()) + } + + fn stop_default(&self) -> Result<(), gst::ErrorMessage> { + Ok(()) + } + + fn set_sink_caps_default(&self, _caps: &gst::Caps) -> bool { + true + } + + fn negotiate_header_extensions( + &self, + state: &State, + src_caps: &mut gst::Caps, + ) -> Result<(), gst::FlowError> { + let s = src_caps.structure(0).unwrap(); + + // Check which header extensions to enable/disable based on the caps. + let mut caps_extensions = BTreeMap::new(); + for (k, v) in s.iter() { + let Some(ext_id) = k.strip_prefix("extmap-") else { + continue; + }; + let Ok(ext_id) = ext_id.parse::() else { + gst::error!(CAT, imp: self, "Can't parse RTP header extension id from caps {src_caps:?}"); + return Err(gst::FlowError::NotNegotiated); + }; + + let uri = if let Ok(uri) = v.get::() { + uri + } else if let Ok(arr) = v.get::() { + if let Some(uri) = arr.get(1).and_then(|v| v.get::().ok()) { + uri + } else { + gst::error!(CAT, imp: self, "Couldn't get URI for RTP header extension id {ext_id} from caps {src_caps:?}"); + return Err(gst::FlowError::NotNegotiated); + } + } else { + gst::error!(CAT, imp: self, "Couldn't get URI for RTP header extension id {ext_id} from caps {src_caps:?}"); + return Err(gst::FlowError::NotNegotiated); + }; + + caps_extensions.insert(ext_id, uri); + } + + let mut extensions = self.extensions.lock().unwrap(); + let mut extensions_changed = false; + for (ext_id, uri) in caps_extensions { + if let Some(extension) = extensions.get(&ext_id) { + if extension.uri().as_deref() == Some(&uri) { + // Same extension, nothing to do here, we will update it with the new caps + // later + continue; + } + gst::debug!( + CAT, + imp: self, + "Extension ID {ext_id} changed from {:?} to {uri}", + extension.uri(), + ); + extensions_changed |= true; + extensions.remove(&ext_id); + } + + gst::debug!(CAT, imp: self, "Requesting extension {uri} for ID {ext_id}"); + let ext = self + .obj() + .emit_by_name::>( + "request-extension", + &[&(ext_id as u32), &uri], + ); + + let Some(ext) = ext else { + gst::warning!(CAT, imp: self, "Couldn't create extension for {uri} with ID {ext_id}"); + continue; + }; + + if ext.id() != ext_id as u32 { + gst::warning!(CAT, imp: self, "Created extension has wrong ID"); + continue; + } + + extensions.insert(ext_id, ext); + extensions_changed |= true; + } + + let sink_caps = state.sink_caps.as_ref().unwrap(); + let mut to_remove = vec![]; + for (ext_id, ext) in extensions.iter() { + if !ext.set_attributes_from_caps(src_caps) { + gst::warning!(CAT, imp: self, "Failed to configure extension {ext_id} from caps {src_caps}"); + to_remove.push(*ext_id); + } + + if !ext.set_non_rtp_sink_caps(sink_caps) { + gst::warning!(CAT, imp: self, "Failed to configure extension {ext_id} from sink caps {sink_caps}"); + to_remove.push(*ext_id); + } + } + for ext_id in to_remove { + extensions.remove(&ext_id); + extensions_changed |= true; + } + + // First remove all header extension related fields from the caps, + // then add them again correctly from the actually selected extensions. + { + let caps = src_caps.make_mut(); + let s = caps.structure_mut(0).unwrap(); + s.filter_map_in_place(|field, value| { + if field.as_str().starts_with("extmap-") { + None + } else { + Some(value) + } + }); + for (_, ext) in extensions.iter() { + ext.set_caps_from_attributes(caps); + } + } + drop(extensions); + + if extensions_changed { + self.obj().notify("extensions"); + } + + Ok(()) + } + + fn negotiate_default(&self, mut src_caps: gst::Caps) { + gst::debug!(CAT, imp: self, "Negotiating caps {src_caps:?}"); + + // Fixate what is left to fixate. + src_caps.fixate(); + + let s = src_caps.structure(0).unwrap(); + assert!( + s.has_name("application/x-rtp"), + "Non-RTP caps {src_caps:?} not supported" + ); + + let clock_rate = match s.get::("clock-rate") { + Ok(clock_rate) if clock_rate > 0 => clock_rate as u32, + _ => { + panic!("RTP caps {src_caps:?} without 'clock-rate'"); + } + }; + + let mut state = self.state.borrow_mut(); + if self + .negotiate_header_extensions(&state, &mut src_caps) + .is_err() + { + state.negotiated_src_caps = None; + return; + } + + let stream = state.stream.as_ref().unwrap(); + let ssrc = stream.ssrc; + let pt = stream.pt; + + // Set SSRC and payload type. We don't negotiate these with downstream! + { + let caps = src_caps.make_mut(); + caps.set("ssrc", ssrc); + caps.set("payload", pt as i32); + } + + if Some(&src_caps) == state.negotiated_src_caps.as_ref() { + gst::debug!(CAT, imp: self, "Setting same caps {src_caps:?} again"); + return; + } + + let clock_rate_changed = state + .clock_rate + .map_or(false, |old_clock_rate| old_clock_rate != clock_rate); + state.negotiated_src_caps = Some(src_caps.clone()); + state.clock_rate = Some(clock_rate); + self.stats.lock().unwrap().as_mut().unwrap().clock_rate = Some(clock_rate); + + let seqnum = if let Some((seqnum, _)) = state.segment { + seqnum + } else { + gst::Seqnum::next() + }; + + let mut segment_event = self.retrieve_pending_segment_event(&mut state); + drop(state); + + // FIXME: Drain also in other conditions? + if clock_rate_changed { + // First push the pending segment event downstream before draining + if let Some(segment_event) = segment_event.take() { + let _ = self.src_pad.push_event(segment_event); + } + + if let Err(err) = self.drain() { + // Just continue here. The payloader is now flushed and can proceed below, + // and if there was a serious error downstream it will happen on the next + // buffer pushed downstream + gst::debug!(CAT, imp: self, "Draining failed: {err:?}"); + } + } + + // Rely on the actual flow return later when pushing a buffer downstream. The bool + // return from pushing the event does not allow to distinguish between different kinds of + // errors. + let _ = self + .src_pad + .push_event(gst::event::Caps::builder(&src_caps).seqnum(seqnum).build()); + + if let Some(segment_event) = segment_event { + let _ = self.src_pad.push_event(segment_event); + } + } + + fn handle_buffer_default( + &self, + _buffer: &gst::Buffer, + _id: u64, + ) -> Result { + unimplemented!() + } + + fn drain_default(&self) -> Result { + Ok(gst::FlowSuccess::Ok) + } + + fn flush_default(&self) {} + + fn sink_event_default(&self, event: gst::Event) -> Result { + match event.view() { + gst::EventView::Caps(event) => { + let caps = event.caps_owned(); + + return self.set_sink_caps(caps); + } + gst::EventView::Segment(event) => { + let segment = event.segment(); + let seqnum = event.seqnum(); + + let state = self.state.borrow(); + if state.sink_caps.is_none() { + gst::warning!(CAT, imp: self, "Received segment before caps"); + return Err(gst::FlowError::NotNegotiated); + } + + let same_segment = state + .segment + .as_ref() + .map(|(old_seqnum, old_segment)| { + (&seqnum, segment) == (old_seqnum, old_segment.upcast_ref()) + }) + .unwrap_or(false); + // Only drain if the segment changed and we actually had one before + let drain = !same_segment && state.segment.is_some(); + drop(state); + + if !same_segment { + gst::debug!(CAT, imp: self, "Received segment {segment:?}"); + + if drain { + if let Err(err) = self.drain() { + // Just continue here. The payloader is now flushed and can proceed below, + // and if there was a serious error downstream it will happen on the next + // buffer pushed downstream + gst::debug!(CAT, imp: self, "Draining failed: {err:?}"); + } + } + + let mut state = self.state.borrow_mut(); + if let Some(segment) = segment.downcast_ref::() { + state.segment = Some((event.seqnum(), segment.clone())); + } else { + gst::error!(CAT, imp: self, "Segments in non-TIME format are not supported"); + state.segment = None; + // FIXME: Forget everything here? + return Err(gst::FlowError::Error); + } + + if state.negotiated_src_caps.is_none() { + state.pending_segment = true; + gst::debug!(CAT, imp: self, "Received segment before knowing src caps, delaying"); + return Ok(gst::FlowSuccess::Ok); + } + + drop(state); + } + } + gst::EventView::Eos(_) => { + if let Err(err) = self.drain() { + gst::debug!(CAT, imp: self, "Draining on EOS failed: {err:?}"); + } + } + gst::EventView::FlushStop(_) => { + self.flush(); + + let mut state = self.state.borrow_mut(); + state.segment = None; + state.pending_segment = false; + + // RTP timestamp and seqnum are not reset here + state.last_pts = None; + + drop(state); + + *self.stats.lock().unwrap() = None; + } + _ => (), + } + + gst::debug!(CAT, imp: self, "Forwarding event: {event:?}"); + if self.src_pad.push_event(event) { + Ok(gst::FlowSuccess::Ok) + } else if self.src_pad.pad_flags().contains(gst::PadFlags::FLUSHING) { + Err(gst::FlowError::Flushing) + } else { + Err(gst::FlowError::Error) + } + } + + fn handle_ssrc_collision_event(&self, s: &gst::StructureRef) { + let Ok(ssrc) = s.get::("ssrc") else { + return; + }; + + let stats = self.stats.lock().unwrap(); + let Some(ref stats) = &*stats else { + return; + }; + + if stats.ssrc != ssrc { + return; + } + + let new_ssrc = if let Ok(suggested_ssrc) = s.get::("suggested-ssrc") { + suggested_ssrc + } else { + use rand::prelude::*; + let mut rng = rand::thread_rng(); + + loop { + let new_ssrc = rng.gen::(); + if new_ssrc != stats.ssrc { + break new_ssrc; + } + } + }; + + self.ssrc_collision + .store(new_ssrc as u64, atomic::Ordering::SeqCst); + } + + fn src_event_default(&self, event: gst::Event) -> Result { + if let gst::EventView::CustomUpstream(ev) = event.view() { + if let Some(s) = ev.structure() { + if s.name() == "GstRTPCollision" { + self.handle_ssrc_collision_event(s); + } + } + } + + if gst::Pad::event_default(&self.src_pad, Some(&*self.obj()), event) { + Ok(gst::FlowSuccess::Ok) + } else if self.sink_pad.pad_flags().contains(gst::PadFlags::FLUSHING) { + Err(gst::FlowError::Flushing) + } else { + Err(gst::FlowError::Error) + } + } + + fn sink_query_default(&self, query: &mut gst::QueryRef) -> bool { + gst::Pad::query_default(&self.sink_pad, Some(&*self.obj()), query) + } + + fn src_query_default(&self, query: &mut gst::QueryRef) -> bool { + gst::Pad::query_default(&self.src_pad, Some(&*self.obj()), query) + } + + fn transform_meta_default( + &self, + _in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ) { + let tags = meta.tags(); + + if tags.len() > 1 { + gst::trace!( + CAT, + imp: self, + "Not copying meta {}: has multiple tags {tags:?}", + meta.api(), + ); + return; + } + + let allowed_tags = self.obj().class().as_ref().allowed_meta_tags; + if tags.len() == 1 { + let meta_tag = &tags[0]; + + if !allowed_tags.iter().copied().any(|tag| tag == *meta_tag) { + gst::trace!( + CAT, + imp: self, + "Not copying meta {}: tag '{meta_tag}' not allowed", + meta.api(), + ); + return; + } + } + + gst::trace!(CAT, imp: self, "Copying meta {}", meta.api()); + + if let Err(err) = meta.transform(out_buf, &gst::meta::MetaTransformCopy::new(false, ..)) { + gst::trace!(CAT, imp: self, "Could not copy meta {}: {err}", meta.api()); + } + } + + fn add_extension(&self, ext: &gst_rtp::RTPHeaderExtension) { + assert_ne!(ext.id(), 0); + + let mut extensions = self.extensions.lock().unwrap(); + extensions.insert(ext.id() as u8, ext.clone()); + self.src_pad.mark_reconfigure(); + drop(extensions); + + self.obj().notify("extensions"); + } + + fn clear_extensions(&self) { + let mut extensions = self.extensions.lock().unwrap(); + extensions.clear(); + self.src_pad.mark_reconfigure(); + drop(extensions); + + self.obj().notify("extensions"); + } + + fn request_extension(&self, ext_id: u32, uri: &str) -> Option { + let settings = self.settings.lock().unwrap(); + if !settings.auto_header_extensions { + return None; + } + drop(settings); + + let Some(ext) = gst_rtp::RTPHeaderExtension::create_from_uri(uri) else { + gst::debug!( + CAT, + imp: self, + "Didn't find any extension implementing URI {uri}", + ); + return None; + }; + + gst::debug!( + CAT, + imp: self, + "Automatically enabling extension {} for URI {uri}", + ext.name(), + ); + + ext.set_id(ext_id); + + Some(ext) + } +} + +/// Pad functions and associated helper functions. +impl RtpBasePay2 { + fn sink_chain( + &self, + _pad: &gst::Pad, + buffer: gst::Buffer, + ) -> Result { + gst::trace!(CAT, imp: self, "Received buffer {buffer:?}"); + + let settings = self.settings.lock().unwrap().clone(); + self.handle_buffer(&settings, buffer) + } + + fn sink_chain_list( + &self, + _pad: &gst::Pad, + list: gst::BufferList, + ) -> Result { + gst::trace!(CAT, imp: self, "Received buffer list {list:?}"); + + let settings = self.settings.lock().unwrap().clone(); + for buffer in list.iter_owned() { + self.handle_buffer(&settings, buffer)?; + } + + Ok(gst::FlowSuccess::Ok) + } + + fn sink_event( + &self, + pad: &gst::Pad, + event: gst::Event, + ) -> Result { + gst::debug!(CAT, obj: pad, "Received event: {event:?}"); + + let obj = self.obj(); + (obj.class().as_ref().sink_event)(&obj, event) + } + + fn src_event( + &self, + pad: &gst::Pad, + event: gst::Event, + ) -> Result { + gst::debug!(CAT, obj: pad, "Received event: {event:?}"); + + let obj = self.obj(); + (obj.class().as_ref().src_event)(&obj, event) + } + + fn sink_query(&self, pad: &gst::Pad, query: &mut gst::QueryRef) -> bool { + gst::trace!(CAT, obj: pad, "Received query: {query:?}"); + + let obj = self.obj(); + (obj.class().as_ref().sink_query)(&obj, query) + } + + fn src_query(&self, pad: &gst::Pad, query: &mut gst::QueryRef) -> bool { + gst::trace!(CAT, obj: pad, "Received query: {query:?}"); + + let obj = self.obj(); + (obj.class().as_ref().src_query)(&obj, query) + } + + fn retrieve_pending_segment_event(&self, state: &mut State) -> Option { + // If a segment event was pending, take it now and push it out later. + if !state.pending_segment { + return None; + } + + state.negotiated_src_caps.as_ref()?; + + let (seqnum, segment) = state.segment.as_ref().unwrap(); + let segment_event = gst::event::Segment::builder(segment) + .seqnum(*seqnum) + .build(); + state.pending_segment = false; + + gst::debug!(CAT, imp: self, "Created segment event {segment:?} with seqnum {seqnum:?}"); + + Some(segment_event) + } + + fn handle_buffer( + &self, + _settings: &Settings, + buffer: gst::Buffer, + ) -> Result { + // Check if renegotiation is needed. + if self.src_pad.check_reconfigure() { + self.negotiate(); + } + + let mut state = self.state.borrow_mut(); + + if state.sink_caps.is_none() { + gst::error!(CAT, imp: self, "No sink pad caps"); + gst::element_imp_error!( + self, + gst::CoreError::Negotiation, + [ + "No input format was negotiated, i.e. no caps event was received. \ + Perhaps you need a parser or typefind element before the payloader", + ] + ); + return Err(gst::FlowError::NotNegotiated); + } + + if buffer.flags().contains(gst::BufferFlags::HEADER) + && self.obj().class().as_ref().drop_header_buffers + { + gst::trace!(CAT, imp: self, "Dropping buffer with HEADER flag"); + return Ok(gst::FlowSuccess::Ok); + } + + if buffer.pts().is_none() { + gst::error!(CAT, imp: self, "Buffers without PTS"); + return Err(gst::FlowError::Error); + }; + + // TODO: Should we drain on DISCONT here, or leave it to the subclass? + + // Check for too many pending packets + if let Some((front, back)) = + Option::zip(state.pending_buffers.front(), state.pending_buffers.back()) + { + let pts_diff = back.buffer.pts().opt_saturating_sub(front.buffer.pts()); + + if let Some(pts_diff) = pts_diff { + if pts_diff > gst::ClockTime::SECOND { + gst::warning!(CAT, imp: self, "More than {pts_diff} of PTS time queued, probably a bug in the subclass"); + } + } + } + + let ssrc_collision = self.ssrc_collision.load(atomic::Ordering::SeqCst); + if ssrc_collision != u64::MAX { + let new_ssrc = ssrc_collision as u32; + let stream = state.stream.as_mut().unwrap(); + gst::debug!( + CAT, + imp: self, + "Switching from SSRC {} to {} because of SSRC collision", + stream.ssrc, + new_ssrc, + ); + stream.ssrc = new_ssrc; + self.ssrc_collision + .store(u64::MAX, atomic::Ordering::SeqCst); + + if let Some(ref src_caps) = state.negotiated_src_caps { + let mut src_caps = src_caps.copy(); + + { + let caps = src_caps.get_mut().unwrap(); + caps.set("ssrc", new_ssrc); + } + state.negotiated_src_caps = Some(src_caps.clone()); + + let seqnum = if let Some((seqnum, _)) = state.segment { + seqnum + } else { + gst::Seqnum::next() + }; + drop(state); + + let _ = self + .src_pad + .push_event(gst::event::Caps::builder(&src_caps).seqnum(seqnum).build()); + + state = self.state.borrow_mut(); + } + + let mut stats_guard = self.stats.lock().unwrap(); + let stats = stats_guard.as_mut().unwrap(); + stats.ssrc = new_ssrc; + drop(stats_guard); + } + + let id = state.current_buffer_id; + state.current_buffer_id += 1; + + gst::trace!(CAT, imp: self, "Handling buffer {buffer:?} with id {id}"); + state.pending_buffers.push_back(PendingBuffer { + id, + buffer: buffer.clone(), + }); + drop(state); + + let obj = self.obj(); + let mut res = (obj.class().as_ref().handle_buffer)(&obj, &buffer, id); + if let Err(err) = res { + gst::error!(CAT, imp: self, "Failed handling buffer: {err:?}"); + } else { + res = self.finish_pending_packets(); + + if let Err(err) = res { + gst::debug!(CAT, imp: self, "Failed finishing pending packets: {err:?}"); + } + } + + // Now drop all pending buffers that were used for producing packets + // except for the very last one as that might still be used. + let mut state = self.state.borrow_mut(); + while state + .pending_buffers + .front() + .map_or(false, |b| b.id < state.last_used_buffer_id) + { + let _ = state.pending_buffers.pop_front(); + } + + res + } +} + +/// Other methods. +impl RtpBasePay2 { + fn set_sink_caps(&self, caps: gst::Caps) -> Result { + gst::debug!(CAT, imp: self, "Received caps {caps:?}"); + + let mut state = self.state.borrow_mut(); + let same_caps = state + .sink_caps + .as_ref() + .map(|old_caps| &caps == old_caps) + .unwrap_or(false); + state.sink_caps = Some(caps.clone()); + drop(state); + + if !same_caps { + let obj = self.obj(); + let set_sink_caps_res = if (obj.class().as_ref().set_sink_caps)(&obj, &caps) { + gst::debug!(CAT, imp: self, "Caps {caps:?} accepted"); + Ok(gst::FlowSuccess::Ok) + } else { + gst::warning!(CAT, imp: self, "Caps {caps:?} not accepted"); + Err(gst::FlowError::NotNegotiated) + }; + + let mut state = self.state.borrow_mut(); + if let Err(err) = set_sink_caps_res { + state.sink_caps = None; + // FIXME: Forget everything here? + return Err(err); + } + drop(state); + } + + Ok(gst::FlowSuccess::Ok) + } + + fn drain(&self) -> Result { + let obj = self.obj(); + if let Err(err) = (obj.class().as_ref().drain)(&obj) { + if ![gst::FlowError::Flushing, gst::FlowError::Eos].contains(&err) { + gst::warning!(CAT, imp: self, "Draining failed: {err:?}"); + } else { + gst::debug!(CAT, imp: self, "Draining failed: {err:?}"); + } + self.flush(); + return Err(err); + } + + let mut state = self.state.borrow_mut(); + state.pending_buffers.clear(); + + if !state.pending_packets.is_empty() { + gst::debug!(CAT, imp: self, "Pushing all pending packets"); + } + + // Update PTS and RTP timestamp of all pending packets that have none yet + let pts = state.last_pts; + let rtptime = state.last_timestamp.0; + let mut discont_pending = state.discont_pending; + state.discont_pending = false; + + for packet in state + .pending_packets + .iter_mut() + .skip_while(|p| p.buffer.pts().is_some()) + { + assert!(packet.buffer.pts().is_none()); + let buffer = packet.buffer.get_mut().unwrap(); + buffer.set_pts(pts); + + if discont_pending { + buffer.set_flags(gst::BufferFlags::DISCONT); + discont_pending = false; + } + + let mut map = buffer.map_writable().unwrap(); + let mut packet = rtp_types::RtpPacketMut::parse(&mut map).unwrap(); + packet.set_timestamp(rtptime); + } + + drop(state); + + // Forward all packets + let res = self.finish_pending_packets(); + + self.flush(); + + res + } + + fn flush(&self) { + let obj = self.obj(); + (obj.class().as_ref().flush)(&obj); + + let mut state = self.state.borrow_mut(); + state.pending_buffers.clear(); + + // Drop any remaining pending packets, this can only really happen on errors + state.pending_packets.clear(); + + // Reset mapping and require the subclass to provide a new one next time + state.last_pts_rtp_mapping = None; + state.discont_pending = true; + } + + fn create_stats(&self) -> gst::Structure { + let stats = self.stats.lock().unwrap().clone(); + if let Some(stats) = stats { + gst::Structure::builder("application/x-rtp-payload-stats") + .field("ssrc", stats.ssrc) + .field("clock-rate", stats.clock_rate.unwrap_or_default()) + .field("running-time", stats.running_time) + .field("seqnum", stats.seqnum as u32) + .field("timestamp", stats.timestamp) + .field("pt", stats.pt as u32) + .field("seqnum-offset", stats.seqnum_offset as u32) + .field("timestamp-offset", stats.timestamp_offset) + .build() + } else { + gst::Structure::builder("application/x-rtp-payload-stats").build() + } + } +} + +#[glib::object_subclass] +impl ObjectSubclass for RtpBasePay2 { + const NAME: &'static str = "GstRtpBasePay2"; + const ABSTRACT: bool = true; + type Type = super::RtpBasePay2; + type ParentType = gst::Element; + type Class = super::Class; + + fn class_init(class: &mut Self::Class) { + class.start = |obj| obj.imp().start_default(); + class.stop = |obj| obj.imp().stop_default(); + class.set_sink_caps = |obj, caps| obj.imp().set_sink_caps_default(caps); + class.negotiate = |obj, caps| obj.imp().negotiate_default(caps); + class.handle_buffer = |obj, buffer, id| obj.imp().handle_buffer_default(buffer, id); + class.drain = |obj| obj.imp().drain_default(); + class.flush = |obj| obj.imp().flush_default(); + class.sink_event = |obj, event| obj.imp().sink_event_default(event); + class.src_event = |obj, event| obj.imp().src_event_default(event); + class.sink_query = |obj, query| obj.imp().sink_query_default(query); + class.src_query = |obj, query| obj.imp().src_query_default(query); + class.transform_meta = + |obj, in_buf, meta, out_buf| obj.imp().transform_meta_default(in_buf, meta, out_buf); + } + + fn with_class(class: &Self::Class) -> Self { + let templ = class + .pad_template("sink") + .expect("Subclass did not provide a \"sink\" pad template"); + let sink_pad = gst::Pad::builder_from_template(&templ) + .chain_function(|pad, parent, buffer| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.sink_chain(pad, buffer), + ) + }) + .chain_list_function(|pad, parent, list| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.sink_chain_list(pad, list), + ) + }) + .event_full_function(|pad, parent, event| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.sink_event(pad, event), + ) + }) + .query_function(|pad, parent, query| { + Self::catch_panic_pad_function(parent, || false, |imp| imp.sink_query(pad, query)) + }) + .build(); + + let templ = class + .pad_template("src") + .expect("Subclass did not provide a \"src\" pad template"); + let src_pad = gst::Pad::builder_from_template(&templ) + .event_full_function(|pad, parent, event| { + Self::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |imp| imp.src_event(pad, event), + ) + }) + .query_function(|pad, parent, query| { + Self::catch_panic_pad_function(parent, || false, |imp| imp.src_query(pad, query)) + }) + .flags(gst::PadFlags::FIXED_CAPS) + .build(); + + Self { + src_pad, + sink_pad, + state: AtomicRefCell::default(), + settings: Mutex::default(), + stats: Mutex::default(), + ssrc_collision: AtomicU64::new(u64::MAX), + extensions: Mutex::default(), + } + } +} + +impl ObjectImpl for RtpBasePay2 { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![ + glib::ParamSpecUInt::builder("mtu") + .nick("MTU") + .blurb("Maximum size of one RTP packet") + .default_value(Settings::default().mtu) + .minimum(28) + .mutable_playing() + .build(), + glib::ParamSpecUInt::builder("pt") + .nick("Payload Type") + .blurb("Payload type of the packets") + .default_value(u32::from(Settings::default().pt)) + .minimum(0) + .maximum(0x7f) + .mutable_ready() + .build(), + glib::ParamSpecInt64::builder("ssrc") + .nick("SSRC") + .blurb("SSRC of the packets (-1 == random)") + .default_value(Settings::default().ssrc.map(i64::from).unwrap_or(-1)) + .minimum(-1) + .maximum(u32::MAX as i64) + .mutable_ready() + .build(), + glib::ParamSpecInt64::builder("timestamp-offset") + .nick("Timestamp Offset") + .blurb("Offset that is added to all RTP timestamps (-1 == random)") + .default_value( + Settings::default() + .timestamp_offset + .map(i64::from) + .unwrap_or(-1), + ) + .minimum(-1) + .maximum(u32::MAX as i64) + .mutable_ready() + .build(), + glib::ParamSpecInt::builder("seqnum-offset") + .nick("Sequence Number Offset") + .blurb("Offset that is added to all RTP sequence numbers (-1 == random)") + .default_value( + Settings::default() + .seqnum_offset + .map(i32::from) + .unwrap_or(-1), + ) + .minimum(-1) + .maximum(u16::MAX as i32) + .mutable_ready() + .build(), + glib::ParamSpecBoolean::builder("onvif-no-rate-control") + .nick("ONVIF No Rate Control") + .blurb("Enable ONVIF Rate-Control=no timestamping mode") + .default_value(Settings::default().onvif_no_rate_control) + .mutable_ready() + .build(), + glib::ParamSpecBoolean::builder("scale-rtptime") + .nick("Scale RTP Time") + .blurb("Whether the RTP timestamp should be scaled with the rate (speed)") + .default_value(Settings::default().scale_rtptime) + .mutable_ready() + .build(), + glib::ParamSpecBoxed::builder::("stats") + .nick("Statistics") + .blurb("Various statistics") + .read_only() + .build(), + glib::ParamSpecUInt::builder("seqnum") + .nick("Sequence Number") + .blurb("RTP sequence number of the last packet") + .minimum(0) + .maximum(u16::MAX as u32) + .read_only() + .build(), + glib::ParamSpecUInt::builder("timestamp") + .nick("Timestamp") + .blurb("RTP timestamp of the last packet") + .minimum(0) + .maximum(u16::MAX as u32) + .read_only() + .build(), + glib::ParamSpecBoolean::builder("source-info") + .nick("RTP Source Info") + .blurb("Add RTP source information as buffer metadata") + .default_value(Settings::default().source_info) + .mutable_playing() + .build(), + glib::ParamSpecBoolean::builder("auto-header-extensions") + .nick("Automatic RTP Header Extensions") + .blurb("Whether RTP header extensions should be automatically enabled, if an implementation is available") + .default_value(Settings::default().auto_header_extensions) + .mutable_ready() + .build(), + gst::ParamSpecArray::builder("extensions") + .nick("RTP Header Extensions") + .blurb("List of enabled RTP header extensions") + .element_spec(&glib::ParamSpecObject::builder::("extension") + .nick("RTP Header Extension") + .blurb("Enabled RTP header extension") + .read_only() + .build() + ) + .read_only() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn signals() -> &'static [glib::subclass::Signal] { + static SIGNALS: Lazy> = Lazy::new(|| { + vec![ + glib::subclass::Signal::builder("add-extension") + .action() + .param_types([gst_rtp::RTPHeaderExtension::static_type()]) + .class_handler(|_token, args| { + let s = args[0].get::().unwrap(); + let ext = args[1].get::<&gst_rtp::RTPHeaderExtension>().unwrap(); + s.imp().add_extension(ext); + + None + }) + .build(), + glib::subclass::Signal::builder("request-extension") + .param_types([u32::static_type(), String::static_type()]) + .return_type::() + .accumulator(|_hint, acc, val| { + if matches!(val.get::>(), Ok(Some(_))) { + *acc = val.clone(); + false + } else { + true + } + }) + .class_handler(|_token, args| { + let s = args[0].get::().unwrap(); + let ext_id = args[1].get::().unwrap(); + let uri = args[2].get::<&str>().unwrap(); + let ext = s.imp().request_extension(ext_id, uri); + + Some(ext.to_value()) + }) + .build(), + glib::subclass::Signal::builder("clear-extensions") + .action() + .class_handler(|_token, args| { + let s = args[0].get::().unwrap(); + s.imp().clear_extensions(); + + None + }) + .build(), + ] + }); + + SIGNALS.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "mtu" => { + self.settings.lock().unwrap().mtu = value.get().unwrap(); + } + "pt" => { + self.settings.lock().unwrap().pt = value.get::().unwrap() as u8; + } + "ssrc" => { + let v = value.get::().unwrap(); + self.settings.lock().unwrap().ssrc = (v != -1).then_some(v as u32); + } + "timestamp-offset" => { + let v = value.get::().unwrap(); + self.settings.lock().unwrap().timestamp_offset = (v != -1).then_some(v as u32); + } + "seqnum-offset" => { + let v = value.get::().unwrap(); + self.settings.lock().unwrap().seqnum_offset = (v != -1).then_some(v as u16); + } + "onvif-no-rate-control" => { + self.settings.lock().unwrap().onvif_no_rate_control = value.get().unwrap(); + } + "scale-rtptime" => { + self.settings.lock().unwrap().scale_rtptime = value.get().unwrap(); + } + "source-info" => { + self.settings.lock().unwrap().source_info = value.get().unwrap(); + } + "auto-header-extensions" => { + self.settings.lock().unwrap().auto_header_extensions = value.get().unwrap(); + } + _ => unimplemented!(), + }; + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "mtu" => self.settings.lock().unwrap().mtu.to_value(), + "pt" => (u32::from(self.settings.lock().unwrap().pt)).to_value(), + "ssrc" => self + .settings + .lock() + .unwrap() + .ssrc + .map(i64::from) + .unwrap_or(-1) + .to_value(), + "timestamp-offset" => self + .settings + .lock() + .unwrap() + .timestamp_offset + .map(i64::from) + .unwrap_or(-1) + .to_value(), + "seqnum-offset" => self + .settings + .lock() + .unwrap() + .seqnum_offset + .map(i32::from) + .unwrap_or(-1) + .to_value(), + "onvif-no-rate-control" => self + .settings + .lock() + .unwrap() + .onvif_no_rate_control + .to_value(), + "scale-rtptime" => self.settings.lock().unwrap().scale_rtptime.to_value(), + "stats" => self.create_stats().to_value(), + "seqnum" => (self + .stats + .lock() + .unwrap() + .as_ref() + .map(|s| s.seqnum) + .unwrap_or(0) as u32) + .to_value(), + "timestamp" => self + .stats + .lock() + .unwrap() + .as_ref() + .map(|s| s.timestamp) + .unwrap_or(0) + .to_value(), + "source-info" => self.settings.lock().unwrap().source_info.to_value(), + "auto-header-extensions" => self + .settings + .lock() + .unwrap() + .auto_header_extensions + .to_value(), + "extensions" => gst::Array::new(self.extensions.lock().unwrap().values()).to_value(), + _ => unimplemented!(), + } + } + + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.obj(); + obj.add_pad(&self.sink_pad).unwrap(); + obj.add_pad(&self.src_pad).unwrap(); + } +} + +impl GstObjectImpl for RtpBasePay2 {} + +impl ElementImpl for RtpBasePay2 { + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result { + gst::debug!(CAT, imp: self, "Changing state: {transition}"); + + if transition == gst::StateChange::ReadyToPaused { + { + use rand::prelude::*; + let mut rng = rand::thread_rng(); + + let settings = self.settings.lock().unwrap(); + + let pt = settings.pt; + let ssrc = settings.ssrc.unwrap_or_else(|| rng.gen::()); + let timestamp_offset = settings + .timestamp_offset + .unwrap_or_else(|| rng.gen::()); + let seqnum_offset = settings.seqnum_offset.unwrap_or_else(|| rng.gen::()); + + // Need to check pt against template caps if it's a non-dynamic PT + if pt < 96 { + let mut pt_found = false; + + let templ_caps = self.src_pad.pad_template_caps(); + for s in templ_caps.iter() { + let Ok(allowed_pt) = s.value("payload") else { + continue; + }; + + if allowed_pt.can_intersect(&(pt as i32).to_value()) { + pt_found = true; + break; + } + } + + if !pt_found { + gst::error!( + CAT, + imp: self, + "Unsupported static payload type {pt}, not found in template caps {templ_caps}", + ); + // We would now return NotNegotiated the next time a buffer is tried to be sent + // downstream. + return Err(gst::StateChangeError); + } + } + + let stream = Stream { + pt, + ssrc, + timestamp_offset, + seqnum_offset, + use_stream_time: settings.onvif_no_rate_control || !settings.scale_rtptime, + }; + + gst::info!(CAT, imp: self, "Configuring {stream:?}"); + + *self.state.borrow_mut() = State { + stream: Some(stream), + last_seqnum: Wrapping(seqnum_offset), + last_timestamp: Wrapping(timestamp_offset), + ..State::default() + }; + + *self.stats.lock().unwrap() = Some(Stats { + ssrc, + pt, + clock_rate: None, + running_time: None, + seqnum: seqnum_offset, + timestamp: timestamp_offset, + seqnum_offset, + timestamp_offset, + }); + } + + let obj = self.obj(); + (obj.class().as_ref().start)(&obj).map_err(|err_msg| { + self.post_error_message(err_msg); + gst::StateChangeError + })?; + } + + let ret = self.parent_change_state(transition)?; + + if transition == gst::StateChange::PausedToReady { + let obj = self.obj(); + (obj.class().as_ref().stop)(&obj).map_err(|err_msg| { + self.post_error_message(err_msg); + gst::StateChangeError + })?; + + *self.state.borrow_mut() = State::default(); + *self.stats.lock().unwrap() = None; + } + + Ok(ret) + } +} diff --git a/net/rtp/src/basepay/mod.rs b/net/rtp/src/basepay/mod.rs new file mode 100644 index 00000000..5c6ee2f8 --- /dev/null +++ b/net/rtp/src/basepay/mod.rs @@ -0,0 +1,494 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use std::ops::{RangeBounds, RangeInclusive}; + +use gst::{glib, prelude::*, subclass::prelude::*}; + +pub mod imp; + +glib::wrapper! { + pub struct RtpBasePay2(ObjectSubclass) + @extends gst::Element, gst::Object; +} + +/// Trait containing extension methods for `RtpBasePay2`. +pub trait RtpBasePay2Ext: IsA { + /// Sends a caps event with the given caps downstream before the next output buffer. + /// + /// The caps must be `application/x-rtp` and contain the `clock-rate` field with a suitable + /// clock-rate for this stream. + /// + /// The caps can be unfixed and will be passed through `RtpBasePay2Impl::negotiate()` to + /// negotiate caps with downstream, and finally fixate them. + fn set_src_caps(&self, src_caps: &gst::Caps) { + self.upcast_ref::() + .imp() + .set_src_caps(src_caps); + } + + /// Drop the buffers from the given buffer range. + /// + /// This should be called when input buffers are dropped because they are not included in any + /// output packet. + /// + /// All pending buffers up to the end of the range are dropped, i.e. the start of the range is + /// irrelevant. + fn drop_buffers(&self, ids: impl RangeBounds) { + self.upcast_ref::().imp().drop_buffers(ids) + } + + /// Queue an RTP packet made from a given range of input buffer ids and timestamp offset. + /// + /// All packets that are queued during one call of `handle_buffer()` are collected in a + /// single buffer list and forwarded once `handle_buffer()` has returned with a successful flow + /// return or `finish_pending_packets()` was called. + /// + /// All pending buffers for which packets were queued are released once `handle_buffer()` + /// returned except for the last one. This means that it is possible for subclasses to queue a + /// buffer and output a remaining chunk of that buffer together with data from the next buffer. + /// + /// If passing `OutOfBand` then the packet is assumed to be produced using some other data, + /// e.g. from the caps, and not associated with any packets. In that case it will be pushed + /// right before the next packet with the timestamp of that packet, or at EOS with the + /// timestamp of the previous packet. + /// + /// In all other cases a buffer id range is provided and needs to be valid. + /// + /// If the id range doesn't start with the first pending buffer then all buffers up to the + /// first one given in the range are considered dropped. + /// + /// Together with the buffer ids it is possible to provide a timestamp offset relative to which + /// the outgoing RTP timestamp and GStreamer PTS should be set. + /// + /// The timestamp offset can be provided in two ways: + /// + /// * Nanoseconds relative to the PTS of the buffer that the first id refers to. This mode is + /// mostly useful for subclasses that consume a buffer with multiple frames and send out + /// one packet per frame. + /// + /// * RTP clock-rate units (without wrap-arounds) relative to the last buffer that had no + /// timestamp offset given in RTP clock-rate units. In this mode the subclass needs to be + /// careful to handle discontinuities of any sort correctly. Also, the subclass needs to + /// provide an explicit timestamp (either by setting no offset or by setting a PTS-based + /// offset) for the first packet ever and after every `drain()` or `flush()`. + fn queue_packet( + &self, + packet_to_buffer_relation: PacketToBufferRelation, + packet: rtp_types::RtpPacketBuilder<&[u8], &[u8]>, + ) -> Result { + self.upcast_ref::() + .imp() + .queue_packet(packet_to_buffer_relation, packet) + } + + /// Finish currently pending packets and push them downstream in a single buffer list. + fn finish_pending_packets(&self) -> Result { + self.upcast_ref::() + .imp() + .finish_pending_packets() + } + + /// Returns a reference to the sink pad. + fn sink_pad(&self) -> &gst::Pad { + self.upcast_ref::().imp().sink_pad() + } + + /// Returns a reference to the src pad. + fn src_pad(&self) -> &gst::Pad { + self.upcast_ref::().imp().src_pad() + } + + /// Returns the currently configured MTU. + fn mtu(&self) -> u32 { + self.upcast_ref::().imp().mtu() + } + + /// Returns the maximum available payload size. + fn max_payload_size(&self) -> u32 { + self.upcast_ref::().imp().max_payload_size() + } +} + +impl> RtpBasePay2Ext for O {} + +/// Trait to implement in `RtpBasePay2` subclasses. +pub trait RtpBasePay2Impl: ElementImpl { + /// Drop buffers with `HEADER` flag. + const DROP_HEADER_BUFFERS: bool = false; + + /// By default only metas without any tags are copied. Adding tags here will also copy the + /// metas that *only* have exactly one of these tags. + /// + /// If more complex copying of metas is needed then [`RtpBasePay2Impl::transform_meta`] has + /// to be implemented. + const ALLOWED_META_TAGS: &'static [&'static str] = &[]; + + /// Called when streaming starts (READY -> PAUSED state change) + /// + /// Optional, can be used to initialise streaming state. + fn start(&self) -> Result<(), gst::ErrorMessage> { + self.parent_start() + } + + /// Called after streaming has stopped (PAUSED -> READY state change) + /// + /// Optional, can be used to clean up streaming state. + fn stop(&self) -> Result<(), gst::ErrorMessage> { + self.parent_stop() + } + + /// Called when new caps are received on the sink pad. + /// + /// Can be used to configure the caps on the src pad or to configure caps-specific state. + /// + /// Optional, by default does nothing. + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + self.parent_set_sink_caps(caps) + } + + /// Called when new caps are configured on the source pad and whenever renegotiation has to happen. + /// + /// The `src_caps` are the caps passed into `set_src_caps()` before, intersected with the + /// supported caps by the peer, and will have to be fixated. + /// + /// Optional, by default sets the `payload` (pt) and `ssrc` fields, and negotiates RTP header + /// extensions with downstream, and finally fixates the caps and configures them on the source + /// pad. + fn negotiate(&self, src_caps: gst::Caps) { + assert!(src_caps.is_writable()); + self.parent_negotiate(src_caps); + } + + /// Called whenever a new buffer is available. + fn handle_buffer( + &self, + buffer: &gst::Buffer, + id: u64, + ) -> Result { + self.parent_handle_buffer(buffer, id) + } + + /// Called whenever a discontinuity or EOS is observed. + /// + /// The subclass should output any pending buffers it can output at this point. + /// + /// This will be followed by a call to [`Self::flush`]. + /// + /// Optional, by default drops all still pending buffers and forwards all still pending packets + /// with the last known timestamp. + fn drain(&self) -> Result { + self.parent_drain() + } + + /// Called on `FlushStop` or whenever all pending data should simply be discarded. + /// + /// The subclass should reset its internal state as necessary. + /// + /// Optional. + fn flush(&self) { + self.parent_flush() + } + + /// Called whenever a new event arrives on the sink pad. + /// + /// Optional, by default does the standard event handling of the base class. + fn sink_event(&self, event: gst::Event) -> Result { + self.parent_sink_event(event) + } + + /// Called whenever a new event arrives on the src pad. + /// + /// Optional, by default does the standard event handling of the base class. + fn src_event(&self, event: gst::Event) -> Result { + self.parent_src_event(event) + } + + /// Called whenever a new query arrives on the sink pad. + /// + /// Optional, by default does the standard query handling of the base class. + fn sink_query(&self, query: &mut gst::QueryRef) -> bool { + self.parent_sink_query(query) + } + + /// Called whenever a new query arrives on the src pad. + /// + /// Optional, by default does the standard query handling of the base class. + fn src_query(&self, query: &mut gst::QueryRef) -> bool { + self.parent_src_query(query) + } + + /// Called whenever a meta from an input buffer has to be copied to the output buffer. + /// + /// Optional, by default simply copies over all metas. + fn transform_meta( + &self, + in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ) { + self.parent_transform_meta(in_buf, meta, out_buf); + } +} + +/// Trait containing extension methods for `RtpBasePay2Impl`, specifically methods for chaining +/// up to the parent implementation of virtual methods. +pub trait RtpBasePay2ImplExt: RtpBasePay2Impl { + fn parent_set_sink_caps(&self, caps: &gst::Caps) -> bool { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.set_sink_caps)(self.obj().unsafe_cast_ref(), caps) + } + } + + fn parent_negotiate(&self, src_caps: gst::Caps) { + assert!(src_caps.is_writable()); + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.negotiate)(self.obj().unsafe_cast_ref(), src_caps); + } + } + + fn parent_start(&self) -> Result<(), gst::ErrorMessage> { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.start)(self.obj().unsafe_cast_ref()) + } + } + + fn parent_stop(&self) -> Result<(), gst::ErrorMessage> { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.stop)(self.obj().unsafe_cast_ref()) + } + } + + fn parent_handle_buffer( + &self, + buffer: &gst::Buffer, + id: u64, + ) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.handle_buffer)(self.obj().unsafe_cast_ref(), buffer, id) + } + } + + fn parent_drain(&self) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.drain)(self.obj().unsafe_cast_ref()) + } + } + + fn parent_flush(&self) { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.flush)(self.obj().unsafe_cast_ref()); + } + } + + fn parent_sink_event(&self, event: gst::Event) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.sink_event)(self.obj().unsafe_cast_ref(), event) + } + } + + fn parent_src_event(&self, event: gst::Event) -> Result { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.src_event)(self.obj().unsafe_cast_ref(), event) + } + } + + fn parent_sink_query(&self, query: &mut gst::QueryRef) -> bool { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.sink_query)(self.obj().unsafe_cast_ref(), query) + } + } + + fn parent_src_query(&self, query: &mut gst::QueryRef) -> bool { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.src_query)(self.obj().unsafe_cast_ref(), query) + } + } + + fn parent_transform_meta( + &self, + in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ) { + unsafe { + let data = Self::type_data(); + let parent_class = &*(data.as_ref().parent_class() as *mut Class); + (parent_class.transform_meta)(self.obj().unsafe_cast_ref(), in_buf, meta, out_buf) + } + } +} + +impl RtpBasePay2ImplExt for T {} + +/// Class struct for `RtpBasePay2`. +#[repr(C)] +pub struct Class { + parent: gst::ffi::GstElementClass, + + start: fn(&RtpBasePay2) -> Result<(), gst::ErrorMessage>, + stop: fn(&RtpBasePay2) -> Result<(), gst::ErrorMessage>, + + set_sink_caps: fn(&RtpBasePay2, caps: &gst::Caps) -> bool, + negotiate: fn(&RtpBasePay2, src_caps: gst::Caps), + handle_buffer: + fn(&RtpBasePay2, buffer: &gst::Buffer, id: u64) -> Result, + drain: fn(&RtpBasePay2) -> Result, + flush: fn(&RtpBasePay2), + + sink_event: fn(&RtpBasePay2, event: gst::Event) -> Result, + src_event: fn(&RtpBasePay2, event: gst::Event) -> Result, + + sink_query: fn(&RtpBasePay2, query: &mut gst::QueryRef) -> bool, + src_query: fn(&RtpBasePay2, query: &mut gst::QueryRef) -> bool, + + transform_meta: fn( + &RtpBasePay2, + in_buf: &gst::BufferRef, + meta: &gst::MetaRef, + out_buf: &mut gst::BufferRef, + ), + + allowed_meta_tags: &'static [&'static str], + drop_header_buffers: bool, +} + +unsafe impl ClassStruct for Class { + type Type = imp::RtpBasePay2; +} + +impl std::ops::Deref for Class { + type Target = glib::Class<<::Type as ObjectSubclass>::ParentType>; + + fn deref(&self) -> &Self::Target { + unsafe { &*(&self.parent as *const _ as *const _) } + } +} + +unsafe impl IsSubclassable for RtpBasePay2 { + fn class_init(class: &mut glib::Class) { + Self::parent_class_init::(class); + + let class = class.as_mut(); + + class.start = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.start() + }; + + class.stop = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.stop() + }; + + class.set_sink_caps = |obj, caps| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.set_sink_caps(caps) + }; + + class.negotiate = |obj, src_caps| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.negotiate(src_caps) + }; + + class.handle_buffer = |obj, buffer, id| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.handle_buffer(buffer, id) + }; + + class.drain = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.drain() + }; + + class.flush = |obj| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.flush() + }; + + class.sink_event = |obj, event| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.sink_event(event) + }; + + class.src_event = |obj, event| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.src_event(event) + }; + + class.sink_query = |obj, query| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.sink_query(query) + }; + + class.src_query = |obj, query| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.src_query(query) + }; + + class.transform_meta = |obj, in_buf, meta, out_buf| unsafe { + let imp = obj.unsafe_cast_ref::().imp(); + imp.transform_meta(in_buf, meta, out_buf) + }; + + class.allowed_meta_tags = T::ALLOWED_META_TAGS; + class.drop_header_buffers = T::DROP_HEADER_BUFFERS; + } +} + +/// Timestamp offset between this packet and the reference. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code)] +pub enum TimestampOffset { + /// Offset in nanoseconds relative to the first buffer id this packet belongs to. + Pts(gst::ClockTime), + /// Offset in RTP clock-time units relative to the last packet that had offset given in RTP + /// clock-rate units. + Rtp(u64), +} + +/// Relation between queued packet and input buffer ids. +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum PacketToBufferRelation { + Ids(RangeInclusive), + IdsWithOffset { + ids: RangeInclusive, + timestamp_offset: TimestampOffset, + }, + OutOfBand, +} + +impl From for PacketToBufferRelation { + fn from(id: u64) -> Self { + PacketToBufferRelation::Ids(id..=id) + } +} diff --git a/net/rtp/src/lib.rs b/net/rtp/src/lib.rs index a11aae9e..e04415c3 100644 --- a/net/rtp/src/lib.rs +++ b/net/rtp/src/lib.rs @@ -1,5 +1,6 @@ // // Copyright (C) 2022 Vivienne Watermeier +// Copyright (C) 2022-24 Sebastian Dröge // // This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. // If a copy of the MPL was not distributed with this file, You can obtain one at @@ -18,11 +19,36 @@ use gst::glib; mod av1; mod gcc; +mod audio_discont; +mod baseaudiopay; +mod basedepay; +mod basepay; + +mod pcmau; + +#[cfg(test)] +mod tests; + fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { av1::depay::register(plugin)?; av1::pay::register(plugin)?; gcc::register(plugin)?; + #[cfg(feature = "doc")] + { + use gst::prelude::*; + + crate::basepay::RtpBasePay2::static_type() // make base classes available in docs + .mark_as_plugin_api(gst::PluginAPIFlags::empty()); + crate::basedepay::RtpBaseDepay2::static_type() + .mark_as_plugin_api(gst::PluginAPIFlags::empty()); + crate::baseaudiopay::RtpBaseAudioPay2::static_type() + .mark_as_plugin_api(gst::PluginAPIFlags::empty()); + } + + pcmau::depay::register(plugin)?; + pcmau::pay::register(plugin)?; + Ok(()) } diff --git a/net/rtp/src/pcmau/depay/imp.rs b/net/rtp/src/pcmau/depay/imp.rs new file mode 100644 index 00000000..243a4d32 --- /dev/null +++ b/net/rtp/src/pcmau/depay/imp.rs @@ -0,0 +1,286 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use atomic_refcell::AtomicRefCell; +use gst::{glib, prelude::*, subclass::prelude::*}; + +use once_cell::sync::Lazy; + +use crate::basedepay::RtpBaseDepay2Ext; + +#[derive(Default)] +pub struct RtpPcmauDepay { + state: AtomicRefCell, +} + +#[derive(Default)] +struct State { + clock_rate: Option, +} + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtppcmaudepay2", + gst::DebugColorFlags::empty(), + Some("RTP PCMA/PCMU Depayloader"), + ) +}); + +#[glib::object_subclass] +impl ObjectSubclass for RtpPcmauDepay { + const NAME: &'static str = "GstRtpPcmauDepay2"; + type Type = super::RtpPcmauDepay; + type ParentType = crate::basedepay::RtpBaseDepay2; +} + +impl ObjectImpl for RtpPcmauDepay {} + +impl GstObjectImpl for RtpPcmauDepay {} + +impl ElementImpl for RtpPcmauDepay {} + +impl crate::basedepay::RtpBaseDepay2Impl for RtpPcmauDepay { + const ALLOWED_META_TAGS: &'static [&'static str] = &["audio"]; + + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + let s = caps.structure(0).unwrap(); + + let clock_rate = s.get::("clock-rate").unwrap_or(8000); + + let src_caps = gst::Caps::builder( + if self.obj().type_() == super::RtpPcmaDepay::static_type() { + "audio/x-alaw" + } else { + "audio/x-mulaw" + }, + ) + .field("channels", 1i32) + .field("rate", clock_rate) + .build(); + + self.state.borrow_mut().clock_rate = Some(clock_rate as u32); + + self.obj().set_src_caps(&src_caps); + + true + } + + fn handle_packet( + &self, + packet: &crate::basedepay::Packet, + ) -> Result { + let mut buffer = packet.payload_buffer(); + + let state = self.state.borrow(); + // Always set when caps are set + let clock_rate = state.clock_rate.unwrap(); + + let buffer_ref = buffer.get_mut().unwrap(); + buffer_ref.set_duration( + (buffer_ref.size() as u64) + .mul_div_floor(*gst::ClockTime::SECOND, clock_rate as u64) + .map(gst::ClockTime::from_nseconds), + ); + + // mark start of talkspurt with RESYNC + if packet.marker_bit() { + buffer_ref.set_flags(gst::BufferFlags::RESYNC); + } + + gst::trace!(CAT, imp: self, "Finishing buffer {buffer:?}"); + + self.obj().queue_buffer(packet.into(), buffer) + } +} + +/** + * SECTION:element-rtppcmadepay2 + * @see_also: rtppcmapay2, rtppcmupay2, rtppcmudepay2, alawenc, alawdec + * + * Extracts A-law encoded audio from RTP packets as per [RFC 3551][rfc-3551]. + * + * [rfc-3551]: https://www.rfc-editor.org/rfc/rfc3551.html#section-4.5.14 + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 udpsrc caps='application/x-rtp, media=audio, clock-rate=8000, payload=8' ! rtpjitterbuffer latency=50 ! rtppcmadepay2 ! alawdec ! audioconvert ! audioresample ! autoaudiosink + * ]| This will depayload an incoming RTP A-law audio stream. You can use the #rtppcmapay2 and + * alawenc elements to create such an RTP stream. + * + * Since: plugins-rs-0.13.0 + */ + +#[derive(Default)] +pub struct RtpPcmaDepay; + +#[glib::object_subclass] +impl ObjectSubclass for RtpPcmaDepay { + const NAME: &'static str = "GstRtpPcmaDepay2"; + type Type = super::RtpPcmaDepay; + type ParentType = super::RtpPcmauDepay; +} + +impl ObjectImpl for RtpPcmaDepay {} + +impl GstObjectImpl for RtpPcmaDepay {} + +impl ElementImpl for RtpPcmaDepay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP PCMA Depayloader", + "Codec/Depayloader/Network/RTP", + "Depayload A-law from RTP packets (RFC 3551)", + "Sebastian Dröge ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder_full() + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("payload", 8i32) + .field("clock-rate", 8000i32) + .build(), + ) + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("clock-rate", gst::IntRange::new(1i32, i32::MAX)) + .field("encoding-name", "PCMA") + .build(), + ) + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("audio/x-alaw") + .field("channels", 1i32) + .field("rate", gst::IntRange::new(1i32, i32::MAX)) + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl crate::basedepay::RtpBaseDepay2Impl for RtpPcmaDepay {} + +impl super::RtpPcmauDepayImpl for RtpPcmaDepay {} + +/** + * SECTION:element-rtppcmudepay2 + * @see_also: rtppcmupay2, rtppcmapay2, rtppcmadepay2, mulawenc, mulawdec + * + * Extracts µ-law encoded audio from RTP packets as per [RFC 3551][rfc-3551]. + * + * [rfc-3551]: https://www.rfc-editor.org/rfc/rfc3551.html#section-4.5.14 + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 udpsrc caps='application/x-rtp, media=audio, clock-rate=8000, payload=0' ! rtpjitterbuffer latency=50 ! rtppcmudepay2 ! mulawdec ! audioconvert ! audioresample ! autoaudiosink + * ]| This will depayload an incoming RTP µ-law audio stream. You can use the #rtppcmupay2 and + * mulawenc elements to create such an RTP stream. + * + * Since: plugins-rs-0.13.0 + */ + +#[derive(Default)] +pub struct RtpPcmuDepay; + +#[glib::object_subclass] +impl ObjectSubclass for RtpPcmuDepay { + const NAME: &'static str = "GstRtpPcmuDepay2"; + type Type = super::RtpPcmuDepay; + type ParentType = super::RtpPcmauDepay; +} + +impl ObjectImpl for RtpPcmuDepay {} + +impl GstObjectImpl for RtpPcmuDepay {} + +impl ElementImpl for RtpPcmuDepay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP PCMU Depayloader", + "Codec/Depayloader/Network/RTP", + "Depayload µ-law from RTP packets (RFC 3551)", + "Sebastian Dröge ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder_full() + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("payload", 0i32) + .field("clock-rate", 8000i32) + .build(), + ) + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("clock-rate", gst::IntRange::new(1i32, i32::MAX)) + .field("encoding-name", "PCMU") + .build(), + ) + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("audio/x-mulaw") + .field("channels", 1i32) + .field("rate", gst::IntRange::new(1i32, i32::MAX)) + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl crate::basedepay::RtpBaseDepay2Impl for RtpPcmuDepay {} + +impl super::RtpPcmauDepayImpl for RtpPcmuDepay {} diff --git a/net/rtp/src/pcmau/depay/mod.rs b/net/rtp/src/pcmau/depay/mod.rs new file mode 100644 index 00000000..b56058e3 --- /dev/null +++ b/net/rtp/src/pcmau/depay/mod.rs @@ -0,0 +1,59 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::{glib, prelude::*, subclass::prelude::*}; + +pub mod imp; + +glib::wrapper! { + pub struct RtpPcmauDepay(ObjectSubclass) + @extends crate::basedepay::RtpBaseDepay2, gst::Element, gst::Object; +} + +pub trait RtpPcmauDepayImpl: crate::basedepay::RtpBaseDepay2Impl {} + +unsafe impl IsSubclassable for RtpPcmauDepay { + fn class_init(class: &mut glib::Class) { + Self::parent_class_init::(class); + } +} + +glib::wrapper! { + pub struct RtpPcmaDepay(ObjectSubclass) + @extends RtpPcmauDepay, crate::basedepay::RtpBaseDepay2, gst::Element, gst::Object; +} + +glib::wrapper! { + pub struct RtpPcmuDepay(ObjectSubclass) + @extends RtpPcmauDepay, crate::basedepay::RtpBaseDepay2, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + #[cfg(feature = "doc")] + { + use gst::prelude::*; + + // Make internal base class available in docs + crate::pcmau::depay::RtpPcmauDepay::static_type() + .mark_as_plugin_api(gst::PluginAPIFlags::empty()); + } + + gst::Element::register( + Some(plugin), + "rtppcmadepay2", + gst::Rank::MARGINAL, + RtpPcmaDepay::static_type(), + )?; + gst::Element::register( + Some(plugin), + "rtppcmudepay2", + gst::Rank::MARGINAL, + RtpPcmuDepay::static_type(), + ) +} diff --git a/net/rtp/src/pcmau/mod.rs b/net/rtp/src/pcmau/mod.rs new file mode 100644 index 00000000..ee965e87 --- /dev/null +++ b/net/rtp/src/pcmau/mod.rs @@ -0,0 +1,14 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +pub mod depay; +pub mod pay; + +#[cfg(test)] +mod tests; diff --git a/net/rtp/src/pcmau/pay/imp.rs b/net/rtp/src/pcmau/pay/imp.rs new file mode 100644 index 00000000..50a0768f --- /dev/null +++ b/net/rtp/src/pcmau/pay/imp.rs @@ -0,0 +1,294 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::{glib, prelude::*, subclass::prelude::*}; + +use once_cell::sync::Lazy; + +use crate::{ + baseaudiopay::RtpBaseAudioPay2Ext, + basepay::{RtpBasePay2Ext, RtpBasePay2ImplExt}, +}; + +#[derive(Default)] +pub struct RtpPcmauPay; + +#[glib::object_subclass] +impl ObjectSubclass for RtpPcmauPay { + const ABSTRACT: bool = true; + const NAME: &'static str = "GstRtpPcmauPay2"; + type Type = super::RtpPcmauPay; + type ParentType = crate::baseaudiopay::RtpBaseAudioPay2; +} + +impl ObjectImpl for RtpPcmauPay {} + +impl GstObjectImpl for RtpPcmauPay {} + +impl ElementImpl for RtpPcmauPay {} + +impl crate::basepay::RtpBasePay2Impl for RtpPcmauPay { + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + let s = caps.structure(0).unwrap(); + + let rate = u32::try_from(s.get::("rate").unwrap()).unwrap(); + + let src_caps = gst::Caps::builder("application/x-rtp") + .field("media", "audio") + .field( + "encoding-name", + if self.obj().type_() == super::RtpPcmaPay::static_type() { + "PCMA" + } else { + "PCMU" + }, + ) + .field("clock-rate", rate as i32) + .build(); + + self.obj().set_src_caps(&src_caps); + self.obj().set_bpf(1); + + true + } + + #[allow(clippy::single_match)] + fn sink_query(&self, query: &mut gst::QueryRef) -> bool { + match query.view_mut() { + gst::QueryViewMut::Caps(query) => { + let mut caps = self.obj().sink_pad().pad_template_caps(); + + // If the payload type is 0 or 8 then only 8000Hz are supported. + if [0, 8].contains(&self.obj().property::("pt")) { + let caps = caps.make_mut(); + caps.set("rate", 8000); + } + + if let Some(filter) = query.filter() { + caps = filter.intersect_with_mode(&caps, gst::CapsIntersectMode::First); + } + + query.set_result(&caps); + + return true; + } + _ => (), + } + + self.parent_sink_query(query) + } +} + +impl crate::baseaudiopay::RtpBaseAudioPay2Impl for RtpPcmauPay {} + +/** + * SECTION:element-rtppcmapay2 + * @see_also: rtppcmadepay2, rtppcmupay2, rtppcmudepay2, alawenc, alawdec + * + * Payloads A-law encoded audio into RTP packets as per [RFC 3551][rfc-3551]. + * + * [rfc-3551]: https://www.rfc-editor.org/rfc/rfc3551.html#section-4.5.10 + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 audiotestsrc wave=ticks ! audio/x-raw,rate=8000,channels=1 ! alawenc ! rtppcmapay2 ! udpsink host=127.0.0.1 port=5004 + * ]| This will generate an A-law audio test signal and payload it as RTP and send it out + * as UDP to localhost port 5004. + * + * Since: plugins-rs-0.13.0 + */ + +#[derive(Default)] +pub struct RtpPcmaPay; + +#[glib::object_subclass] +impl ObjectSubclass for RtpPcmaPay { + const NAME: &'static str = "GstRtpPcmaPay2"; + type Type = super::RtpPcmaPay; + type ParentType = super::RtpPcmauPay; +} + +impl ObjectImpl for RtpPcmaPay { + fn constructed(&self) { + self.parent_constructed(); + + // Default to payload type 8 + self.obj().set_property("pt", 8u32); + } +} + +impl GstObjectImpl for RtpPcmaPay {} + +impl ElementImpl for RtpPcmaPay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP PCMA Payloader", + "Codec/Payloader/Network/RTP", + "Payload A-law Audio into RTP packets (RFC 3551)", + "Sebastian Dröge ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder("audio/x-alaw") + .field("channels", 1i32) + .field("rate", gst::IntRange::new(1i32, i32::MAX)) + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder_full() + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("payload", 8i32) + .field("clock-rate", 8000i32) + .build(), + ) + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("payload", gst::IntRange::new(96i32, 127i32)) + .field("encoding-name", "PCMA") + .field("clock-rate", gst::IntRange::new(1, i32::MAX)) + .build(), + ) + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl crate::basepay::RtpBasePay2Impl for RtpPcmaPay {} + +impl crate::baseaudiopay::RtpBaseAudioPay2Impl for RtpPcmaPay {} + +impl super::RtpPcmauPayImpl for RtpPcmaPay {} + +/** + * SECTION:element-rtppcmupay2 + * @see_also: rtppcmudepay2, rtppcmapay2, rtppcmadepay2, mulawenc, mulawdec + * + * Payloads µ-law encoded audio into RTP packets as per [RFC 3551][rfc-3551]. + * + * [rfc-3551]: https://www.rfc-editor.org/rfc/rfc3551.html#section-4.5.10 + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 audiotestsrc wave=ticks ! audio/x-raw,rate=8000,channels=1 ! mulawenc ! rtppcmupay2 ! udpsink host=127.0.0.1 port=5004 + * ]| This will generate a µ-law audio test signal and payload it as RTP and send it out + * as UDP to localhost port 5004. + * + * Since: plugins-rs-0.13.0 + */ + +#[derive(Default)] +pub struct RtpPcmuPay; + +#[glib::object_subclass] +impl ObjectSubclass for RtpPcmuPay { + const NAME: &'static str = "GstRtpPcmuPay2"; + type Type = super::RtpPcmuPay; + type ParentType = super::RtpPcmauPay; +} + +impl ObjectImpl for RtpPcmuPay { + fn constructed(&self) { + self.parent_constructed(); + + // Default to payload type 0 + self.obj().set_property("pt", 0u32); + } +} + +impl GstObjectImpl for RtpPcmuPay {} + +impl ElementImpl for RtpPcmuPay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP PCMU Payloader", + "Codec/Payloader/Network/RTP", + "Payload µ-law Audio into RTP packets (RFC 3551)", + "Sebastian Dröge ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder("audio/x-mulaw") + .field("channels", 1i32) + .field("rate", gst::IntRange::new(1i32, i32::MAX)) + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder_full() + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("payload", 0i32) + .field("clock-rate", 8000i32) + .build(), + ) + .structure( + gst::Structure::builder("application/x-rtp") + .field("media", "audio") + .field("payload", gst::IntRange::new(96i32, 127i32)) + .field("encoding-name", "PCMU") + .field("clock-rate", gst::IntRange::new(1, i32::MAX)) + .build(), + ) + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl crate::basepay::RtpBasePay2Impl for RtpPcmuPay {} + +impl crate::baseaudiopay::RtpBaseAudioPay2Impl for RtpPcmuPay {} + +impl super::RtpPcmauPayImpl for RtpPcmuPay {} diff --git a/net/rtp/src/pcmau/pay/mod.rs b/net/rtp/src/pcmau/pay/mod.rs new file mode 100644 index 00000000..d4097112 --- /dev/null +++ b/net/rtp/src/pcmau/pay/mod.rs @@ -0,0 +1,59 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::{glib, prelude::*, subclass::prelude::*}; + +pub mod imp; + +glib::wrapper! { + pub struct RtpPcmauPay(ObjectSubclass) + @extends crate::baseaudiopay::RtpBaseAudioPay2, crate::basepay::RtpBasePay2, gst::Element, gst::Object; +} + +pub trait RtpPcmauPayImpl: crate::baseaudiopay::RtpBaseAudioPay2Impl {} + +unsafe impl IsSubclassable for RtpPcmauPay { + fn class_init(class: &mut glib::Class) { + Self::parent_class_init::(class); + } +} + +glib::wrapper! { + pub struct RtpPcmaPay(ObjectSubclass) + @extends RtpPcmauPay, crate::baseaudiopay::RtpBaseAudioPay2, crate::basepay::RtpBasePay2, gst::Element, gst::Object; +} + +glib::wrapper! { + pub struct RtpPcmuPay(ObjectSubclass) + @extends RtpPcmauPay, crate::baseaudiopay::RtpBaseAudioPay2, crate::basepay::RtpBasePay2, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + #[cfg(feature = "doc")] + { + use gst::prelude::*; + + // Make internal base class available in docs + crate::pcmau::pay::RtpPcmauPay::static_type() + .mark_as_plugin_api(gst::PluginAPIFlags::empty()); + } + + gst::Element::register( + Some(plugin), + "rtppcmapay2", + gst::Rank::MARGINAL, + RtpPcmaPay::static_type(), + )?; + gst::Element::register( + Some(plugin), + "rtppcmupay2", + gst::Rank::MARGINAL, + RtpPcmuPay::static_type(), + ) +} diff --git a/net/rtp/src/pcmau/tests.rs b/net/rtp/src/pcmau/tests.rs new file mode 100644 index 00000000..a3291b96 --- /dev/null +++ b/net/rtp/src/pcmau/tests.rs @@ -0,0 +1,239 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use crate::tests::{run_test_pipeline, ExpectedBuffer, ExpectedPacket, Source}; + +use std::cmp; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + crate::plugin_register_static().expect("rtppcmau test"); + }); +} + +#[test] +fn test_pcma() { + init(); + + let src = "audiotestsrc num-buffers=100 samplesperbuffer=400 ! audio/x-raw,rate=8000,channels=1 ! alawenc"; + let pay = "rtppcmapay2"; + let depay = "rtppcmadepay2"; + + let mut expected_pay = Vec::with_capacity(100); + for i in 0..100 { + expected_pay.push(vec![ExpectedPacket::builder() + .pts(gst::ClockTime::from_mseconds(i * 50)) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else { + gst::BufferFlags::empty() + }) + .pt(8) + .rtp_time(((i * 400) & 0xffff_ffff) as u32) + .marker_bit(i == 0) + .build()]); + } + + let mut expected_depay = Vec::with_capacity(100); + for i in 0..100 { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(gst::ClockTime::from_mseconds(i * 50)) + .size(400) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::RESYNC + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} + +#[test] +fn test_pcma_splitting() { + init(); + + let src = "audiotestsrc num-buffers=100 samplesperbuffer=480 ! audio/x-raw,rate=8000,channels=1 ! alawenc"; + let pay = "rtppcmapay2 min-ptime=25000000 max-ptime=50000000"; + let depay = "rtppcmadepay2"; + + // Every input buffer is 480 samples, every packet can contain between 200 and 400 samples + // so a bit of splitting is necessary and sometimes the remaining queued data ends up filling + // one additional packet + let mut expected_pay = Vec::with_capacity(134); + let mut queued = 0; + let mut pos = 0; + for i in 0..134 { + if i < 100 { + queued += 480; + } + + while (i < 100 && queued >= 200) || (i == 100 && queued > 0) { + let size = cmp::min(queued, 400); + queued -= size; + + expected_pay.push(vec![ExpectedPacket::builder() + .pts(gst::ClockTime::from_mseconds(pos / 8)) + .size(size as usize + 12) + .flags(if pos == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else { + gst::BufferFlags::empty() + }) + .pt(8) + .rtp_time((pos & 0xffff_ffff) as u32) + .marker_bit(pos == 0) + .build()]); + + pos += size; + } + } + + let mut expected_depay = Vec::with_capacity(134); + for packets in &expected_pay { + for packet in packets { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(packet.pts) + .maybe_size(packet.size.map(|size| size - 12)) + .flags(if packet.pts.is_zero() { + gst::BufferFlags::DISCONT | gst::BufferFlags::RESYNC + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} + +#[test] +fn test_pcma_discont() { + init(); + + let caps = gst::Caps::builder("audio/x-alaw") + .field("channels", 1) + .field("rate", 8000i32) + .build(); + + let mut buffers = Vec::with_capacity(10); + let mut pos = 0; + // First 5 buffers are normal, then a 10s jump + for _ in 0..10 { + let mut buffer = gst::Buffer::with_size(400).unwrap(); + { + let buffer = buffer.get_mut().unwrap(); + buffer.set_pts(gst::ClockTime::from_mseconds(pos / 8)); + } + + buffers.push(buffer); + + pos += 400; + if pos == 2000 { + pos += 80000; + } + } + + let pay = "rtppcmapay2 discont-wait=25000000"; + let depay = "rtppcmadepay2"; + + let mut expected_pay = Vec::with_capacity(10); + let mut pos = 0; + for _ in 0..10 { + expected_pay.push(vec![ExpectedPacket::builder() + .pts(gst::ClockTime::from_mseconds(pos / 8)) + .size(412) + .flags(if pos == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else if pos == 82000 { + gst::BufferFlags::MARKER + } else { + gst::BufferFlags::empty() + }) + .pt(8) + .rtp_time((pos & 0xffff_ffff) as u32) + .marker_bit(pos == 0 || pos == 82000) + .build()]); + + pos += 400; + if pos == 2000 { + pos += 80000; + } + } + + let mut expected_depay = Vec::with_capacity(10); + for packets in &expected_pay { + for packet in packets { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(packet.pts) + .maybe_size(packet.size.map(|size| size - 12)) + .flags(if packet.pts.is_zero() { + gst::BufferFlags::DISCONT | gst::BufferFlags::RESYNC + } else if packet.flags.contains(gst::BufferFlags::MARKER) { + gst::BufferFlags::RESYNC + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + } + + run_test_pipeline( + Source::Buffers(caps, buffers), + pay, + depay, + expected_pay, + expected_depay, + ); +} + +#[test] +fn test_pcmu() { + init(); + + let src = "audiotestsrc num-buffers=100 samplesperbuffer=400 ! audio/x-raw,rate=8000,channels=1 ! mulawenc"; + let pay = "rtppcmupay2"; + let depay = "rtppcmudepay2"; + + let mut expected_pay = Vec::with_capacity(100); + for i in 0..100 { + expected_pay.push(vec![ExpectedPacket::builder() + .pts(gst::ClockTime::from_mseconds(i * 50)) + .size(412) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else { + gst::BufferFlags::empty() + }) + .pt(0) + .rtp_time(((i * 400) & 0xffff_ffff) as u32) + .marker_bit(i == 0) + .build()]); + } + + let mut expected_depay = Vec::with_capacity(100); + for i in 0..100 { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(gst::ClockTime::from_mseconds(i * 50)) + .size(400) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::RESYNC + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} diff --git a/net/rtp/src/tests.rs b/net/rtp/src/tests.rs new file mode 100644 index 00000000..931dddcf --- /dev/null +++ b/net/rtp/src/tests.rs @@ -0,0 +1,515 @@ +// +// Copyright (C) 2023 Sebastian Dröge +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use std::{ + mem, + sync::{Arc, Mutex}, +}; + +use gst::prelude::*; + +/// Expected packet produced by the payloader +pub struct ExpectedPacket { + /// All packets are expected to have a known and fixed PTS. + pub pts: gst::ClockTime, + /// If not set the size will not be checked. + pub size: Option, + pub flags: gst::BufferFlags, + pub pt: u8, + pub rtp_time: u32, + pub marker: bool, +} + +impl ExpectedPacket { + /// Creates a builder for an `ExpectedPacket`. + /// + /// Assigns the following packet default values: + /// + /// * pts: gst::ClockTime::ZERO + /// * size: None => not checked + /// * flags: gst::BufferFlags::empty() + /// * pt: 96 + /// * rtp_time: 0 + /// * marker: true + pub fn builder() -> ExpectedPacketBuilder { + ExpectedPacketBuilder(ExpectedPacket { + pts: gst::ClockTime::ZERO, + size: None, + flags: gst::BufferFlags::empty(), + pt: 96, + rtp_time: 0, + marker: true, + }) + } +} + +pub struct ExpectedPacketBuilder(ExpectedPacket); +impl ExpectedPacketBuilder { + pub fn pts(mut self, pts: gst::ClockTime) -> Self { + self.0.pts = pts; + self + } + + pub fn size(mut self, size: usize) -> Self { + self.0.size = Some(size); + self + } + + pub fn flags(mut self, flags: gst::BufferFlags) -> Self { + self.0.flags = flags; + self + } + + pub fn pt(mut self, pt: u8) -> Self { + self.0.pt = pt; + self + } + + pub fn rtp_time(mut self, rtp_time: u32) -> Self { + self.0.rtp_time = rtp_time; + self + } + + pub fn marker_bit(mut self, marker: bool) -> Self { + self.0.marker = marker; + self + } + + pub fn build(self) -> ExpectedPacket { + self.0 + } +} + +/// Expected buffer produced by the depayloader +#[derive(Debug)] +pub struct ExpectedBuffer { + /// If not set then it is asserted that the depayloaded buffer also has no PTS. + pub pts: Option, + /// If not set then it is asserted that the depayloaded buffer also has no DTS. + pub dts: Option, + /// If not set the size will not be checked. + pub size: Option, + pub flags: gst::BufferFlags, +} + +impl ExpectedBuffer { + /// Creates a builder for an `ExpectedBuffer`. + /// + /// Assigns the following buffer default values: + /// + /// * pts: None + /// * dts: None + /// * size: None => not checked + /// * flags: gst::BufferFlags::empty() + pub fn builder() -> ExpectedBufferBuilder { + ExpectedBufferBuilder(ExpectedBuffer { + pts: None, + dts: None, + size: None, + flags: gst::BufferFlags::empty(), + }) + } +} + +pub struct ExpectedBufferBuilder(ExpectedBuffer); +#[allow(dead_code)] +impl ExpectedBufferBuilder { + pub fn pts(mut self, pts: gst::ClockTime) -> Self { + self.0.pts = Some(pts); + self + } + + pub fn maybe_pts(mut self, pts: Option) -> Self { + self.0.pts = pts; + self + } + + pub fn dts(mut self, dts: gst::ClockTime) -> Self { + self.0.dts = Some(dts); + self + } + + pub fn size(mut self, size: usize) -> Self { + self.0.size = Some(size); + self + } + + pub fn maybe_size(mut self, size: Option) -> Self { + self.0.size = size; + self + } + + pub fn flags(mut self, flags: gst::BufferFlags) -> Self { + self.0.flags = flags; + self + } + + pub fn build(self) -> ExpectedBuffer { + self.0 + } +} + +/// Source of the test +pub enum Source<'a> { + #[allow(dead_code)] + Buffers(gst::Caps, Vec), + Bin(&'a str), +} + +/// Pipeline wrapper to automatically set state to `Null` on drop +/// +/// Useful to not get critical warnings on panics from unwinding. +struct Pipeline(gst::Pipeline); + +impl Drop for Pipeline { + fn drop(&mut self) { + let _ = self.0.set_state(gst::State::Null); + } +} + +impl std::ops::Deref for Pipeline { + type Target = gst::Pipeline; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +pub fn run_test_pipeline( + src: Source, + pay: &str, + depay: &str, + expected_pay: Vec>, + expected_depay: Vec>, +) { + let pipeline = Pipeline(gst::Pipeline::new()); + + // Return if the pipelines can't be built: this likely means that encoders are missing + + let src = match src { + Source::Bin(src) => { + let Ok(src) = gst::parse::bin_from_description_with_name(src, true, "rtptestsrc") + else { + return; + }; + + src.upcast::() + } + Source::Buffers(caps, buffers) => { + let mut buffers = buffers.into_iter(); + let appsrc = gst_app::AppSrc::builder() + .format(gst::Format::Time) + .caps(&caps) + .callbacks( + gst_app::AppSrcCallbacks::builder() + .need_data(move |appsrc, _offset| { + let Some(buffer) = buffers.next() else { + let _ = appsrc.end_of_stream(); + return; + }; + + // appsrc already handles the error for us + let _ = appsrc.push_buffer(buffer); + }) + .build(), + ) + .build(); + + appsrc.upcast::() + } + }; + + let Ok(pay) = gst::parse::bin_from_description_with_name(pay, true, "rtptestpay") else { + return; + }; + let pay = pay.upcast::(); + + // Collect samples from after the payloader + let pay_samples = Arc::new(Mutex::new(Vec::new())); + pay.static_pad("src") + .unwrap() + .add_probe( + gst::PadProbeType::BUFFER | gst::PadProbeType::BUFFER_LIST, + { + let pay_samples = pay_samples.clone(); + move |pad, info| { + let segment_event = pad.sticky_event::(0).unwrap(); + let segment = segment_event.segment().clone(); + + let caps = pad.current_caps().unwrap(); + + let mut sample_builder = gst::Sample::builder().segment(&segment).caps(&caps); + if let Some(buffer) = info.buffer() { + sample_builder = sample_builder.buffer(buffer); + } else if let Some(list) = info.buffer_list() { + sample_builder = sample_builder.buffer_list(list); + } else { + unreachable!(); + } + + pay_samples.lock().unwrap().push(sample_builder.build()); + + gst::PadProbeReturn::Ok + } + }, + ) + .unwrap(); + + let Ok(depay) = gst::parse::bin_from_description_with_name(depay, true, "rtptestdepay") else { + return; + }; + let depay = depay.upcast::(); + + let depay_samples = Arc::new(Mutex::new(Vec::new())); + let appsink = gst_app::AppSink::builder() + .sync(false) + .buffer_list(true) + .callbacks( + gst_app::AppSinkCallbacks::builder() + .new_sample({ + let depay_samples = depay_samples.clone(); + move |appsink| { + let Ok(sample) = appsink.pull_sample() else { + return Err(gst::FlowError::Flushing); + }; + + depay_samples.lock().unwrap().push(sample); + + Ok(gst::FlowSuccess::Ok) + } + }) + .build(), + ) + .build(); + + pipeline + .add_many([&src, &pay, &depay, appsink.as_ref()]) + .unwrap(); + gst::Element::link_many([&src, &pay, &depay, appsink.as_ref()]).unwrap(); + + pipeline + .set_state(gst::State::Playing) + .expect("Failed to set pipeline to Playing"); + + let msg = pipeline + .bus() + .unwrap() + .timed_pop_filtered( + gst::ClockTime::NONE, + &[gst::MessageType::Error, gst::MessageType::Eos], + ) + .expect("Didn't receive ERROR or EOS message"); + + assert_ne!( + msg.type_(), + gst::MessageType::Error, + "Received error message {msg:?}" + ); + + pipeline + .set_state(gst::State::Null) + .expect("Failed to set pipeline to Null"); + + drop(msg); + drop(src); + drop(pay); + drop(depay); + drop(appsink); + drop(pipeline); + + let pay_samples = mem::take(&mut *pay_samples.lock().unwrap()); + let depay_samples = mem::take(&mut *depay_samples.lock().unwrap()); + + // Now check against the expected values + + assert_eq!( + pay_samples.len(), + expected_pay.len(), + "Expected {} payload packets but got {}", + expected_pay.len(), + pay_samples.len() + ); + + let mut initial_timestamp = None; + + for (i, (expected_list, sample)) in + Iterator::zip(expected_pay.into_iter(), pay_samples.into_iter()).enumerate() + { + let mut iter_a; + let mut iter_b; + + let buffer_iter: &mut dyn Iterator = + if let Some(list) = sample.buffer_list() { + assert_eq!( + list.len(), + expected_list.len(), + "Expected {} buffers in {}-th payload list but got {}", + expected_list.len(), + i, + list.len() + ); + iter_a = list.iter(); + &mut iter_a + } else { + let buffer = sample.buffer().unwrap(); + assert_eq!( + expected_list.len(), + 1, + "Expected {} buffers in {}-th payload list but got 1", + expected_list.len(), + i, + ); + iter_b = Some(buffer).into_iter(); + &mut iter_b + }; + + for (j, (expected_buffer, buffer)) in + Iterator::zip(expected_list.into_iter(), buffer_iter).enumerate() + { + let buffer_pts = buffer.pts().expect("Buffer without PTS"); + assert_eq!( + buffer_pts, expected_buffer.pts, + "Buffer {} of payload buffer list {} has unexpected PTS {} instead of {}", + j, i, buffer_pts, expected_buffer.pts, + ); + + if let Some(expected_size) = expected_buffer.size { + assert_eq!( + buffer.size(), + expected_size, + "Buffer {} of payload buffer list {} has unexpected size {} instead of {}", + j, + i, + buffer.size(), + expected_size, + ); + } + + let buffer_flags = buffer.flags() - gst::BufferFlags::TAG_MEMORY; + assert_eq!( + buffer_flags, expected_buffer.flags, + "Buffer {} of payload buffer list {} has unexpected flags {:?} instead of {:?}", + j, i, buffer_flags, expected_buffer.flags, + ); + + let map = buffer.map_readable().unwrap(); + let rtp_packet = rtp_types::RtpPacket::parse(&map).expect("Invalid RTP packet"); + assert_eq!( + rtp_packet.payload_type(), + expected_buffer.pt, + "Buffer {} of payload buffer list {} has unexpected payload type {:?} instead of {:?}", + j, + i, + rtp_packet.payload_type(), + expected_buffer.pt, + ); + + assert_eq!( + rtp_packet.marker_bit(), + expected_buffer.marker, + "Buffer {} of payload buffer list {} has unexpected marker {:?} instead of {:?}", + j, + i, + rtp_packet.marker_bit(), + expected_buffer.marker, + ); + + if initial_timestamp.is_none() { + initial_timestamp = Some(rtp_packet.timestamp()); + } + let initial_timestamp = initial_timestamp.unwrap(); + + let expected_timestamp = expected_buffer.rtp_time.wrapping_add(initial_timestamp); + + assert_eq!( + rtp_packet.timestamp(), + expected_timestamp, + "Buffer {} of payload buffer list {} has unexpected RTP timestamp {:?} instead of {:?}", + j, + i, + rtp_packet.timestamp(), + expected_timestamp, + ); + } + } + + assert_eq!( + depay_samples.len(), + expected_depay.len(), + "Expected {} depayload samples but got {}", + expected_depay.len(), + depay_samples.len() + ); + + for (i, (expected_list, sample)) in + Iterator::zip(expected_depay.into_iter(), depay_samples.into_iter()).enumerate() + { + let mut iter_a; + let mut iter_b; + + let buffer_iter: &mut dyn Iterator = + if let Some(list) = sample.buffer_list() { + assert_eq!( + list.len(), + expected_list.len(), + "Expected {} depayload buffers in {}-th list but got {}", + expected_list.len(), + i, + list.len() + ); + iter_a = list.iter(); + &mut iter_a + } else { + let buffer = sample.buffer().unwrap(); + assert_eq!( + expected_list.len(), + 1, + "Expected {} depayload buffers in {}-th list but got 1", + expected_list.len(), + i, + ); + iter_b = Some(buffer).into_iter(); + &mut iter_b + }; + + for (j, (expected_buffer, buffer)) in + Iterator::zip(expected_list.into_iter(), buffer_iter).enumerate() + { + let buffer_pts = buffer.pts(); + assert_eq!( + buffer_pts, + expected_buffer.pts, + "Buffer {} of depayload buffer list {} has unexpected PTS {} instead of {}", + j, + i, + buffer_pts.display(), + expected_buffer.pts.display(), + ); + + if let Some(expected_size) = expected_buffer.size { + assert_eq!( + buffer.size(), + expected_size, + "Buffer {} of depayload buffer list {} has unexpected size {} instead of {}", + j, + i, + buffer.size(), + expected_size, + ); + } + + let buffer_flags = buffer.flags() - gst::BufferFlags::TAG_MEMORY; + assert_eq!( + buffer_flags, expected_buffer.flags, + "Buffer {} of depayload buffer list {} has unexpected flags {:?} instead of {:?}", + j, i, buffer_flags, expected_buffer.flags, + ); + } + } +}