From 1de7754616709f624a51c32a71c3ccf0a02450e5 Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Sat, 9 Sep 2023 21:39:26 +0900
Subject: [PATCH] webrtcsink: Add support for d3d11 memory and qsvh264enc

Adding d3d11 memory and qsvh264enc support

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1322>
---
 docs/plugins/gst_plugins_cache.json |  8 ++++----
 net/webrtc/src/webrtcsink/imp.rs    | 24 ++++++++++++++++++++++--
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json
index bb6efb53..b4d508ba 100644
--- a/docs/plugins/gst_plugins_cache.json
+++ b/docs/plugins/gst_plugins_cache.json
@@ -6199,7 +6199,7 @@
                         "presence": "request"
                     },
                     "video_%%u": {
-                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
+                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\n\nvideo/x-raw(memory:D3D11Memory):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
                         "direction": "sink",
                         "presence": "request"
                     }
@@ -6230,7 +6230,7 @@
                         "presence": "request"
                     },
                     "video_%%u": {
-                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
+                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\n\nvideo/x-raw(memory:D3D11Memory):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
                         "direction": "sink",
                         "presence": "request"
                     }
@@ -6261,7 +6261,7 @@
                         "presence": "request"
                     },
                     "video_%%u": {
-                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
+                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\n\nvideo/x-raw(memory:D3D11Memory):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
                         "direction": "sink",
                         "presence": "request"
                     }
@@ -6415,7 +6415,7 @@
                         "presence": "request"
                     },
                     "video_%%u": {
-                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
+                        "caps": "video/x-raw:\n\nvideo/x-raw(memory:CUDAMemory):\n\nvideo/x-raw(memory:GLMemory):\n\nvideo/x-raw(memory:NVMM):\n\nvideo/x-raw(memory:D3D11Memory):\nvideo/x-vp8:\nvideo/x-h264:\nvideo/x-vp9:\nvideo/x-h265:\n",
                         "direction": "sink",
                         "presence": "request"
                     }
diff --git a/net/webrtc/src/webrtcsink/imp.rs b/net/webrtc/src/webrtcsink/imp.rs
index 0e3dad74..45c68d92 100644
--- a/net/webrtc/src/webrtcsink/imp.rs
+++ b/net/webrtc/src/webrtcsink/imp.rs
@@ -39,6 +39,7 @@ static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
 const CUDA_MEMORY_FEATURE: &str = "memory:CUDAMemory";
 const GL_MEMORY_FEATURE: &str = "memory:GLMemory";
 const NVMM_MEMORY_FEATURE: &str = "memory:NVMM";
+const D3D11_MEMORY_FEATURE: &str = "memory:D3D11Memory";
 
 const RTP_TWCC_URI: &str =
     "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01";
@@ -372,6 +373,14 @@ fn make_converter_for_video_caps(caps: &gst::Caps, codec: &Codec) -> Result<gst:
                 gst::Element::link_many([&queue, &nvconvert])?;
 
                 (queue, nvconvert)
+            } else if feature.contains(D3D11_MEMORY_FEATURE) {
+                let d3d11upload = make_element("d3d11upload", None)?;
+                let d3d11convert = make_element("d3d11convert", None)?;
+
+                ret.add_many([&d3d11upload, &d3d11convert])?;
+                d3d11upload.link(&d3d11convert)?;
+
+                (d3d11upload, d3d11convert)
             } else if feature.contains(CUDA_MEMORY_FEATURE) {
                 if let Some(convert_factory) = gst::ElementFactory::find("cudaconvert") {
                     let cudaupload = make_element("cudaupload", None)?;
@@ -537,6 +546,13 @@ fn configure_encoder(enc: &gst::Element, start_bitrate: u32) {
                 enc.set_property_from_str("control-rate", "constant_bitrate");
                 add_nv4l2enc_force_keyunit_workaround(enc);
             }
+            "qsvh264enc" => {
+                enc.set_property("bitrate", start_bitrate / 1000);
+                enc.set_property("gop-size", 2560u32);
+                enc.set_property("low-latency", true);
+                enc.set_property("disable-hrd-conformance", true);
+                enc.set_property_from_str("rate-control", "cbr");
+            }
             _ => (),
         }
     }
@@ -730,7 +746,7 @@ impl VideoEncoder {
     fn bitrate(&self) -> i32 {
         match self.factory_name.as_str() {
             "vp8enc" | "vp9enc" => self.element.property::<i32>("target-bitrate"),
-            "x264enc" | "nvh264enc" | "vaapih264enc" | "vaapivp8enc" => {
+            "x264enc" | "nvh264enc" | "vaapih264enc" | "vaapivp8enc" | "qsvh264enc" => {
                 (self.element.property::<u32>("bitrate") * 1000) as i32
             }
             "nvv4l2h264enc" | "nvv4l2vp8enc" | "nvv4l2vp9enc" => {
@@ -757,7 +773,7 @@ impl VideoEncoder {
     pub(crate) fn set_bitrate(&mut self, element: &super::BaseWebRTCSink, bitrate: i32) {
         match self.factory_name.as_str() {
             "vp8enc" | "vp9enc" => self.element.set_property("target-bitrate", bitrate),
-            "x264enc" | "nvh264enc" | "vaapih264enc" | "vaapivp8enc" => self
+            "x264enc" | "nvh264enc" | "vaapih264enc" | "vaapivp8enc" | "qsvh264enc" => self
                 .element
                 .set_property("bitrate", (bitrate / 1000) as u32),
             "nvv4l2h264enc" | "nvv4l2vp8enc" | "nvv4l2vp9enc" => {
@@ -3603,6 +3619,10 @@ impl ElementImpl for BaseWebRTCSink {
                 .structure_with_features(
                     gst::Structure::builder("video/x-raw").build(),
                     gst::CapsFeatures::new([NVMM_MEMORY_FEATURE]),
+                )
+                .structure_with_features(
+                    gst::Structure::builder("video/x-raw").build(),
+                    gst::CapsFeatures::new([D3D11_MEMORY_FEATURE]),
                 );
 
             for codec in Codecs::video_codecs() {