From 926d5366b99b3498632a45147cfa329dbbf2cc30 Mon Sep 17 00:00:00 2001
From: Alexander Slobodeniuk <aslobodeniuk@fluendo.com>
Date: Thu, 18 Apr 2024 10:18:05 +0200
Subject: [PATCH] d3d11device: protect device_lock vs device_new

It seems that when D3D11CreateDevice collides in time
with other D3D11 calls, in particular the proccess of
creating a shader, it can corrupt the memory in the driver.
D3D11 spec doesn't seem to require any thread safety from
D3D11CreateDevice. Following MSDN, it is supposed to be called
in the beginning of the proccess, while GStreamer calls it with each
new pipeline.
Such crashes in the driver were frequently reproducing on the
Intel UHD 630 machine.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6686>
---
 .../gst-libs/gst/d3d11/gstd3d11device.cpp         |  9 +++++++++
 .../tests/check/libs/d3d11device.cpp              | 15 +++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11device.cpp b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11device.cpp
index 3a18b2cb44..ab8faff0bd 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11device.cpp
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11device.cpp
@@ -731,6 +731,8 @@ gst_d3d11_device_log_live_objects (GstD3D11Device * device,
 #endif
 }
 
+static SRWLOCK _device_creation_rwlock = SRWLOCK_INIT;
+
 static void
 gst_d3d11_device_dispose (GObject * object)
 {
@@ -739,6 +741,7 @@ gst_d3d11_device_dispose (GObject * object)
 
   GST_LOG_OBJECT (self, "dispose");
 
+  AcquireSRWLockExclusive (&_device_creation_rwlock);
   priv->ps_cache.clear ();
   priv->vs_cache.clear ();
   priv->sampler_cache.clear ();
@@ -764,6 +767,7 @@ gst_d3d11_device_dispose (GObject * object)
   GST_D3D11_CLEAR_COM (priv->dxgi_info_queue);
 #endif
 
+  ReleaseSRWLockExclusive (&_device_creation_rwlock);
   G_OBJECT_CLASS (parent_class)->dispose (object);
 }
 
@@ -996,6 +1000,7 @@ gst_d3d11_device_new_internal (const GstD3D11DeviceConstructData * data)
 
   debug_init_once ();
 
+  GstD3D11SRWLockGuard lk (&_device_creation_rwlock);
   hr = CreateDXGIFactory1 (IID_PPV_ARGS (&factory));
   if (!gst_d3d11_result (hr, NULL)) {
     GST_WARNING ("cannot create dxgi factory, hr: 0x%x", (guint) hr);
@@ -1357,6 +1362,8 @@ gst_d3d11_device_lock (GstD3D11Device * device)
 
   priv = device->priv;
 
+  AcquireSRWLockShared (&_device_creation_rwlock);
+
   GST_TRACE_OBJECT (device, "device locking");
   priv->extern_lock.lock ();
   GST_TRACE_OBJECT (device, "device locked");
@@ -1382,6 +1389,8 @@ gst_d3d11_device_unlock (GstD3D11Device * device)
 
   priv->extern_lock.unlock ();
   GST_TRACE_OBJECT (device, "device unlocked");
+
+  ReleaseSRWLockShared (&_device_creation_rwlock);
 }
 
 /**
diff --git a/subprojects/gst-plugins-bad/tests/check/libs/d3d11device.cpp b/subprojects/gst-plugins-bad/tests/check/libs/d3d11device.cpp
index 30b89f9adf..62f4356ec5 100644
--- a/subprojects/gst-plugins-bad/tests/check/libs/d3d11device.cpp
+++ b/subprojects/gst-plugins-bad/tests/check/libs/d3d11device.cpp
@@ -241,6 +241,20 @@ GST_START_TEST (test_device_new_concurrency)
 
 GST_END_TEST;
 
+GST_START_TEST (test_device_lock_recursiveness)
+{
+  GstD3D11Device *device = gst_d3d11_device_new (0, 0);
+
+  gst_d3d11_device_lock (device);
+  gst_d3d11_device_lock (device);
+  gst_d3d11_device_unlock (device);
+  gst_d3d11_device_unlock (device);
+
+  gst_object_unref (device);
+}
+
+GST_END_TEST;
+
 static Suite *
 d3d11device_suite (void)
 {
@@ -256,6 +270,7 @@ d3d11device_suite (void)
   tcase_add_test (tc_basic, test_device_for_adapter_luid);
   tcase_add_test (tc_basic, test_device_new_wrapped);
   tcase_add_test (tc_basic, test_device_new_concurrency);
+  tcase_add_test (tc_basic, test_device_lock_recursiveness);
 
 out:
   return s;