From 4888a25bac9917886779d704b811b50d37d6893f Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Mon, 15 May 2023 04:56:47 +0900 Subject: [PATCH] webview2: Add Microsoft WebView2 based web browser source Adding webview2src element Part-of: --- .indent_cpp_list | 1 + subprojects/gst-plugins-bad/meson_options.txt | 1 + subprojects/gst-plugins-bad/sys/meson.build | 1 + .../sys/webview2/gstwebview2object.cpp | 1260 +++++++++++++++++ .../sys/webview2/gstwebview2object.h | 51 + .../sys/webview2/gstwebview2src.cpp | 630 +++++++++ .../sys/webview2/gstwebview2src.h | 34 + .../gst-plugins-bad/sys/webview2/meson.build | 105 ++ .../gst-plugins-bad/sys/webview2/plugin.cpp | 49 + 9 files changed, 2132 insertions(+) create mode 100644 subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.cpp create mode 100644 subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.h create mode 100644 subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.cpp create mode 100644 subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.h create mode 100644 subprojects/gst-plugins-bad/sys/webview2/meson.build create mode 100644 subprojects/gst-plugins-bad/sys/webview2/plugin.cpp diff --git a/.indent_cpp_list b/.indent_cpp_list index 418ad6495e..438d048349 100644 --- a/.indent_cpp_list +++ b/.indent_cpp_list @@ -13,5 +13,6 @@ subprojects/gst-plugins-bad/sys/nvcodec ^(subprojects/gst-plugins-bad/sys/qsv/)+(\w)+([^/])+(cpp$) subprojects/gst-plugins-bad/sys/va subprojects/gst-plugins-bad/sys/wasapi2 +subprojects/gst-plugins-bad/sys/webview2 subprojects/gst-plugins-bad/sys/wic ^(subprojects/gst-plugins-bad/sys/win32ipc/)+(\w)+([^/])+(cpp$) diff --git a/subprojects/gst-plugins-bad/meson_options.txt b/subprojects/gst-plugins-bad/meson_options.txt index 2a91898999..683d56a2ca 100644 --- a/subprojects/gst-plugins-bad/meson_options.txt +++ b/subprojects/gst-plugins-bad/meson_options.txt @@ -181,6 +181,7 @@ option('voamrwbenc', type : 'feature', value : 'auto', description : 'AMR-WB aud option('vulkan', type : 'feature', value : 'auto', description : 'Vulkan video sink plugin') option('wasapi', type : 'feature', value : 'auto', description : 'Windows Audio Session API source/sink plugin') option('wasapi2', type : 'feature', value : 'auto', description : 'Windows Audio Session API source/sink plugin with WinRT API') +option('webview2', type : 'feature', value : 'auto', description : 'WebView2 plugin') option('webp', type : 'feature', value : 'auto', description : 'WebP image codec plugin') option('webrtc', type : 'feature', value : 'auto', yield: true, description : 'WebRTC audio/video network bin plugin') option('webrtcdsp', type : 'feature', value : 'auto', description : 'Plugin with various audio filters provided by the WebRTC audio processing library') diff --git a/subprojects/gst-plugins-bad/sys/meson.build b/subprojects/gst-plugins-bad/sys/meson.build index 64c5eccf68..6c6cbdc92d 100644 --- a/subprojects/gst-plugins-bad/sys/meson.build +++ b/subprojects/gst-plugins-bad/sys/meson.build @@ -29,6 +29,7 @@ subdir('uvcgadget') subdir('va') subdir('wasapi') subdir('wasapi2') +subdir('webview2') subdir('wic') subdir('win32ipc') subdir('winks') diff --git a/subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.cpp b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.cpp new file mode 100644 index 0000000000..d7da54dbe6 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.cpp @@ -0,0 +1,1260 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WINAPI_PARTITION_APP +#undef WINAPI_PARTITION_APP +#endif + +#define WINAPI_PARTITION_APP 1 + +#include "gstwebview2object.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_EXTERN (gst_webview2_src_debug); +#define GST_CAT_DEFAULT gst_webview2_src_debug + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +using namespace ABI::Windows::Foundation; +using namespace ABI::Windows::Graphics; +using namespace ABI::Windows::Graphics::Capture; +using namespace ABI::Windows::Graphics::DirectX; +using namespace ABI::Windows::Graphics::DirectX::Direct3D11; +using namespace Windows::Graphics::DirectX::Direct3D11; +/* *INDENT-ON* */ + +#define WEBVIEW2_OBJECT_PROP_NAME "gst-d3d11-webview2-object" +#define WEBVIEW2_WINDOW_OFFSET (-16384) + +template < typename InterfaceType, PCNZWCH runtime_class_id > static HRESULT +GstGetActivationFactory (InterfaceType ** factory) +{ + HSTRING class_id_hstring; + HRESULT hr = WindowsCreateString (runtime_class_id, + wcslen (runtime_class_id), &class_id_hstring); + + if (FAILED (hr)) + return hr; + + hr = RoGetActivationFactory (class_id_hstring, IID_PPV_ARGS (factory)); + + if (FAILED (hr)) { + WindowsDeleteString (class_id_hstring); + return hr; + } + + return WindowsDeleteString (class_id_hstring); +} + +enum +{ + PROP_0, + PROP_DEVICE, +}; + +enum WebView2State +{ + WEBVIEW2_STATE_INIT, + WEBVIEW2_STATE_RUNNING, + WEBVIEW2_STATE_ERROR, +}; + +struct WebView2StatusData +{ + GstWebView2Object *object; + WebView2State state; +}; + +struct GstWebView2; + +struct GstWebView2ObjectPrivate +{ + GstWebView2ObjectPrivate () + { + context = g_main_context_new (); + loop = g_main_loop_new (context, FALSE); + } + + ~GstWebView2ObjectPrivate () + { + g_main_loop_quit (loop); + g_clear_pointer (&main_thread, g_thread_join); + g_main_loop_unref (loop); + g_main_context_unref (context); + if (pool) + gst_buffer_pool_set_active (pool, FALSE); + gst_clear_object (&pool); + gst_clear_object (&device); + gst_clear_caps (&caps); + } + + GstD3D11Device *device = nullptr; + std::mutex lock; + std::condition_variable cond; + std::shared_ptr < GstWebView2 > webview; + ComPtr < ID3D11Texture2D > staging; + GThread *main_thread = nullptr; + GMainContext *context = nullptr; + GMainLoop *loop = nullptr; + GstBufferPool *pool = nullptr; + GstCaps *caps = nullptr; + GstVideoInfo info; + std::string location; + HWND hwnd; + WebView2State state = WEBVIEW2_STATE_INIT; +}; + +struct _GstWebView2Object +{ + GstObject parent; + + GstWebView2ObjectPrivate *priv; +}; + +static gboolean gst_webview2_callback (WebView2StatusData * data); + +#define CLOSE_COM(obj) G_STMT_START { \ + if (obj) { \ + ComPtr closable; \ + obj.As (&closable); \ + if (closable) \ + closable->Close (); \ + obj = nullptr; \ + } \ +} G_STMT_END + +struct GstWebView2 +{ + GstWebView2 (GstWebView2Object * object, HWND hwnd) + :object_ (object), hwnd_ (hwnd) + { + ID3D11Device *device_handle; + ComPtr < ID3D10Multithread > multi_thread; + HRESULT hr; + + device_ = (GstD3D11Device *) gst_object_ref (object->priv->device); + + device_handle = gst_d3d11_device_get_device_handle (device_); + hr = device_handle->QueryInterface (IID_PPV_ARGS (&multi_thread)); + if (SUCCEEDED (hr)) + multi_thread->SetMultithreadProtected (TRUE); + + device_handle->QueryInterface (IID_PPV_ARGS (&dxgi_device_)); + } + + ~GstWebView2 () + { + if (comp_ctrl_) + comp_ctrl_->put_RootVisualTarget (nullptr); + webview_ = nullptr; + ctrl_ = nullptr; + comp_ctrl_ = nullptr; + env_ = nullptr; + + root_visual_ = nullptr; + comp_target_ = nullptr; + comp_device_ = nullptr; + + CLOSE_COM (session_); + CLOSE_COM (pool_); + CLOSE_COM (item_); + CLOSE_COM (d3d_device_); + + dxgi_device_ = nullptr; + + gst_object_unref (device_); + } + + HRESULT Open () + { + HRESULT hr; + + if (!dxgi_device_) + return E_FAIL; + + hr = SetupCapture (); + if (FAILED (hr)) + return hr; + + hr = SetupComposition (); + if (FAILED (hr)) + return hr; + + return CreateCoreWebView2Environment (Callback < + ICoreWebView2CreateCoreWebView2EnvironmentCompletedHandler > (this, + &GstWebView2::OnCreateEnvironmentCompleted).Get ()); + } + + HRESULT SetupCapture () + { + ComPtr < ID3D10Multithread > multi_thread; + ComPtr < IGraphicsCaptureItemInterop > interop; + ComPtr < IDXGIDevice > dxgi_device; + ComPtr < IInspectable > inspectable; + ComPtr < IDirect3D11CaptureFramePoolStatics > pool_statics; + ComPtr < IDirect3D11CaptureFramePoolStatics2 > pool_statics2; + ComPtr < IGraphicsCaptureSession2 > session2; + HRESULT hr; + + hr = GstGetActivationFactory < IGraphicsCaptureItemInterop, + RuntimeClass_Windows_Graphics_Capture_GraphicsCaptureItem > (&interop); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = interop->CreateForWindow (hwnd_, IID_PPV_ARGS (&item_)); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = item_->get_Size (&pool_size_); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = CreateDirect3D11DeviceFromDXGIDevice (dxgi_device_.Get (), + &inspectable); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = inspectable.As (&d3d_device_); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = GstGetActivationFactory < IDirect3D11CaptureFramePoolStatics, + RuntimeClass_Windows_Graphics_Capture_Direct3D11CaptureFramePool > + (&pool_statics); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = pool_statics.As (&pool_statics2); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = pool_statics2->CreateFreeThreaded (d3d_device_.Get (), + DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized, + 1, pool_size_, &pool_); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = pool_->CreateCaptureSession (item_.Get (), &session_); + if (!gst_d3d11_result (hr, device_)) + return hr; + + session_.As (&session2); + if (session2) + session2->put_IsCursorCaptureEnabled (FALSE); + + hr = session_->StartCapture (); + if (!gst_d3d11_result (hr, device_)) + return hr; + + return S_OK; + } + + HRESULT SetupComposition () + { + HRESULT hr; + + hr = DCompositionCreateDevice (dxgi_device_.Get (), + IID_PPV_ARGS (&comp_device_)); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = comp_device_->CreateTargetForHwnd (hwnd_, TRUE, &comp_target_); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = comp_device_->CreateVisual (&root_visual_); + if (!gst_d3d11_result (hr, device_)) + return hr; + + hr = comp_target_->SetRoot (root_visual_.Get ()); + if (!gst_d3d11_result (hr, device_)) + return hr; + + return S_OK; + } + + HRESULT + OnCreateEnvironmentCompleted (HRESULT hr, ICoreWebView2Environment * env) + { + ComPtr < ICoreWebView2Environment3 > env3; + + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, "Couldn't create environment"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + env_ = env; + hr = env_.As (&env3); + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, + "ICoreWebView2Environment3 interface is unavailable"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + hr = env3->CreateCoreWebView2CompositionController (hwnd_, + Callback < + ICoreWebView2CreateCoreWebView2CompositionControllerCompletedHandler > + (this, &GstWebView2::OnCreateCoreWebView2ControllerCompleted).Get ()); + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, + "CreateCoreWebView2CompositionController failed"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + return S_OK; + } + + HRESULT + OnCreateCoreWebView2ControllerCompleted (HRESULT hr, + ICoreWebView2CompositionController * comp_ctr) { + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, "Couldn't create composition controller"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + comp_ctrl_ = comp_ctr; + hr = comp_ctrl_.As (&ctrl_); + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, "Couldn't get controller interface"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + hr = comp_ctrl_->put_RootVisualTarget (root_visual_.Get ()); + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, "Couldn't set root visual object"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + hr = ctrl_->get_CoreWebView2 (&webview_); + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, "Couldn't get webview2 interface"); + NotifyState (WEBVIEW2_STATE_ERROR); + return hr; + } + + /* TODO: add audio mute property */ +#if 0 + ComPtr < ICoreWebView2_8 > webview8; + hr = webview_.As (&webview8); + if (!gst_d3d11_result (hr, nullptr)) { + GST_WARNING_OBJECT (object_, "ICoreWebView2_8 interface is unavailable"); + NotifyState (WEBVIEW2_STATE_ERROR); + return E_FAIL; + } + + webview8->put_IsMuted (TRUE); +#endif + + RECT bounds; + GetClientRect (hwnd_, &bounds); + ctrl_->put_Bounds (bounds); + ctrl_->put_IsVisible (TRUE); + + GST_INFO_OBJECT (object_, "All configured"); + + NotifyState (WEBVIEW2_STATE_RUNNING); + + return S_OK; + } + + void NotifyState (WebView2State state) + { + WebView2StatusData *data = g_new0 (WebView2StatusData, 1); + + data->object = object_; + data->state = state; + + g_main_context_invoke_full (object_->priv->context, + G_PRIORITY_DEFAULT, + (GSourceFunc) gst_webview2_callback, data, (GDestroyNotify) g_free); + } + + HRESULT DoCompose () + { + HRESULT hr; + GstD3D11DeviceLockGuard lk (device_); + hr = comp_device_->Commit (); + if (!gst_d3d11_result (hr, device_)) + return hr; + + return comp_device_->WaitForCommitCompletion (); + } + + GstFlowReturn DoCapture (ID3D11Texture2D * dst_texture) + { + HRESULT hr; + ComPtr < IDirect3D11CaptureFrame > frame; + GstClockTime timeout = gst_util_get_timestamp () + 5 * GST_SECOND; + SizeInt32 size; + ComPtr < IDirect3DSurface > surface; + ComPtr < IDirect3DDxgiInterfaceAccess > access; + ComPtr < ID3D11Texture2D > texture; + TimeSpan time; + GstClockTime pts; + RECT object_rect, bound_rect; + POINT object_pos = { 0, }; + UINT width, height; + D3D11_TEXTURE2D_DESC src_desc; + D3D11_TEXTURE2D_DESC dst_desc; + UINT x_offset = 0; + UINT y_offset = 0; + D3D11_BOX box = { 0, }; + + again: + std::unique_lock < std::mutex > flush_lk (lock_); + do { + if (flushing_) + return GST_FLOW_FLUSHING; + + hr = pool_->TryGetNextFrame (&frame); + if (frame) + break; + + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + cond_.wait_for (flush_lk, std::chrono::milliseconds (1)); + } while (gst_util_get_timestamp () < timeout); + flush_lk.unlock (); + + if (!frame) { + GST_ERROR_OBJECT (object_, "Timeout"); + return GST_FLOW_ERROR; + } + + hr = frame->get_ContentSize (&size); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + if (size.Width != pool_size_.Width || size.Height != pool_size_.Height) { + GST_DEBUG_OBJECT (object_, "Size changed %dx%d -> %dx%d", + pool_size_.Width, pool_size_.Height, size.Width, size.Height); + pool_size_ = size; + frame = nullptr; + hr = pool_->Recreate (d3d_device_.Get (), + DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized, 1, + size); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + goto again; + } + + hr = frame->get_SystemRelativeTime (&time); + if (SUCCEEDED (hr)) + pts = time.Duration * 100; + else + pts = gst_util_get_timestamp (); + + hr = frame->get_Surface (&surface); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + hr = surface.As (&access); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + hr = access->GetInterface (IID_PPV_ARGS (&texture)); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + if (!GetClientRect (hwnd_, &object_rect)) { + GST_ERROR_OBJECT (object_, "Couldn't get object rect"); + return GST_FLOW_ERROR; + } + + hr = DwmGetWindowAttribute (hwnd_, DWMWA_EXTENDED_FRAME_BOUNDS, &bound_rect, + sizeof (RECT)); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + if (!ClientToScreen (hwnd_, &object_pos)) { + GST_ERROR_OBJECT (object_, "Couldn't get position"); + return GST_FLOW_ERROR; + } + + width = object_rect.right - object_rect.left; + height = object_rect.bottom - object_rect.top; + + width = MAX (width, 1); + height = MAX (height, 1); + + if (object_pos.x > bound_rect.left) + x_offset = object_pos.x - bound_rect.left; + + if (object_pos.y > bound_rect.top) + y_offset = object_pos.y - bound_rect.top; + + box.front = 0; + box.back = 1; + + texture->GetDesc (&src_desc); + dst_texture->GetDesc (&dst_desc); + + box.left = x_offset; + box.left = MIN (src_desc.Width - 1, box.left); + + box.top = y_offset; + box.top = MIN (src_desc.Height - 1, box.top); + + box.right = dst_desc.Width + x_offset; + box.right = MIN (src_desc.Width, box.right); + + box.bottom = dst_desc.Height + y_offset; + box.bottom = MIN (src_desc.Height, box.right); + + { + auto context = gst_d3d11_device_get_device_context_handle (device_); + GstD3D11DeviceLockGuard lk (device_); + + context->CopySubresourceRegion (dst_texture, 0, 0, 0, 0, + texture.Get (), 0, &box); + } + + return GST_FLOW_OK; + } + + void SetFlushing (bool flushing) + { + std::lock_guard < std::mutex > lk (lock_); + flushing_ = flushing; + cond_.notify_all (); + } + + HWND hwnd_; + ComPtr < IDXGIDevice > dxgi_device_; + + ComPtr < IDCompositionDevice > comp_device_; + ComPtr < IDCompositionTarget > comp_target_; + ComPtr < IDCompositionVisual > root_visual_; + + ComPtr < ICoreWebView2Environment > env_; + ComPtr < ICoreWebView2 > webview_; + ComPtr < ICoreWebView2Controller > ctrl_; + ComPtr < ICoreWebView2CompositionController > comp_ctrl_; + + ComPtr < IDirect3DDevice > d3d_device_; + ComPtr < IGraphicsCaptureItem > item_; + ComPtr < IDirect3D11CaptureFramePool > pool_; + ComPtr < IGraphicsCaptureSession > session_; + SizeInt32 pool_size_; + + HRESULT last_hr_ = S_OK; + GstWebView2Object *object_; + GstD3D11Device *device_; + + std::mutex lock_; + std::condition_variable cond_; + + bool flushing_ = false; +}; + +static void gst_webview2_object_constructed (GObject * object); +static void gst_webview2_object_finalize (GObject * object); +static void gst_webview2_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static gpointer gst_webview2_thread_func (GstWebView2Object * self); + +#define gst_webview2_object_parent_class parent_class +G_DEFINE_TYPE (GstWebView2Object, gst_webview2_object, GST_TYPE_OBJECT); + +static void +gst_webview2_object_class_init (GstWebView2ObjectClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->constructed = gst_webview2_object_constructed; + object_class->finalize = gst_webview2_object_finalize; + object_class->set_property = gst_webview2_set_property; + + g_object_class_install_property (object_class, PROP_DEVICE, + g_param_spec_object ("device", "D3D11 Device", + "GstD3D11Device object for operating", + GST_TYPE_D3D11_DEVICE, (GParamFlags) + (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS))); +} + +static void +gst_webview2_object_init (GstWebView2Object * self) +{ + self->priv = new GstWebView2ObjectPrivate (); +} + +static void +gst_webview2_object_constructed (GObject * object) +{ + GstWebView2Object *self = GST_WEBVIEW2_OBJECT (object); + auto priv = self->priv; + + priv->main_thread = g_thread_new ("d3d11-webview2", + (GThreadFunc) gst_webview2_thread_func, self); + + std::unique_lock < std::mutex > lk (priv->lock); + while (!priv->state != WEBVIEW2_STATE_INIT) + priv->cond.wait (lk); + lk.unlock (); + + G_OBJECT_CLASS (parent_class)->constructed (object); +} + +static void +gst_webview2_object_finalize (GObject * object) +{ + GstWebView2Object *self = GST_WEBVIEW2_OBJECT (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_webview2_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstWebView2Object *self = GST_WEBVIEW2_OBJECT (object); + auto priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_DEVICE: + priv->device = (GstD3D11Device *) g_value_dup_object (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_webview2_callback (WebView2StatusData * data) +{ + GstWebView2Object *self = data->object; + auto priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + GST_DEBUG_OBJECT (self, "Got callback, state: %d", data->state); + + priv->state = data->state; + priv->cond.notify_all (); + + return G_SOURCE_REMOVE; +} + +static LRESULT CALLBACK +WndProc (HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam) +{ + GstWebView2Object *self; + + switch (msg) { + case WM_CREATE: + self = (GstWebView2Object *) + ((LPCREATESTRUCTA) lparam)->lpCreateParams; + SetPropA (hwnd, WEBVIEW2_OBJECT_PROP_NAME, self); + break; + case WM_SIZE: + self = (GstWebView2Object *) + GetPropA (hwnd, WEBVIEW2_OBJECT_PROP_NAME); + if (self && self->priv->webview && self->priv->webview->ctrl_) { + RECT bounds; + GetClientRect (hwnd, &bounds); + self->priv->webview->ctrl_->put_Bounds (bounds); + } + break; + default: + break; + } + + return DefWindowProcA (hwnd, msg, wparam, lparam); +} + +static HWND +gst_webview2_create_hwnd (GstWebView2Object * self) +{ + HINSTANCE inst = GetModuleHandle (nullptr); + + GST_D3D11_CALL_ONCE_BEGIN { + WNDCLASSEXA wc; + memset (&wc, 0, sizeof (WNDCLASSEXA)); + + wc.cbSize = sizeof (WNDCLASSEXA); + wc.lpfnWndProc = WndProc; + wc.hInstance = inst; + wc.style = CS_HREDRAW | CS_VREDRAW; + wc.lpszClassName = "GstD3D11Webview2Window"; + RegisterClassExA (&wc); + } + GST_D3D11_CALL_ONCE_END; + + return CreateWindowExA (0, "GstD3D11Webview2Window", "GstD3D11Webview2Window", + WS_POPUP, WEBVIEW2_WINDOW_OFFSET, + WEBVIEW2_WINDOW_OFFSET, 1920, 1080, nullptr, nullptr, inst, self); +} + +static gboolean +msg_cb (GIOChannel * source, GIOCondition condition, gpointer data) +{ + MSG msg; + + if (!PeekMessage (&msg, nullptr, 0, 0, PM_REMOVE)) + return G_SOURCE_CONTINUE; + + TranslateMessage (&msg); + DispatchMessage (&msg); + + return G_SOURCE_CONTINUE; +} + +static gpointer +gst_webview2_thread_func (GstWebView2Object * self) +{ + auto priv = self->priv; + GSource *msg_source; + GIOChannel *msg_io_channel; + ComPtr < ITaskbarList > taskbar_list; + HRESULT hr; + TIMECAPS time_caps; + guint timer_res = 0; + + if (timeGetDevCaps (&time_caps, sizeof (TIMECAPS)) == TIMERR_NOERROR) { + guint resolution; + + resolution = MIN (MAX (time_caps.wPeriodMin, 1), time_caps.wPeriodMax); + + if (timeBeginPeriod (resolution) != TIMERR_NOERROR) + timer_res = resolution; + } + + GST_DEBUG_OBJECT (self, "Entering thread"); + + RoInitialize (RO_INIT_SINGLETHREADED); + g_main_context_push_thread_default (priv->context); + + SetThreadDpiAwarenessContext (DPI_AWARENESS_CONTEXT_SYSTEM_AWARE); + + priv->hwnd = gst_webview2_create_hwnd (self); + + msg_io_channel = g_io_channel_win32_new_messages (0); + msg_source = g_io_create_watch (msg_io_channel, G_IO_IN); + g_source_set_callback (msg_source, (GSourceFunc) msg_cb, self, NULL); + g_source_attach (msg_source, priv->context); + + ShowWindow (priv->hwnd, SW_SHOW); + + priv->webview = std::make_shared < GstWebView2 > (self, priv->hwnd); + hr = priv->webview->Open (); + if (FAILED (hr) || priv->state == WEBVIEW2_STATE_ERROR) { + GST_ERROR_OBJECT (self, "Couldn't open webview2"); + goto out; + } + + hr = CoCreateInstance (CLSID_TaskbarList, nullptr, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS (&taskbar_list)); + if (SUCCEEDED (hr)) { + taskbar_list->DeleteTab (priv->hwnd); + taskbar_list = nullptr; + } + + GST_DEBUG_OBJECT (self, "Run loop"); + g_main_loop_run (priv->loop); + GST_DEBUG_OBJECT (self, "Exit loop"); + +out: + g_source_destroy (msg_source); + g_source_unref (msg_source); + g_io_channel_unref (msg_io_channel); + + priv->webview = nullptr; + DestroyWindow (priv->hwnd); + + GST_DEBUG_OBJECT (self, "Leaving thread"); + + g_main_context_pop_thread_default (priv->context); + RoUninitialize (); + + if (timer_res != 0) + timeEndPeriod (timer_res); + + return nullptr; +} + +GstWebView2Object * +gst_webview2_object_new (GstD3D11Device * device) +{ + GstWebView2Object *self; + + g_return_val_if_fail (GST_IS_D3D11_DEVICE (device), nullptr); + + self = (GstWebView2Object *) + g_object_new (GST_TYPE_WEBVIEW2_OBJECT, "device", device, nullptr); + gst_object_ref_sink (self); + + if (self->priv->state != WEBVIEW2_STATE_RUNNING) { + gst_object_unref (self); + return nullptr; + } + + return self; +} + +static gboolean +gst_webview2_update_location (GstWebView2Object * self) +{ + auto priv = self->priv; + std::wstring_convert < std::codecvt_utf8 < wchar_t >>conv; + std::wstring location_wide = conv.from_bytes (priv->location); + HRESULT hr; + + GST_DEBUG_OBJECT (self, "Navigate to %s", priv->location.c_str ()); + hr = priv->webview->webview_->Navigate (location_wide.c_str ()); + + if (FAILED (hr)) + GST_WARNING_OBJECT (self, "Couldn't navigate to %s", + priv->location.c_str ()); + + return G_SOURCE_REMOVE; +} + +gboolean +gst_webview2_object_set_location (GstWebView2Object * object, + const std::string & location) +{ + auto priv = object->priv; + std::unique_lock < std::mutex > lk (priv->lock); + + if (priv->state != WEBVIEW2_STATE_RUNNING) { + GST_WARNING_OBJECT (object, "Not running state"); + return FALSE; + } + priv->location = location; + lk.unlock (); + + g_main_context_invoke (priv->context, + (GSourceFunc) gst_webview2_update_location, object); + + return TRUE; +} + +static gboolean +gst_d3d11_webview_object_update_size (GstWebView2Object * self) +{ + auto priv = self->priv; + + GST_DEBUG_OBJECT (self, "Updating size to %dx%d", priv->info.width, + priv->info.height); + + MoveWindow (priv->hwnd, WEBVIEW2_WINDOW_OFFSET, + WEBVIEW2_WINDOW_OFFSET, priv->info.width, priv->info.height, TRUE); + + return G_SOURCE_REMOVE; +} + +gboolean +gst_webview2_object_set_caps (GstWebView2Object * object, GstCaps * caps) +{ + auto priv = object->priv; + std::unique_lock < std::mutex > lk (priv->lock); + bool is_d3d11 = false; + + if (priv->pool) { + gst_buffer_pool_set_active (priv->pool, FALSE); + gst_object_unref (priv->pool); + } + + priv->staging = nullptr; + + gst_video_info_from_caps (&priv->info, caps); + + auto features = gst_caps_get_features (caps, 0); + if (features + && gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { + priv->pool = gst_d3d11_buffer_pool_new (priv->device); + is_d3d11 = true; + } else { + priv->pool = gst_video_buffer_pool_new (); + } + + auto config = gst_buffer_pool_get_config (priv->pool); + + if (is_d3d11) { + auto params = gst_d3d11_allocation_params_new (priv->device, &priv->info, + GST_D3D11_ALLOCATION_FLAG_DEFAULT, + D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET, 0); + gst_buffer_pool_config_set_d3d11_allocation_params (config, params); + gst_d3d11_allocation_params_free (params); + } else { + D3D11_TEXTURE2D_DESC desc = { 0, }; + ID3D11Device *device_handle = + gst_d3d11_device_get_device_handle (priv->device); + HRESULT hr; + + desc.Width = priv->info.width; + desc.Height = priv->info.height; + desc.MipLevels = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.ArraySize = 1; + desc.Usage = D3D11_USAGE_STAGING; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + + hr = device_handle->CreateTexture2D (&desc, nullptr, &priv->staging); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (object, "Couldn't create staging texture"); + gst_clear_object (&priv->pool); + return FALSE; + } + } + + gst_buffer_pool_config_set_params (config, caps, priv->info.size, 0, 0); + gst_caps_replace (&priv->caps, caps); + + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + if (!gst_buffer_pool_set_config (priv->pool, config)) { + GST_ERROR_OBJECT (object, "Couldn't set pool config"); + gst_clear_object (&priv->pool); + return FALSE; + } + + if (!gst_buffer_pool_set_active (priv->pool, TRUE)) { + GST_ERROR_OBJECT (object, "Couldn't set active"); + gst_clear_object (&priv->pool); + return FALSE; + } + + lk.unlock (); + + g_main_context_invoke (priv->context, + (GSourceFunc) gst_d3d11_webview_object_update_size, object); + + return TRUE; +} + +struct NavigationEventData +{ + NavigationEventData () + { + if (event) + gst_event_unref (event); + } + + GstWebView2Object *object; + GstEvent *event = nullptr; +}; + +static void +navigation_event_free (NavigationEventData * data) +{ + delete data; +} + +static gboolean +gst_webview2_on_navigation_event (NavigationEventData * data) +{ + GstWebView2Object *self = data->object; + auto priv = self->priv; + GstEvent *event = data->event; + GstNavigationEventType type; + gdouble x, y; + gint button; + + if (!priv->webview || !priv->webview->comp_ctrl_) + goto out; + + type = gst_navigation_event_get_type (event); + + switch (type) { + /* FIXME: Implement key event */ + case GST_NAVIGATION_EVENT_MOUSE_BUTTON_PRESS: + if (gst_navigation_event_parse_mouse_button_event (event, + &button, &x, &y)) { + GST_TRACE_OBJECT (self, "Mouse press, button %d, %lfx%lf", + button, x, y); + COREWEBVIEW2_MOUSE_EVENT_KIND kind; + POINT point; + + point.x = (LONG) x; + point.y = (LONG) y; + + switch (button) { + case 1: + kind = COREWEBVIEW2_MOUSE_EVENT_KIND_LEFT_BUTTON_DOWN; + break; + case 2: + kind = COREWEBVIEW2_MOUSE_EVENT_KIND_RIGHT_BUTTON_DOWN; + break; + case 3: + kind = COREWEBVIEW2_MOUSE_EVENT_KIND_MIDDLE_BUTTON_DOWN; + break; + default: + goto out; + } + + /* FIXME: need to know the virtual key state */ + priv->webview->comp_ctrl_->SendMouseInput (kind, + COREWEBVIEW2_MOUSE_EVENT_VIRTUAL_KEYS_NONE, 0, point); + } + break; + case GST_NAVIGATION_EVENT_MOUSE_BUTTON_RELEASE: + if (gst_navigation_event_parse_mouse_button_event (event, + &button, &x, &y)) { + GST_TRACE_OBJECT (self, "Mouse release, button %d, %lfx%lf", + button, x, y); + COREWEBVIEW2_MOUSE_EVENT_KIND kind; + POINT point; + + point.x = (LONG) x; + point.y = (LONG) y; + + switch (button) { + case 1: + kind = COREWEBVIEW2_MOUSE_EVENT_KIND_LEFT_BUTTON_UP; + break; + case 2: + kind = COREWEBVIEW2_MOUSE_EVENT_KIND_RIGHT_BUTTON_UP; + break; + case 3: + kind = COREWEBVIEW2_MOUSE_EVENT_KIND_MIDDLE_BUTTON_UP; + break; + default: + goto out; + } + + /* FIXME: need to know the virtual key state */ + priv->webview->comp_ctrl_->SendMouseInput (kind, + COREWEBVIEW2_MOUSE_EVENT_VIRTUAL_KEYS_NONE, 0, point); + } + break; + case GST_NAVIGATION_EVENT_MOUSE_MOVE: + if (gst_navigation_event_parse_mouse_move_event (event, &x, &y)) { + GST_TRACE_OBJECT (self, "Mouse move, %lfx%lf", x, y); + POINT point; + + point.x = (LONG) x; + point.y = (LONG) y; + + /* FIXME: need to know the virtual key state */ + priv->webview-> + comp_ctrl_->SendMouseInput (COREWEBVIEW2_MOUSE_EVENT_KIND_MOVE, + COREWEBVIEW2_MOUSE_EVENT_VIRTUAL_KEYS_NONE, 0, point); + } + break; + default: + break; + } + +out: + return G_SOURCE_REMOVE; +} + +void +gst_webview2_object_send_event (GstWebView2Object * object, GstEvent * event) +{ + auto priv = object->priv; + auto data = new NavigationEventData (); + data->object = object; + data->event = gst_event_ref (event); + + g_main_context_invoke_full (priv->context, G_PRIORITY_DEFAULT, + (GSourceFunc) gst_webview2_on_navigation_event, data, + (GDestroyNotify) navigation_event_free); +} + +struct CaptureData +{ + GstWebView2Object *object; + bool notified = false; + std::mutex lock; + std::condition_variable cond; + GstBuffer *buffer = nullptr; + GstFlowReturn ret = GST_FLOW_ERROR; +}; + +static gboolean +gst_webview2_do_capture (CaptureData * data) +{ + GstWebView2Object *self = data->object; + auto priv = self->priv; + HRESULT hr; + GstFlowReturn ret; + GstBuffer *buffer; + GstMemory *mem; + GstMapInfo info; + GstClockTime pts; + ID3D11Texture2D *texture; + + if (!priv->pool) { + GST_ERROR_OBJECT (self, "Pool was not configured"); + goto out; + } + + hr = priv->webview->DoCompose (); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't compose"); + goto out; + } + + pts = gst_util_get_timestamp (); + + ret = gst_buffer_pool_acquire_buffer (priv->pool, &buffer, nullptr); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (self, "Couldn't acquire buffer"); + goto out; + } + + if (priv->staging) { + texture = priv->staging.Get (); + } else { + mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_memory_map (mem, &info, + (GstMapFlags) (GST_MAP_WRITE | GST_MAP_D3D11))) { + GST_ERROR_OBJECT (self, "Couldn't map memory"); + gst_buffer_unref (buffer); + goto out; + } + + texture = (ID3D11Texture2D *) info.data; + } + + ret = priv->webview->DoCapture (texture); + + if (!priv->staging) + gst_memory_unmap (mem, &info); + + if (ret != GST_FLOW_OK) { + gst_buffer_unref (buffer); + data->ret = ret; + goto out; + } + + if (priv->staging) { + GstVideoFrame frame; + D3D11_MAPPED_SUBRESOURCE map; + ID3D11DeviceContext *context = + gst_d3d11_device_get_device_context_handle (priv->device); + GstD3D11DeviceLockGuard lk (priv->device); + guint8 *dst; + guint8 *src; + guint width_in_bytes; + + hr = context->Map (priv->staging.Get (), 0, D3D11_MAP_READ, 0, &map); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't map staging texture"); + gst_buffer_unref (buffer); + goto out; + } + + if (!gst_video_frame_map (&frame, &priv->info, buffer, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Couldn't map frame"); + gst_buffer_unref (buffer); + context->Unmap (priv->staging.Get (), 0); + goto out; + } + + src = (guint8 *) map.pData; + dst = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + width_in_bytes = GST_VIDEO_FRAME_COMP_PSTRIDE (&frame, 0) + * GST_VIDEO_FRAME_WIDTH (&frame); + + for (guint i = 0; i < GST_VIDEO_FRAME_HEIGHT (&frame); i++) { + memcpy (dst, src, width_in_bytes); + dst += GST_VIDEO_FRAME_PLANE_STRIDE (&frame, 0); + src += map.RowPitch; + } + + gst_video_frame_unmap (&frame); + context->Unmap (priv->staging.Get (), 0); + } + + GST_BUFFER_PTS (buffer) = pts; + GST_BUFFER_DTS (buffer) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION (buffer) = GST_CLOCK_TIME_NONE; + + data->buffer = buffer; + data->ret = GST_FLOW_OK; + +out: + std::lock_guard < std::mutex > lk (data->lock); + data->notified = true; + data->cond.notify_one (); + + return G_SOURCE_REMOVE; +} + +GstFlowReturn +gst_webview2_object_get_buffer (GstWebView2Object * object, GstBuffer ** buffer) +{ + auto priv = object->priv; + CaptureData data; + + data.object = object; + + g_main_context_invoke (priv->context, + (GSourceFunc) gst_webview2_do_capture, &data); + + std::unique_lock < std::mutex > lk (data.lock); + while (!data.notified) + data.cond.wait (lk); + + if (!data.buffer) + return data.ret; + + *buffer = data.buffer; + return GST_FLOW_OK; +} + +void +gst_webview2_object_set_flushing (GstWebView2Object * object, bool flushing) +{ + auto priv = object->priv; + + priv->webview->SetFlushing (flushing); +} diff --git a/subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.h b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.h new file mode 100644 index 0000000000..ca7f42cfc3 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2object.h @@ -0,0 +1,51 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_WEBVIEW2_OBJECT (gst_webview2_object_get_type()) +G_DECLARE_FINAL_TYPE (GstWebView2Object, gst_webview2_object, + GST, WEBVIEW2_OBJECT, GstObject); + +GstWebView2Object * gst_webview2_object_new (GstD3D11Device * device); + +gboolean gst_webview2_object_set_location (GstWebView2Object * client, + const std::string & location); + +gboolean gst_webview2_object_set_caps (GstWebView2Object * client, + GstCaps * caps); + +void gst_webview2_object_send_event (GstWebView2Object * client, + GstEvent * event); + +GstFlowReturn gst_webview2_object_get_buffer (GstWebView2Object * client, + GstBuffer ** buffer); + +void gst_webview2_object_set_flushing (GstWebView2Object * client, + bool flushing); + +G_END_DECLS + diff --git a/subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.cpp b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.cpp new file mode 100644 index 0000000000..4db262b5dd --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.cpp @@ -0,0 +1,630 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-webview2src + * @title: webview2src + * @short_description: WebView2 based browser source + * + * Since: 1.26 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstwebview2src.h" +#include "gstwebview2object.h" +#include +#include +#include + +GST_DEBUG_CATEGORY (gst_webview2_src_debug); +#define GST_CAT_DEFAULT gst_webview2_src_debug + +static GstStaticPadTemplate pad_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, + "BGRA") ", pixel-aspect-ratio = 1/1;" GST_VIDEO_CAPS_MAKE ("BGRA") + ", pixel-aspect-ratio = 1/1")); + +enum +{ + PROP_0, + PROP_ADAPTER, + PROP_LOCATION, + PROP_PROCESSING_DEADLINE, +}; + +#define DEFAULT_LOCATION "about:blank" +#define DEFAULT_PROCESSING_DEADLINE (20 * GST_MSECOND) +#define DEFAULT_ADAPTER -1 + +/* *INDENT-OFF* */ +struct GstWebView2SrcPrivate +{ + GstD3D11Device *device = nullptr; + + GstWebView2Object *object = nullptr; + + std::mutex lock; + GstVideoInfo info; + guint64 last_frame_no; + GstClockID clock_id = nullptr; + + /* properties */ + gint adapter_index = DEFAULT_ADAPTER; + std::string location = DEFAULT_LOCATION; + GstClockTime processing_deadline = DEFAULT_PROCESSING_DEADLINE; +}; +/* *INDENT-ON* */ + +struct _GstWebView2Src +{ + GstBaseSrc parent; + + GstWebView2SrcPrivate *priv; +}; + +static void gst_webview2_src_finalize (GObject * object); +static void gst_webview2_src_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_win32_video_src_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstClock *gst_webview2_src_provide_clock (GstElement * elem); +static void gst_webview2_src_set_context (GstElement * elem, + GstContext * context); + +static gboolean gst_webview2_src_start (GstBaseSrc * src); +static gboolean gst_webview2_src_stop (GstBaseSrc * src); +static gboolean gst_webview2_src_unlock (GstBaseSrc * src); +static gboolean gst_webview2_src_unlock_stop (GstBaseSrc * src); +static gboolean gst_webview2_src_query (GstBaseSrc * src, GstQuery * query); +static GstCaps *gst_webview2_src_fixate (GstBaseSrc * src, GstCaps * caps); +static gboolean gst_webview2_src_set_caps (GstBaseSrc * src, GstCaps * caps); +static gboolean gst_webview2_src_event (GstBaseSrc * src, GstEvent * event); +static GstFlowReturn gst_webview2_src_create (GstBaseSrc * src, + guint64 offset, guint size, GstBuffer ** buf); +static void gst_webview2_src_uri_handler_init (gpointer iface, gpointer data); + +#define gst_webview2_src_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstWebView2Src, gst_webview2_src, GST_TYPE_BASE_SRC, + G_IMPLEMENT_INTERFACE (GST_TYPE_URI_HANDLER, + gst_webview2_src_uri_handler_init)); + +static void +gst_webview2_src_class_init (GstWebView2SrcClass * klass) +{ + auto object_class = G_OBJECT_CLASS (klass); + auto element_class = GST_ELEMENT_CLASS (klass); + auto src_class = GST_BASE_SRC_CLASS (klass); + + object_class->finalize = gst_webview2_src_finalize; + object_class->set_property = gst_webview2_src_set_property; + object_class->get_property = gst_win32_video_src_get_property; + + g_object_class_install_property (object_class, PROP_ADAPTER, + g_param_spec_int ("adapter", "Adapter", + "DXGI Adapter index (-1 for any device)", + -1, G_MAXINT32, DEFAULT_ADAPTER, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_LOCATION, + g_param_spec_string ("location", "location", + "The URL to display", + nullptr, (GParamFlags) (G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + + g_object_class_install_property (object_class, PROP_PROCESSING_DEADLINE, + g_param_spec_uint64 ("processing-deadline", "Processing deadline", + "Maximum processing time for a buffer in nanoseconds", 0, G_MAXUINT64, + DEFAULT_PROCESSING_DEADLINE, (GParamFlags) (G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + + gst_element_class_set_static_metadata (element_class, + "WebView2 Source", "Source/Video", + "Creates a video stream rendered by WebView2", + "Seungha Yang "); + + gst_element_class_add_static_pad_template (element_class, &pad_template); + + element_class->provide_clock = + GST_DEBUG_FUNCPTR (gst_webview2_src_provide_clock); + element_class->set_context = GST_DEBUG_FUNCPTR (gst_webview2_src_set_context); + + src_class->start = GST_DEBUG_FUNCPTR (gst_webview2_src_start); + src_class->stop = GST_DEBUG_FUNCPTR (gst_webview2_src_stop); + src_class->unlock = GST_DEBUG_FUNCPTR (gst_webview2_src_unlock); + src_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_webview2_src_unlock_stop); + src_class->query = GST_DEBUG_FUNCPTR (gst_webview2_src_query); + src_class->fixate = GST_DEBUG_FUNCPTR (gst_webview2_src_fixate); + src_class->set_caps = GST_DEBUG_FUNCPTR (gst_webview2_src_set_caps); + src_class->event = GST_DEBUG_FUNCPTR (gst_webview2_src_event); + src_class->create = GST_DEBUG_FUNCPTR (gst_webview2_src_create); + + GST_DEBUG_CATEGORY_INIT (gst_webview2_src_debug, "webview2src", + 0, "webview2src"); +} + +static void +gst_webview2_src_init (GstWebView2Src * self) +{ + gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME); + gst_base_src_set_live (GST_BASE_SRC (self), TRUE); + + self->priv = new GstWebView2SrcPrivate (); + + GST_OBJECT_FLAG_SET (self, GST_ELEMENT_FLAG_PROVIDE_CLOCK); + GST_OBJECT_FLAG_SET (self, GST_ELEMENT_FLAG_REQUIRE_CLOCK); +} + +static void +gst_webview2_src_finalize (GObject * object) +{ + auto self = GST_WEBVIEW2_SRC (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_webview2_src_set_location (GstWebView2Src * self, const gchar * location) +{ + auto priv = self->priv; + priv->location.clear (); + if (location) + priv->location = location; + else + priv->location = DEFAULT_LOCATION; + + if (priv->object) + gst_webview2_object_set_location (priv->object, priv->location); +} + +static void +gst_webview2_src_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + auto self = GST_WEBVIEW2_SRC (object); + auto priv = self->priv; + std::unique_lock < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_ADAPTER: + priv->adapter_index = g_value_get_int (value); + break; + case PROP_LOCATION: + gst_webview2_src_set_location (self, g_value_get_string (value)); + break; + case PROP_PROCESSING_DEADLINE: + { + GstClockTime prev_val, new_val; + prev_val = priv->processing_deadline; + new_val = g_value_get_uint64 (value); + priv->processing_deadline = new_val; + + if (prev_val != new_val) { + lk.unlock (); + GST_DEBUG_OBJECT (self, "Posting latency message"); + gst_element_post_message (GST_ELEMENT_CAST (self), + gst_message_new_latency (GST_OBJECT_CAST (self))); + } + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_win32_video_src_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + auto self = GST_WEBVIEW2_SRC (object); + auto priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + switch (prop_id) { + case PROP_ADAPTER: + g_value_set_int (value, priv->adapter_index); + break; + case PROP_LOCATION: + g_value_set_string (value, priv->location.c_str ()); + break; + case PROP_PROCESSING_DEADLINE: + g_value_set_uint64 (value, priv->processing_deadline); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstClock * +gst_webview2_src_provide_clock (GstElement * elem) +{ + return gst_system_clock_obtain (); +} + +static void +gst_webview2_src_set_context (GstElement * elem, GstContext * context) +{ + auto self = GST_WEBVIEW2_SRC (elem); + auto priv = self->priv; + + gst_d3d11_handle_set_context (elem, + context, priv->adapter_index, &priv->device); + + GST_ELEMENT_CLASS (parent_class)->set_context (elem, context); +} + +static gboolean +gst_webview2_src_start (GstBaseSrc * src) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + + GST_DEBUG_OBJECT (self, "Start"); + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), + priv->adapter_index, &priv->device)) { + GST_ERROR_OBJECT (self, "Couldn't get D3D11 context"); + return FALSE; + } + + std::lock_guard < std::mutex > lk (priv->lock); + priv->object = gst_webview2_object_new (priv->device); + if (!priv->object) { + GST_ERROR_OBJECT (self, "Couldn't create object"); + return FALSE; + } + + gst_webview2_object_set_location (priv->object, priv->location); + + priv->last_frame_no = -1; + + return TRUE; +} + +static gboolean +gst_webview2_src_stop (GstBaseSrc * src) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + + std::lock_guard < std::mutex > lk (priv->lock); + + GST_DEBUG_OBJECT (self, "Stop"); + + gst_clear_object (&priv->object); + gst_clear_object (&priv->device); + + return TRUE; +} + +static gboolean +gst_webview2_src_unlock (GstBaseSrc * src) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + + GST_DEBUG_OBJECT (self, "Unlock"); + + std::lock_guard < std::mutex > lk (priv->lock); + if (priv->object) + gst_webview2_object_set_flushing (priv->object, true); + + if (priv->clock_id) + gst_clock_id_unschedule (priv->clock_id); + + return TRUE; +} + +static gboolean +gst_webview2_src_unlock_stop (GstBaseSrc * src) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + + GST_DEBUG_OBJECT (self, "Unlock stop"); + + std::lock_guard < std::mutex > lk (priv->lock); + if (priv->object) + gst_webview2_object_set_flushing (priv->object, false); + + return TRUE; +} + +static gboolean +gst_webview2_src_query (GstBaseSrc * src, GstQuery * query) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_LATENCY: + { + std::lock_guard < std::mutex > lk (priv->lock); + if (GST_CLOCK_TIME_IS_VALID (priv->processing_deadline)) { + gst_query_set_latency (query, TRUE, priv->processing_deadline, + GST_CLOCK_TIME_NONE); + } else { + gst_query_set_latency (query, TRUE, 0, 0); + } + return TRUE; + } + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (self), query, + priv->device)) { + return TRUE; + } + break; + default: + break; + } + + return GST_BASE_SRC_CLASS (parent_class)->query (src, query); +} + +static GstCaps * +gst_webview2_src_fixate (GstBaseSrc * src, GstCaps * caps) +{ + caps = gst_caps_make_writable (caps); + auto s = gst_caps_get_structure (caps, 0); + + gst_structure_fixate_field_nearest_int (s, "width", 1920); + gst_structure_fixate_field_nearest_int (s, "height", 1080); + gst_structure_fixate_field_nearest_fraction (s, "framerate", 30, 1); + + return GST_BASE_SRC_CLASS (parent_class)->fixate (src, caps); +} + +static gboolean +gst_webview2_src_set_caps (GstBaseSrc * src, GstCaps * caps) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + if (!gst_video_info_from_caps (&priv->info, caps)) { + GST_ERROR_OBJECT (self, "Invalid caps %" GST_PTR_FORMAT, caps); + return FALSE; + } + + if (priv->object) + gst_webview2_object_set_caps (priv->object, caps); + + return TRUE; +} + +static gboolean +gst_webview2_src_event (GstBaseSrc * src, GstEvent * event) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + std::unique_lock < std::mutex > lk (priv->lock); + + if (priv->object || GST_EVENT_TYPE (event) == GST_EVENT_NAVIGATION) { + gst_webview2_object_send_event (priv->object, event); + return TRUE; + } + lk.unlock (); + + return GST_BASE_SRC_CLASS (parent_class)->event (src, event); +} + +static bool +gst_webview2_clock_is_system (GstClock * clock) +{ + GstClockType clock_type = GST_CLOCK_TYPE_MONOTONIC; + GstClock *mclock; + + if (G_OBJECT_TYPE (clock) != GST_TYPE_SYSTEM_CLOCK) + return false; + + g_object_get (clock, "clock-type", &clock_type, nullptr); + if (clock_type != GST_CLOCK_TYPE_MONOTONIC) + return false; + + mclock = gst_clock_get_master (clock); + if (!mclock) + return true; + + gst_object_unref (mclock); + return false; +} + +static GstFlowReturn +gst_webview2_src_create (GstBaseSrc * src, guint64 offset, guint size, + GstBuffer ** buf) +{ + auto self = GST_WEBVIEW2_SRC (src); + auto priv = self->priv; + GstFlowReturn ret; + GstClock *clock; + bool is_system_clock; + GstClockTime pts; + GstClockTime base_time; + GstClockTime now_system; + GstClockTime now_gst; + GstClockTime capture_pts; + GstClockTime next_capture_ts; + guint64 next_frame_no = 0; + GstBuffer *buffer; + gint fps_n, fps_d; + GstClockTime dur = GST_CLOCK_TIME_NONE; + + clock = gst_element_get_clock (GST_ELEMENT_CAST (self)); + now_gst = gst_clock_get_time (clock); + base_time = GST_ELEMENT_CAST (self)->base_time; + next_capture_ts = now_gst - base_time; + is_system_clock = gst_webview2_clock_is_system (clock); + + fps_n = priv->info.fps_n; + fps_d = priv->info.fps_d; + + if (fps_n > 0 && fps_d > 0) { + next_frame_no = gst_util_uint64_scale (next_capture_ts, + fps_n, GST_SECOND * fps_d); + + if (next_frame_no == priv->last_frame_no) { + GstClockID id; + GstClockReturn clock_ret; + std::unique_lock < std::mutex > lk (priv->lock); + + next_frame_no++; + + next_capture_ts = gst_util_uint64_scale (next_frame_no, + fps_d * GST_SECOND, fps_n); + + id = gst_clock_new_single_shot_id (GST_ELEMENT_CLOCK (self), + next_capture_ts + base_time); + priv->clock_id = id; + lk.unlock (); + + clock_ret = gst_clock_id_wait (id, nullptr); + + lk.lock (); + + gst_clock_id_unref (id); + priv->clock_id = nullptr; + + if (clock_ret == GST_CLOCK_UNSCHEDULED) { + gst_object_unref (clock); + return GST_FLOW_FLUSHING; + } + + dur = gst_util_uint64_scale_int (GST_SECOND, fps_d, fps_n); + } else { + GstClockTime next_frame_ts = gst_util_uint64_scale (next_frame_no + 1, + fps_d * GST_SECOND, fps_n); + dur = next_frame_ts - next_capture_ts; + } + } + + priv->last_frame_no = next_frame_no; + + ret = gst_webview2_object_get_buffer (priv->object, &buffer); + if (ret != GST_FLOW_OK) { + gst_object_unref (clock); + return ret; + } + + capture_pts = GST_BUFFER_PTS (buffer); + now_system = gst_util_get_timestamp (); + now_gst = gst_clock_get_time (clock); + gst_object_unref (clock); + + if (!is_system_clock) { + GstClockTimeDiff now_pts = now_gst - base_time + capture_pts - now_system; + + if (now_pts >= 0) + pts = now_pts; + else + pts = 0; + } else { + if (capture_pts >= base_time) { + pts = capture_pts - base_time; + } else { + GST_WARNING_OBJECT (self, + "Captured time is smaller than our base time, remote %" + GST_TIME_FORMAT ", base_time %" GST_TIME_FORMAT, + GST_TIME_ARGS (capture_pts), GST_TIME_ARGS (base_time)); + pts = 0; + } + } + + GST_BUFFER_PTS (buffer) = pts; + GST_BUFFER_DURATION (buffer) = dur; + *buf = buffer; + + return GST_FLOW_OK; +} + +static GstURIType +gst_webview2_src_uri_get_type (GType type) +{ + return GST_URI_SRC; +} + +static const gchar *const * +gst_webview2_src_get_protocols (GType type) +{ + static const gchar *protocols[] = { "web+http", "web+https", nullptr }; + + return protocols; +} + +static gchar * +gst_webview2_src_get_uri (GstURIHandler * handler) +{ + auto self = GST_WEBVIEW2_SRC (handler); + auto priv = self->priv; + std::lock_guard < std::mutex > lk (priv->lock); + + if (priv->location.empty ()) + return nullptr; + + return g_strdup (priv->location.c_str ()); +} + +static gboolean +gst_webview2_src_set_uri (GstURIHandler * handler, const gchar * uri_str, + GError ** err) +{ + auto self = GST_WEBVIEW2_SRC (handler); + auto priv = self->priv; + + auto protocol = gst_uri_get_protocol (uri_str); + if (!g_str_has_prefix (protocol, "web+")) { + g_free (protocol); + return FALSE; + } + + auto uri = gst_uri_from_string (uri_str); + gst_uri_set_scheme (uri, protocol + 4); + + auto location = gst_uri_to_string (uri); + + std::lock_guard < std::mutex > lk (priv->lock); + gst_webview2_src_set_location (self, location); + + gst_uri_unref (uri); + g_free (protocol); + g_free (location); + + return TRUE; +} + +static void +gst_webview2_src_uri_handler_init (gpointer iface, gpointer data) +{ + auto handler = (GstURIHandlerInterface *) iface; + + handler->get_type = gst_webview2_src_uri_get_type; + handler->get_protocols = gst_webview2_src_get_protocols; + handler->get_uri = gst_webview2_src_get_uri; + handler->set_uri = gst_webview2_src_set_uri; +} diff --git a/subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.h b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.h new file mode 100644 index 0000000000..26ffef1e28 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/webview2/gstwebview2src.h @@ -0,0 +1,34 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_WEBVIEW2_SRC (gst_webview2_src_get_type()) +G_DECLARE_FINAL_TYPE (GstWebView2Src, gst_webview2_src, + GST, WEBVIEW2_SRC, GstBaseSrc); + +G_END_DECLS + diff --git a/subprojects/gst-plugins-bad/sys/webview2/meson.build b/subprojects/gst-plugins-bad/sys/webview2/meson.build new file mode 100644 index 0000000000..20d0768e40 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/webview2/meson.build @@ -0,0 +1,105 @@ +webview2_sources = [ + 'gstwebview2object.cpp', + 'gstwebview2src.cpp', + 'plugin.cpp', +] + +extra_args = ['-DGST_USE_UNSTABLE_API'] + +webview2_option = get_option('webview2') +if host_system != 'windows' or webview2_option.disabled() + subdir_done() +endif + +if not gstd3d11_dep.found() + if webview2_option.enabled() + error('The webview2 was enabled explicitly, but required dependencies were not found.') + endif + subdir_done() +endif + +if cc.get_id() != 'msvc' + if webview2_option.enabled() + error('webview2 plugin supports only MSVC build.') + endif + subdir_done() +endif + +if d3d11_winapi_only_app + if webview2_option.enabled() + error('UWP only build is not supported.') + endif + subdir_done() +endif + +have_wgc = cxx.compiles(''' + #include + #include + #include + #include, + #include + #include + #include + #include + using namespace Microsoft::WRL; + using namespace ABI::Windows::Graphics::Capture; + ComPtr pool_statics; + ComPtr pool_statics2; + ComPtr pool; + ComPtr session; + ComPtr session2; + ''', + name: 'Windows Graphics Capture support in Windows SDK') + +if not have_wgc + if webview2_option.enabled() + error('Windows Graphics Capture API is unavailable.') + endif + subdir_done() +endif + +building_for_win10 = cxx.compiles('''#include + #ifndef WINVER + #error "unknown minimum supported OS version" + #endif + #if (WINVER < 0x0A00) + #error "Windows 10 API is not guaranteed" + #endif + ''', + name: 'building for Windows 10') + +if not building_for_win10 + message('Bumping target Windows version to Windows 10 for building webview2 plugin') + extra_args += ['-DWINVER=0x0A00', '-D_WIN32_WINNT=0x0A00', '-DNTDDI_VERSION=WDK_NTDDI_VERSION'] +endif + +winmm_lib = cc.find_library('winmm', required : webview2_option) +dwmapi_lib = cc.find_library('dwmapi', required : webview2_option) +dcomp_lib = cc.find_library('dcomp', required : webview2_option) +runtimeobject_dep = cc.find_library('runtimeobject', required : webview2_option) +loader_lib = cc.find_library('WebView2LoaderStatic', required: false) +sdk_deps = [] +if loader_lib.found() and cc.has_header('WebView2.h') and cc.has_header('WebView2EnvironmentOptions.h') + sdk_deps += [loader_lib] +else + webview2_dep = dependency('webview2', required : webview2_option, + fallback: ['webview2', 'webview2_dep']) + if not webview2_dep.found() + subdir_done() + endif + + sdk_deps += [webview2_dep] +endif + +gstwebview2 = library('gstwebview2', + webview2_sources, + c_args : gst_plugins_bad_args + extra_args, + cpp_args : gst_plugins_bad_args + extra_args, + include_directories : [configinc], + dependencies : [gstbase_dep, gstvideo_dep, gmodule_dep, + gstd3d11_dep, winmm_lib, runtimeobject_dep, dwmapi_lib, + dcomp_lib] + sdk_deps, + install : true, + install_dir : plugins_install_dir, +) +plugins += [gstwebview2] diff --git a/subprojects/gst-plugins-bad/sys/webview2/plugin.cpp b/subprojects/gst-plugins-bad/sys/webview2/plugin.cpp new file mode 100644 index 0000000000..7a7f155851 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/webview2/plugin.cpp @@ -0,0 +1,49 @@ +/* GStreamer + * Copyright (C) 2024 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * plugin-webview2: + * + * Microsoft WebView2 plugin + * + * Since: 1.26 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "gstwebview2src.h" +#include "gstwebview2object.h" + +static gboolean +plugin_init (GstPlugin * plugin) +{ + gst_element_register (plugin, + "webview2src", GST_RANK_MARGINAL, GST_TYPE_WEBVIEW2_SRC); + + return TRUE; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + webview2, + "WebView2 plugin", + plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)