From 803550111a785a4f462da6d8d61f17f1d87098d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 7 Feb 2024 18:12:29 +0200 Subject: [PATCH] gtk4: Improve handling of RGBA GL textures in GTK GTK 4.14 comes with a new GL renderer that does not support GL shader nodes anymore, so the conversion from non-premultiplied alpha to premultiplied alpha has to happen differently. For GTK 4.14 or newer we use the correct format directly when building the texture, but only if a GLES3+ context is used. In that case the NGL renderer is used by GTK, which supports non-premultiplied formats correctly and fast. For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a self-mask to pre-multiply the alpha. For GTK before 4.10, we use a GL shader and hope that it works. Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/488 Part-of: --- video/gtk4/Cargo.toml | 3 + video/gtk4/src/lib.rs | 7 ++ video/gtk4/src/sink/frame.rs | 104 +++++++++++++++++++++------ video/gtk4/src/sink/imp.rs | 2 +- video/gtk4/src/sink/mod.rs | 2 +- video/gtk4/src/sink/paintable/imp.rs | 82 ++++++++++++++++++--- 6 files changed, 166 insertions(+), 34 deletions(-) diff --git a/video/gtk4/Cargo.toml b/video/gtk4/Cargo.toml index 0b4c83f9..fce75fd9 100644 --- a/video/gtk4/Cargo.toml +++ b/video/gtk4/Cargo.toml @@ -52,6 +52,9 @@ x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"] winegl = ["gdk-win32/egl", "gst-gl-egl"] capi = [] doc = ["gst/v1_18"] +gtk_v4_10 = ["gtk/v4_10"] +gtk_v4_12 = ["gtk/v4_12", "gtk_v4_10"] +gtk_v4_14 = ["gtk/v4_14", "gtk_v4_12"] [package.metadata.capi] min_version = "0.9.21" diff --git a/video/gtk4/src/lib.rs b/video/gtk4/src/lib.rs index 1ccd6116..0734da03 100644 --- a/video/gtk4/src/lib.rs +++ b/video/gtk4/src/lib.rs @@ -22,6 +22,13 @@ mod utils; pub use sink::PaintableSink; fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + #[cfg(not(feature = "gtk_v4_10"))] + { + if gtk::micro_version() >= 13 { + gst::warning!(sink::imp::CAT, obj: plugin, "GTK 4.13 or newer detected but plugin not compiled with support for this version. Rendering of video frames with alpha will likely be wrong"); + } + } + sink::register(plugin) } diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index 987732e8..aaa7e224 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -94,6 +94,18 @@ impl AsRef<[u8]> for FrameWrapper { } } +fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat { + match f { + gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8, + gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8, + gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8, + gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8, + gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8, + gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8, + _ => unreachable!(), + } +} + fn video_frame_to_memory_texture( frame: gst_video::VideoFrame, cached_textures: &mut HashMap, @@ -109,15 +121,7 @@ fn video_frame_to_memory_texture( return (texture.clone(), pixel_aspect_ratio); } - let format = match frame.format() { - gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8, - gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8, - gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8, - gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8, - gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8, - gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8, - _ => unreachable!(), - }; + let format = video_format_to_memory_format(frame.format()); let width = frame.width(); let height = frame.height(); let rowstride = frame.plane_stride()[0] as usize; @@ -143,7 +147,7 @@ fn video_frame_to_gl_texture( cached_textures: &mut HashMap, used_textures: &mut HashSet, gdk_context: &gdk::GLContext, - wrapped_context: &gst_gl::GLContext, + #[allow(unused)] wrapped_context: &gst_gl::GLContext, ) -> (gdk::Texture, f64) { let texture_id = frame.texture_id(0).expect("Invalid texture id") as usize; @@ -159,19 +163,77 @@ fn video_frame_to_gl_texture( let height = frame.height(); let sync_meta = frame.buffer().meta::().unwrap(); - sync_meta.wait(wrapped_context); let texture = unsafe { - gdk::GLTexture::with_release_func( - gdk_context, - texture_id as u32, - width as i32, - height as i32, - move || { - // Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier - drop(frame); - }, - ) + #[cfg(feature = "gtk_v4_12")] + { + let format = { + let format = video_format_to_memory_format(frame.format()); + #[cfg(feature = "gtk_v4_14")] + { + use gtk::prelude::*; + + if gdk_context.api() != gdk::GLAPI::GLES || gdk_context.version().0 < 3 { + // Map alpha formats to the pre-multiplied versions because we pre-multiply + // ourselves if not GLES3 with the new GL renderer is used as the GTK GL + // backend does not natively support non-premultiplied formats. + match format { + gdk::MemoryFormat::B8g8r8a8 => gdk::MemoryFormat::B8g8r8a8Premultiplied, + gdk::MemoryFormat::A8r8g8b8 => gdk::MemoryFormat::A8r8g8b8Premultiplied, + gdk::MemoryFormat::R8g8b8a8 => gdk::MemoryFormat::R8g8b8a8Premultiplied, + gdk::MemoryFormat::A8b8g8r8 => gdk::MemoryFormat::A8r8g8b8Premultiplied, + gdk::MemoryFormat::R8g8b8 | gdk::MemoryFormat::B8g8r8 => format, + _ => unreachable!(), + } + } else { + format + } + } + + #[cfg(not(feature = "gtk_v4_14"))] + { + // Map alpha formats to the pre-multiplied versions because we pre-multiply + // ourselves in pre-4.14 versions as the GTK GL backend does not natively + // support non-premultiplied formats + match format { + gdk::MemoryFormat::B8g8r8a8 => gdk::MemoryFormat::B8g8r8a8Premultiplied, + gdk::MemoryFormat::A8r8g8b8 => gdk::MemoryFormat::A8r8g8b8Premultiplied, + gdk::MemoryFormat::R8g8b8a8 => gdk::MemoryFormat::R8g8b8a8Premultiplied, + gdk::MemoryFormat::A8b8g8r8 => gdk::MemoryFormat::A8r8g8b8Premultiplied, + gdk::MemoryFormat::R8g8b8 | gdk::MemoryFormat::B8g8r8 => format, + _ => unreachable!(), + } + } + }; + let sync_point = (*sync_meta.as_ptr()).data; + + gdk::GLTextureBuilder::new() + .set_context(Some(gdk_context)) + .set_id(texture_id as u32) + .set_width(width as i32) + .set_height(height as i32) + .set_format(format) + .set_sync(Some(sync_point)) + .build_with_release_func(move || { + // Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier + drop(frame); + }) + } + #[cfg(not(feature = "gtk_v4_12"))] + { + sync_meta.wait(wrapped_context); + + gdk::GLTexture::with_release_func( + gdk_context, + texture_id as u32, + width as i32, + height as i32, + move || { + // Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier + drop(frame); + }, + ) + } .upcast::() }; diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index 8017bb06..c3e291e1 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -48,7 +48,7 @@ enum GLContext { #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] static GL_CONTEXT: Mutex = Mutex::new(GLContext::Uninitialized); -static CAT: Lazy = Lazy::new(|| { +pub(crate) static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( "gtk4paintablesink", gst::DebugColorFlags::empty(), diff --git a/video/gtk4/src/sink/mod.rs b/video/gtk4/src/sink/mod.rs index 79282ee3..96ff9299 100644 --- a/video/gtk4/src/sink/mod.rs +++ b/video/gtk4/src/sink/mod.rs @@ -13,7 +13,7 @@ use gtk::glib; use gtk::glib::prelude::*; mod frame; -mod imp; +pub(super) mod imp; mod paintable; enum SinkEvent { diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index 5fc9eb9c..d051d42f 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -33,6 +33,7 @@ pub struct Paintable { paintables: RefCell>, cached_textures: RefCell>, gl_context: RefCell>, + #[cfg(not(feature = "gtk_v4_10"))] premult_shader: gsk::GLShader, } @@ -42,6 +43,7 @@ impl Default for Paintable { paintables: Default::default(), cached_textures: Default::default(), gl_context: Default::default(), + #[cfg(not(feature = "gtk_v4_10"))] premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!( "premult.glsl" ))), @@ -167,20 +169,78 @@ impl PaintableImpl for Paintable { let bounds = graphene::Rect::new(*x, *y, *paintable_width, *paintable_height); // Only premultiply GL textures that expect to be in premultiplied RGBA format. - let do_premult = texture.is::() && *has_alpha; - if do_premult { - snapshot.push_gl_shader( - &self.premult_shader, - &bounds, - gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(), - ); + // + // For GTK 4.14 or newer we use the correct format directly when building the + // texture, but only if a GLES3+ context is used. In that case the NGL renderer is + // used by GTK, which supports non-premultiplied formats correctly and fast. + // + // For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a + // self-mask to pre-multiply the alpha. + // + // For GTK before 4.10, we use a GL shader and hope that it works. + #[cfg(feature = "gtk_v4_10")] + { + let context_requires_premult = { + #[cfg(feature = "gtk_v4_14")] + { + self.gl_context.borrow().as_ref().map_or(false, |context| { + context.api() != gdk::GLAPI::GLES || context.version().0 < 3 + }) + } + + #[cfg(not(feature = "gtk_v4_14"))] + { + true + } + }; + + let do_premult = + context_requires_premult && texture.is::() && *has_alpha; + if do_premult { + snapshot.push_mask(gsk::MaskMode::Alpha); + snapshot.append_texture(texture, &bounds); + snapshot.pop(); // pop mask + + // color matrix to set alpha of the source to 1.0 as it was + // already applied via the mask just above. + snapshot.push_color_matrix( + &graphene::Matrix::from_float({ + [ + 1.0, 0.0, 0.0, 0.0, // + 0.0, 1.0, 0.0, 0.0, // + 0.0, 0.0, 1.0, 0.0, // + 0.0, 0.0, 0.0, 0.0, + ] + }), + &graphene::Vec4::new(0.0, 0.0, 0.0, 1.0), + ); + } + + snapshot.append_texture(texture, &bounds); + + if do_premult { + snapshot.pop(); // pop color matrix + snapshot.pop(); // pop mask 2 + } } + #[cfg(not(feature = "gtk_v4_10"))] + { + let do_premult = + texture.is::() && *has_alpha && gtk::micro_version() < 13; + if do_premult { + snapshot.push_gl_shader( + &self.premult_shader, + &bounds, + gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(), + ); + } - snapshot.append_texture(texture, &bounds); + snapshot.append_texture(texture, &bounds); - if do_premult { - snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader - snapshot.pop(); // pop shader + if do_premult { + snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader + snapshot.pop(); // pop shader + } } snapshot.pop(); // pop opacity