gtk4: Improve handling of RGBA GL textures in GTK

GTK 4.14 comes with a new GL renderer that does not support GL shader
nodes anymore, so the conversion from non-premultiplied alpha to
premultiplied alpha has to happen differently.

For GTK 4.14 or newer we use the correct format directly when building the
texture, but only if a GLES3+ context is used. In that case the NGL renderer is
used by GTK, which supports non-premultiplied formats correctly and fast.

For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
self-mask to pre-multiply the alpha.

For GTK before 4.10, we use a GL shader and hope that it works.

Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/488

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1452>

video/gtk4/Cargo.toml

@@ -52,6 +52,9 @@ x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"]
 winegl = ["gdk-win32/egl", "gst-gl-egl"]
 capi = []
 doc = ["gst/v1_18"]
+gtk_v4_10 = ["gtk/v4_10"]
+gtk_v4_12 = ["gtk/v4_12", "gtk_v4_10"]
+gtk_v4_14 = ["gtk/v4_14", "gtk_v4_12"]
 
 [package.metadata.capi]
 min_version = "0.9.21"

video/gtk4/src/lib.rs

@@ -22,6 +22,13 @@ mod utils;
 pub use sink::PaintableSink;
 
 fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    #[cfg(not(feature = "gtk_v4_10"))]
+    {
+        if gtk::micro_version() >= 13 {
+            gst::warning!(sink::imp::CAT, obj: plugin, "GTK 4.13 or newer detected but plugin not compiled with support for this version. Rendering of video frames with alpha will likely be wrong");
+        }
+    }
+
     sink::register(plugin)
 }
 

video/gtk4/src/sink/frame.rs

@@ -94,6 +94,18 @@ impl AsRef<[u8]> for FrameWrapper {
     }
 }
 
+fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat {
+    match f {
+        gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8,
+        gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8,
+        gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8,
+        gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8,
+        gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8,
+        gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8,
+        _ => unreachable!(),
+    }
+}
+
 fn video_frame_to_memory_texture(
     frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
     cached_textures: &mut HashMap<usize, gdk::Texture>,
@@ -109,15 +121,7 @@ fn video_frame_to_memory_texture(
         return (texture.clone(), pixel_aspect_ratio);
     }
 
-    let format = match frame.format() {
-        gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8,
-        gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8,
-        gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8,
-        gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8,
-        gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8,
-        gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8,
-        _ => unreachable!(),
-    };
+    let format = video_format_to_memory_format(frame.format());
     let width = frame.width();
     let height = frame.height();
     let rowstride = frame.plane_stride()[0] as usize;
@@ -143,7 +147,7 @@ fn video_frame_to_gl_texture(
     cached_textures: &mut HashMap<usize, gdk::Texture>,
     used_textures: &mut HashSet<usize>,
     gdk_context: &gdk::GLContext,
-    wrapped_context: &gst_gl::GLContext,
+    #[allow(unused)] wrapped_context: &gst_gl::GLContext,
 ) -> (gdk::Texture, f64) {
     let texture_id = frame.texture_id(0).expect("Invalid texture id") as usize;
 
@@ -159,19 +163,77 @@ fn video_frame_to_gl_texture(
     let height = frame.height();
 
     let sync_meta = frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
-    sync_meta.wait(wrapped_context);
 
     let texture = unsafe {
-        gdk::GLTexture::with_release_func(
-            gdk_context,
-            texture_id as u32,
-            width as i32,
-            height as i32,
-            move || {
-                // Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier
-                drop(frame);
-            },
-        )
+        #[cfg(feature = "gtk_v4_12")]
+        {
+            let format = {
+                let format = video_format_to_memory_format(frame.format());
+                #[cfg(feature = "gtk_v4_14")]
+                {
+                    use gtk::prelude::*;
+
+                    if gdk_context.api() != gdk::GLAPI::GLES || gdk_context.version().0 < 3 {
+                        // Map alpha formats to the pre-multiplied versions because we pre-multiply
+                        // ourselves if not GLES3 with the new GL renderer is used as the GTK GL
+                        // backend does not natively support non-premultiplied formats.
+                        match format {
+                            gdk::MemoryFormat::B8g8r8a8 => gdk::MemoryFormat::B8g8r8a8Premultiplied,
+                            gdk::MemoryFormat::A8r8g8b8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
+                            gdk::MemoryFormat::R8g8b8a8 => gdk::MemoryFormat::R8g8b8a8Premultiplied,
+                            gdk::MemoryFormat::A8b8g8r8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
+                            gdk::MemoryFormat::R8g8b8 | gdk::MemoryFormat::B8g8r8 => format,
+                            _ => unreachable!(),
+                        }
+                    } else {
+                        format
+                    }
+                }
+
+                #[cfg(not(feature = "gtk_v4_14"))]
+                {
+                    // Map alpha formats to the pre-multiplied versions because we pre-multiply
+                    // ourselves in pre-4.14 versions as the GTK GL backend does not natively
+                    // support non-premultiplied formats
+                    match format {
+                        gdk::MemoryFormat::B8g8r8a8 => gdk::MemoryFormat::B8g8r8a8Premultiplied,
+                        gdk::MemoryFormat::A8r8g8b8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
+                        gdk::MemoryFormat::R8g8b8a8 => gdk::MemoryFormat::R8g8b8a8Premultiplied,
+                        gdk::MemoryFormat::A8b8g8r8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
+                        gdk::MemoryFormat::R8g8b8 | gdk::MemoryFormat::B8g8r8 => format,
+                        _ => unreachable!(),
+                    }
+                }
+            };
+            let sync_point = (*sync_meta.as_ptr()).data;
+
+            gdk::GLTextureBuilder::new()
+                .set_context(Some(gdk_context))
+                .set_id(texture_id as u32)
+                .set_width(width as i32)
+                .set_height(height as i32)
+                .set_format(format)
+                .set_sync(Some(sync_point))
+                .build_with_release_func(move || {
+                    // Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier
+                    drop(frame);
+                })
+        }
+        #[cfg(not(feature = "gtk_v4_12"))]
+        {
+            sync_meta.wait(wrapped_context);
+
+            gdk::GLTexture::with_release_func(
+                gdk_context,
+                texture_id as u32,
+                width as i32,
+                height as i32,
+                move || {
+                    // Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier
+                    drop(frame);
+                },
+            )
+        }
         .upcast::<gdk::Texture>()
     };
 

video/gtk4/src/sink/imp.rs

@@ -48,7 +48,7 @@ enum GLContext {
 #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
 static GL_CONTEXT: Mutex<GLContext> = Mutex::new(GLContext::Uninitialized);
 
-static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
+pub(crate) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
     gst::DebugCategory::new(
         "gtk4paintablesink",
         gst::DebugColorFlags::empty(),

video/gtk4/src/sink/mod.rs

@@ -13,7 +13,7 @@ use gtk::glib;
 use gtk::glib::prelude::*;
 
 mod frame;
-mod imp;
+pub(super) mod imp;
 mod paintable;
 
 enum SinkEvent {

video/gtk4/src/sink/paintable/imp.rs

@@ -33,6 +33,7 @@ pub struct Paintable {
     paintables: RefCell<Vec<Texture>>,
     cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
     gl_context: RefCell<Option<gdk::GLContext>>,
+    #[cfg(not(feature = "gtk_v4_10"))]
     premult_shader: gsk::GLShader,
 }
 
@@ -42,6 +43,7 @@ impl Default for Paintable {
             paintables: Default::default(),
             cached_textures: Default::default(),
             gl_context: Default::default(),
+            #[cfg(not(feature = "gtk_v4_10"))]
             premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
                 "premult.glsl"
             ))),
@@ -167,20 +169,78 @@ impl PaintableImpl for Paintable {
                 let bounds = graphene::Rect::new(*x, *y, *paintable_width, *paintable_height);
 
                 // Only premultiply GL textures that expect to be in premultiplied RGBA format.
-                let do_premult = texture.is::<gdk::GLTexture>() && *has_alpha;
-                if do_premult {
-                    snapshot.push_gl_shader(
-                        &self.premult_shader,
-                        &bounds,
-                        gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
-                    );
+                //
+                // For GTK 4.14 or newer we use the correct format directly when building the
+                // texture, but only if a GLES3+ context is used. In that case the NGL renderer is
+                // used by GTK, which supports non-premultiplied formats correctly and fast.
+                //
+                // For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
+                // self-mask to pre-multiply the alpha.
+                //
+                // For GTK before 4.10, we use a GL shader and hope that it works.
+                #[cfg(feature = "gtk_v4_10")]
+                {
+                    let context_requires_premult = {
+                        #[cfg(feature = "gtk_v4_14")]
+                        {
+                            self.gl_context.borrow().as_ref().map_or(false, |context| {
+                                context.api() != gdk::GLAPI::GLES || context.version().0 < 3
+                            })
+                        }
+
+                        #[cfg(not(feature = "gtk_v4_14"))]
+                        {
+                            true
+                        }
+                    };
+
+                    let do_premult =
+                        context_requires_premult && texture.is::<gdk::GLTexture>() && *has_alpha;
+                    if do_premult {
+                        snapshot.push_mask(gsk::MaskMode::Alpha);
+                        snapshot.append_texture(texture, &bounds);
+                        snapshot.pop(); // pop mask
+
+                        // color matrix to set alpha of the source to 1.0 as it was
+                        // already applied via the mask just above.
+                        snapshot.push_color_matrix(
+                            &graphene::Matrix::from_float({
+                                [
+                                    1.0, 0.0, 0.0, 0.0, //
+                                    0.0, 1.0, 0.0, 0.0, //
+                                    0.0, 0.0, 1.0, 0.0, //
+                                    0.0, 0.0, 0.0, 0.0,
+                                ]
+                            }),
+                            &graphene::Vec4::new(0.0, 0.0, 0.0, 1.0),
+                        );
+                    }
+
+                    snapshot.append_texture(texture, &bounds);
+
+                    if do_premult {
+                        snapshot.pop(); // pop color matrix
+                        snapshot.pop(); // pop mask 2
+                    }
                 }
+                #[cfg(not(feature = "gtk_v4_10"))]
+                {
+                    let do_premult =
+                        texture.is::<gdk::GLTexture>() && *has_alpha && gtk::micro_version() < 13;
+                    if do_premult {
+                        snapshot.push_gl_shader(
+                            &self.premult_shader,
+                            &bounds,
+                            gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
+                        );
+                    }
 
-                snapshot.append_texture(texture, &bounds);
+                    snapshot.append_texture(texture, &bounds);
 
-                if do_premult {
-                    snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
-                    snapshot.pop(); // pop shader
+                    if do_premult {
+                        snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
+                        snapshot.pop(); // pop shader
+                    }
                 }
 
                 snapshot.pop(); // pop opacity