mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-05-13 13:52:43 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'gtk-dmabuf-import' into 'main'

Browse source 
gtk4: Implement support for directly importing dmabufs

Closes #441

See merge request gstreamer/gst-plugins-rs!1547
This commit is contained in:
Sebastian Dröge 2024-04-26 09:30:15 +00:00
parent 7573caa8e9 927c3fcdb6
commit 6fd86c22e9
12 changed files with 781 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
Cargo.lock generated
View file

@ -2454,6 +2454,7 @@ dependencies = [
"gdk4-x11",
"gst-plugin-version-helper",
"gstreamer",
"gstreamer-allocators",
"gstreamer-base",
"gstreamer-gl",
"gstreamer-gl-egl",
@ -3059,6 +3060,30 @@ dependencies = [
"thiserror",
]
[[package]]
name = "gstreamer-allocators"
version = "0.23.0"
source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae"
dependencies = [
"glib",
"gstreamer",
"gstreamer-allocators-sys",
"libc",
"once_cell",
]
[[package]]
name = "gstreamer-allocators-sys"
version = "0.23.0"
source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae"
dependencies = [
"glib-sys",
"gobject-sys",
"gstreamer-sys",
"libc",
"system-deps",
]
[[package]]
name = "gstreamer-app"
version = "0.23.0"

1
Cargo.toml
View file

@ -134,6 +134,7 @@ gdk-wayland = { package = "gdk4-wayland", git = "https://github.com/gtk-rs/gtk4-
gdk-x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"}
gdk-win32 = { package = "gdk4-win32", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"}
gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
gst-allocators = { package = "gstreamer-allocators", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
gst-app = { package = "gstreamer-app", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }

5
docs/plugins/gst_plugins_cache.json
View file

@ -2472,11 +2472,14 @@
"GInitiallyUnowned",
"GObject"
],
"interfaces": [
"GstChildProxy"
],
"klass": "Sink/Video",
"long-name": "GTK 4 Paintable Sink",
"pad-templates": {
"sink": {
"caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
"caps": "video/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\nvideo/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
"direction": "sink",
"presence": "always"
}

17
meson.build
View file

@ -304,6 +304,23 @@ if get_option('gtk4').allowed()
gtk4_features += 'winegl'
endif
endif
gst_allocators_dep = dependency('gstreamer-allocators-1.0', version: '>=1.24', required: false)
gtk_dep = dependency('gtk4', version: '>=4.6', required: get_option('gtk4'))
if gtk_dep.found()
if host_system == 'linux' and gtk_dep.version().version_compare('>=4.14') and gst_allocators_dep.found()
gtk4_features += 'dmabuf'
endif
if gtk_dep.version().version_compare('>=4.14')
gtk4_features += 'gtk_v4_14'
elif gtk_dep.version().version_compare('>=4.12')
gtk4_features += 'gtk_v4_12'
elif gtk_dep.version().version_compare('>=4.10')
gtk4_features += 'gtk_v4_10'
endif
endif
plugins += {
'gtk4': {
'library': 'libgstgtk4',

2
video/gtk4/Cargo.toml
View file

@ -17,6 +17,7 @@ gst = { workspace = true, features = ["v1_16"] }
gst-base.workspace = true
gst-video.workspace = true
gst-gl = { workspace = true, features = ["v1_16"], optional = true }
gst-allocators = { workspace = true, features = ["v1_24"], optional = true }
gst-gl-wayland = { workspace = true, features = ["v1_16"], optional = true }
gst-gl-x11 = { workspace = true, features = ["v1_16"], optional = true }
@ -50,6 +51,7 @@ wayland = ["gtk/v4_6", "gdk-wayland", "gst-gl", "gst-gl-wayland"]
x11glx = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-x11"]
x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"]
winegl = ["gdk-win32/egl", "gst-gl-egl"]
dmabuf = ["gst-allocators", "wayland", "gtk_v4_14", "gst-video/v1_24"]
capi = []
doc = ["gst/v1_18"]
gtk_v4_10 = ["gtk/v4_10"]

18
video/gtk4/README.md
View file

@ -1,10 +1,20 @@
# Gtk 4 Sink & Paintable
# GTK 4 Sink & Paintable
GTK 4 provides `gtk::Video` & `gtk::Picture` for rendering media such as videos. As the default `gtk::Video` widget doesn't
offer the possibility to use a custom `gst::Pipeline`. The plugin provides a `gst_video::VideoSink` along with a `gdk::Paintable` that's capable of rendering the sink's frames.
The Sink can generate GL Textures if the system is capable of it, but it needs to be compiled
with either `wayland`, `x11glx` or `x11egl` cargo features.
The sink can generate GL Textures if the system is capable of it, but it needs
to be compiled with either `wayland`, `x11glx` or `x11egl` cargo features. On
Windows and macOS this is enabled by default.
Additionally, the sink can render DMABufs directly on Linux if GTK 4.14 or
newer is used. For this the `dmabuf` feature needs to be enabled.
Depending on the GTK version that is used and should be supported as minimum,
new features or more efficient processing can be opted in with the `gtk_v4_10`,
`gtk_v4_12` and `gtk_v4_14` features. The minimum GTK version required by the
sink is GTK 4.4 on Linux without GL support, and 4.6 on Windows and macOS, and
on Linux with GL support.
# Flatpak Integration
@ -44,7 +54,7 @@ To build and include the plugin in a Flatpak manifest, you can add the following
{
"type": "git",
"url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs",
"branch": "0.10"
"branch": "0.12"
}
],
"build-options": {

27
video/gtk4/examples/gtksink.rs
View file

@ -6,13 +6,6 @@ use gtk::{gdk, gio, glib};
use std::cell::RefCell;
fn create_ui(app: &gtk::Application) {
let window = gtk::ApplicationWindow::new(app);
window.set_default_size(640, 480);
let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0);
let picture = gtk::Picture::new();
let label = gtk::Label::new(Some("Position: 00:00:00"));
let pipeline = gst::Pipeline::new();
let overlay = gst::ElementFactory::make("clockoverlay")
@ -64,8 +57,26 @@ fn create_ui(app: &gtk::Application) {
src.link_filtered(&overlay, &caps).unwrap();
overlay.link(&sink).unwrap();
let window = gtk::ApplicationWindow::new(app);
window.set_default_size(640, 480);
let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0);
let picture = gtk::Picture::new();
picture.set_paintable(Some(&paintable));
vbox.append(&picture);
#[cfg(feature = "gtk_v4_14")]
{
let offload = gtk::GraphicsOffload::new(Some(&picture));
offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled);
vbox.append(&offload);
}
#[cfg(not(feature = "gtk_v4_14"))]
{
vbox.append(&picture);
}
let label = gtk::Label::new(Some("Position: 00:00:00"));
vbox.append(&label);
window.set_child(Some(&vbox));

357
video/gtk4/src/sink/frame.rs
View file

@ -14,7 +14,61 @@ use gst_video::prelude::*;
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
use gst_gl::prelude::*;
use gtk::{gdk, glib};
use std::collections::{HashMap, HashSet};
use std::{
collections::{HashMap, HashSet},
ops,
};
#[derive(Clone, PartialEq, Eq, Debug)]
pub enum VideoInfo {
VideoInfo(gst_video::VideoInfo),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
DmaDrm(gst_video::VideoInfoDmaDrm),
}
impl From<gst_video::VideoInfo> for VideoInfo {
fn from(v: gst_video::VideoInfo) -> Self {
VideoInfo::VideoInfo(v)
}
}
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
impl From<gst_video::VideoInfoDmaDrm> for VideoInfo {
fn from(v: gst_video::VideoInfoDmaDrm) -> Self {
VideoInfo::DmaDrm(v)
}
}
impl ops::Deref for VideoInfo {
type Target = gst_video::VideoInfo;
fn deref(&self) -> &Self::Target {
match self {
VideoInfo::VideoInfo(info) => info,
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
VideoInfo::DmaDrm(info) => info,
}
}
}
impl VideoInfo {
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
fn dma_drm(&self) -> Option<&gst_video::VideoInfoDmaDrm> {
match self {
VideoInfo::VideoInfo(..) => None,
VideoInfo::DmaDrm(info) => Some(info),
}
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub enum TextureCacheId {
Memory(usize),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
GL(usize),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
DmaBuf([i32; 4]),
}
#[derive(Debug)]
enum MappedFrame {
@ -24,6 +78,17 @@ enum MappedFrame {
frame: gst_gl::GLVideoFrame<gst_gl::gl_video_frame::Readable>,
wrapped_context: gst_gl::GLContext,
},
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
DmaBuf {
buffer: gst::Buffer,
info: gst_video::VideoInfoDmaDrm,
n_planes: u32,
fds: [i32; 4],
offsets: [usize; 4],
strides: [usize; 4],
width: u32,
height: u32,
},
}
impl MappedFrame {
@ -32,6 +97,8 @@ impl MappedFrame {
MappedFrame::SysMem(frame) => frame.buffer(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.buffer(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf { buffer, .. } => buffer,
}
}
@ -40,6 +107,8 @@ impl MappedFrame {
MappedFrame::SysMem(frame) => frame.width(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.width(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf { info, .. } => info.width(),
}
}
@ -48,6 +117,8 @@ impl MappedFrame {
MappedFrame::SysMem(frame) => frame.height(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.height(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf { info, .. } => info.height(),
}
}
@ -56,6 +127,8 @@ impl MappedFrame {
MappedFrame::SysMem(frame) => frame.format_info(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.format_info(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf { info, .. } => info.format_info(),
}
}
}
@ -108,16 +181,16 @@ fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat
fn video_frame_to_memory_texture(
frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
cached_textures: &mut HashMap<usize, gdk::Texture>,
used_textures: &mut HashSet<usize>,
cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
used_textures: &mut HashSet<TextureCacheId>,
) -> (gdk::Texture, f64) {
let texture_id = frame.plane_data(0).unwrap().as_ptr() as usize;
let ptr = frame.plane_data(0).unwrap().as_ptr() as usize;
let pixel_aspect_ratio =
(frame.info().par().numer() as f64) / (frame.info().par().denom() as f64);
if let Some(texture) = cached_textures.get(&texture_id) {
used_textures.insert(texture_id);
if let Some(texture) = cached_textures.get(&TextureCacheId::Memory(ptr)) {
used_textures.insert(TextureCacheId::Memory(ptr));
return (texture.clone(), pixel_aspect_ratio);
}
@ -135,8 +208,8 @@ fn video_frame_to_memory_texture(
)
.upcast::<gdk::Texture>();
cached_textures.insert(texture_id, texture.clone());
used_textures.insert(texture_id);
cached_textures.insert(TextureCacheId::Memory(ptr), texture.clone());
used_textures.insert(TextureCacheId::Memory(ptr));
(texture, pixel_aspect_ratio)
}
@ -144,8 +217,8 @@ fn video_frame_to_memory_texture(
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
fn video_frame_to_gl_texture(
frame: gst_gl::GLVideoFrame<gst_gl::gl_video_frame::Readable>,
cached_textures: &mut HashMap<usize, gdk::Texture>,
used_textures: &mut HashSet<usize>,
cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
used_textures: &mut HashSet<TextureCacheId>,
gdk_context: &gdk::GLContext,
#[allow(unused)] wrapped_context: &gst_gl::GLContext,
) -> (gdk::Texture, f64) {
@ -154,8 +227,8 @@ fn video_frame_to_gl_texture(
let pixel_aspect_ratio =
(frame.info().par().numer() as f64) / (frame.info().par().denom() as f64);
if let Some(texture) = cached_textures.get(&(texture_id)) {
used_textures.insert(texture_id);
if let Some(texture) = cached_textures.get(&TextureCacheId::GL(texture_id)) {
used_textures.insert(TextureCacheId::GL(texture_id));
return (texture.clone(), pixel_aspect_ratio);
}
@ -237,18 +310,64 @@ fn video_frame_to_gl_texture(
.upcast::<gdk::Texture>()
};
cached_textures.insert(texture_id, texture.clone());
used_textures.insert(texture_id);
cached_textures.insert(TextureCacheId::GL(texture_id), texture.clone());
used_textures.insert(TextureCacheId::GL(texture_id));
(texture, pixel_aspect_ratio)
}
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
#[allow(clippy::too_many_arguments)]
fn video_frame_to_dmabuf_texture(
buffer: gst::Buffer,
cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
used_textures: &mut HashSet<TextureCacheId>,
info: &gst_video::VideoInfoDmaDrm,
n_planes: u32,
fds: &[i32; 4],
offsets: &[usize; 4],
strides: &[usize; 4],
width: u32,
height: u32,
) -> Result<(gdk::Texture, f64), glib::Error> {
let pixel_aspect_ratio = (info.par().numer() as f64) / (info.par().denom() as f64);
if let Some(texture) = cached_textures.get(&TextureCacheId::DmaBuf(*fds)) {
used_textures.insert(TextureCacheId::DmaBuf(*fds));
return Ok((texture.clone(), pixel_aspect_ratio));
}
let builder = gdk::DmabufTextureBuilder::new();
builder.set_display(&gdk::Display::default().unwrap());
builder.set_fourcc(info.fourcc());
builder.set_modifier(info.modifier());
builder.set_width(width);
builder.set_height(height);
builder.set_n_planes(n_planes);
for plane in 0..(n_planes as usize) {
builder.set_fd(plane as u32, fds[plane]);
builder.set_offset(plane as u32, offsets[plane] as u32);
builder.set_stride(plane as u32, strides[plane] as u32);
}
let texture = unsafe {
builder.build_with_release_func(move || {
drop(buffer);
})?
};
cached_textures.insert(TextureCacheId::DmaBuf(*fds), texture.clone());
used_textures.insert(TextureCacheId::DmaBuf(*fds));
Ok((texture, pixel_aspect_ratio))
}
impl Frame {
pub(crate) fn into_textures(
self,
#[allow(unused_variables)] gdk_context: Option<&gdk::GLContext>,
cached_textures: &mut HashMap<usize, gdk::Texture>,
) -> Vec<Texture> {
cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
) -> Result<Vec<Texture>, glib::Error> {
let mut textures = Vec::with_capacity(1 + self.overlays.len());
let mut used_textures = HashSet::with_capacity(1 + self.overlays.len());
@ -278,6 +397,28 @@ impl Frame {
&wrapped_context,
)
}
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf {
buffer,
info,
n_planes,
fds,
offsets,
strides,
width,
height,
} => video_frame_to_dmabuf_texture(
buffer,
cached_textures,
&mut used_textures,
&info,
n_planes,
&fds,
&offsets,
&strides,
width,
height,
)?,
};
textures.push(Texture {
@ -309,14 +450,14 @@ impl Frame {
// Remove textures that were not used this time
cached_textures.retain(|id, _| used_textures.contains(id));
textures
Ok(textures)
}
}
impl Frame {
pub(crate) fn new(
buffer: &gst::Buffer,
info: &gst_video::VideoInfo,
info: &VideoInfo,
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] wrapped_context: Option<
&gst_gl::GLContext,
>,
@ -327,77 +468,125 @@ impl Frame {
// Empty buffers get filtered out in show_frame
debug_assert!(buffer.n_memory() > 0);
let mut frame;
#[allow(unused_mut)]
let mut frame = None;
#[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst-gl")))]
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
{
frame = Self {
frame: MappedFrame::SysMem(
// Check we received a buffer with dmabuf memory and if so do some checks before
// passing it onwards
if frame.is_none()
&& buffer
.peek_memory(0)
.is_memory_type::<gst_allocators::DmaBufMemory>()
{
if let Some((vmeta, info)) =
Option::zip(buffer.meta::<gst_video::VideoMeta>(), info.dma_drm())
{
let mut fds = [-1i32; 4];
let mut offsets = [0; 4];
let mut strides = [0; 4];
let n_planes = vmeta.n_planes() as usize;
let vmeta_offsets = vmeta.offset();
let vmeta_strides = vmeta.stride();
for plane in 0..n_planes {
let Some((range, skip)) =
buffer.find_memory(vmeta_offsets[plane]..(vmeta_offsets[plane] + 1))
else {
break;
};
let mem = buffer.peek_memory(range.start);
let Some(mem) = mem.downcast_memory_ref::<gst_allocators::DmaBufMemory>()
else {
break;
};
let fd = mem.fd();
fds[plane] = fd;
offsets[plane] = mem.offset() + skip;
strides[plane] = vmeta_strides[plane] as usize;
}
// All fds valid?
if fds[0..n_planes].iter().all(|fd| *fd != -1) {
frame = Some(MappedFrame::DmaBuf {
buffer: buffer.clone(),
info: info.clone(),
n_planes: n_planes as u32,
fds,
offsets,
strides,
width: vmeta.width(),
height: vmeta.height(),
});
}
}
}
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
{
if frame.is_none() {
// Check we received a buffer with GL memory and if the context of that memory
// can share with the wrapped context around the GDK GL context.
//
// If not it has to be uploaded to the GPU.
let memory_ctx = buffer
.peek_memory(0)
.downcast_memory_ref::<gst_gl::GLBaseMemory>()
.and_then(|m| {
let ctx = m.context();
if wrapped_context
.map_or(false, |wrapped_context| wrapped_context.can_share(ctx))
{
Some(ctx)
} else {
None
}
});
if let Some(memory_ctx) = memory_ctx {
// If there is no GLSyncMeta yet then we need to add one here now, which requires
// obtaining a writable buffer.
let mapped_frame = if buffer.meta::<gst_gl::GLSyncMeta>().is_some() {
gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?
} else {
let mut buffer = buffer.clone();
{
let buffer = buffer.make_mut();
gst_gl::GLSyncMeta::add(buffer, memory_ctx);
}
gst_gl::GLVideoFrame::from_buffer_readable(buffer, info)
.map_err(|_| gst::FlowError::Error)?
};
// Now that it's guaranteed that there is a sync meta and the frame is mapped, set
// a sync point so we can ensure that the texture is ready later when making use of
// it as gdk::GLTexture.
let meta = mapped_frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
meta.set_sync_point(memory_ctx);
frame = Some(MappedFrame::GL {
frame: mapped_frame,
wrapped_context: wrapped_context.unwrap().clone(),
});
}
}
}
}
let mut frame = Self {
frame: match frame {
Some(frame) => frame,
None => MappedFrame::SysMem(
gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?,
),
overlays: vec![],
};
}
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
{
// Check we received a buffer with GL memory and if the context of that memory
// can share with the wrapped context around the GDK GL context.
//
// If not it has to be uploaded to the GPU.
let memory_ctx = buffer
.peek_memory(0)
.downcast_memory_ref::<gst_gl::GLBaseMemory>()
.and_then(|m| {
let ctx = m.context();
if wrapped_context
.map_or(false, |wrapped_context| wrapped_context.can_share(ctx))
{
Some(ctx)
} else {
None
}
});
if let Some(memory_ctx) = memory_ctx {
// If there is no GLSyncMeta yet then we need to add one here now, which requires
// obtaining a writable buffer.
let mapped_frame = if buffer.meta::<gst_gl::GLSyncMeta>().is_some() {
gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?
} else {
let mut buffer = buffer.clone();
{
let buffer = buffer.make_mut();
gst_gl::GLSyncMeta::add(buffer, memory_ctx);
}
gst_gl::GLVideoFrame::from_buffer_readable(buffer, info)
.map_err(|_| gst::FlowError::Error)?
};
// Now that it's guaranteed that there is a sync meta and the frame is mapped, set
// a sync point so we can ensure that the texture is ready later when making use of
// it as gdk::GLTexture.
let meta = mapped_frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
meta.set_sync_point(memory_ctx);
frame = Self {
frame: MappedFrame::GL {
frame: mapped_frame,
wrapped_context: wrapped_context.unwrap().clone(),
},
overlays: vec![],
};
} else {
frame = Self {
frame: MappedFrame::SysMem(
gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?,
),
overlays: vec![],
};
}
}
},
overlays: vec![],
};
frame.overlays = frame
.frame

285
video/gtk4/src/sink/imp.rs
View file

@ -1,7 +1,7 @@
//
// Copyright (C) 2021 Bilal Elmoussaoui <bil.elmoussaoui@gmail.com>
// Copyright (C) 2021 Jordan Petridis <jordan@centricular.com>
// Copyright (C) 2021 Sebastian Dröge <sebastian@centricular.com>
// Copyright (C) 2021-2024 Sebastian Dröge <sebastian@centricular.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
@ -62,7 +62,7 @@ pub(crate) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
pub struct PaintableSink {
paintable: Mutex<Option<ThreadGuard<Paintable>>>,
window: Mutex<Option<ThreadGuard<gtk::Window>>>,
info: Mutex<Option<gst_video::VideoInfo>>,
info: Mutex<Option<super::frame::VideoInfo>>,
sender: Mutex<Option<async_channel::Sender<SinkEvent>>>,
pending_frame: Mutex<Option<Frame>>,
cached_caps: Mutex<Option<gst::Caps>>,
@ -82,6 +82,7 @@ impl ObjectSubclass for PaintableSink {
const NAME: &'static str = "GstGtk4PaintableSink";
type Type = super::PaintableSink;
type ParentType = gst_video::VideoSink;
type Interfaces = (gst::ChildProxy,);
}
impl ObjectImpl for PaintableSink {
@ -110,12 +111,14 @@ impl ObjectImpl for PaintableSink {
return None::<&gdk::Paintable>.to_value();
}
let mut paintable = self.paintable.lock().unwrap();
if paintable.is_none() {
self.create_paintable(&mut paintable);
let mut paintable_guard = self.paintable.lock().unwrap();
let mut created = false;
if paintable_guard.is_none() {
created = true;
self.create_paintable(&mut paintable_guard);
}
let paintable = match &*paintable {
let paintable = match &*paintable_guard {
Some(ref paintable) => paintable,
None => {
gst::error!(CAT, imp: self, "Failed to create paintable");
@ -124,16 +127,31 @@ impl ObjectImpl for PaintableSink {
};
// Getter must be called from the main thread
if paintable.is_owner() {
paintable.get_ref().to_value()
} else {
if !paintable.is_owner() {
gst::error!(
CAT,
imp: self,
"Can't retrieve Paintable from non-main thread"
);
None::<&gdk::Paintable>.to_value()
return None::<&gdk::Paintable>.to_value();
}
let paintable = paintable.get_ref().clone();
drop(paintable_guard);
if created {
let self_ = self.to_owned();
glib::MainContext::default().invoke(move || {
let paintable_guard = self_.paintable.lock().unwrap();
if let Some(paintable) = &*paintable_guard {
let paintable_clone = paintable.get_ref().clone();
drop(paintable_guard);
self_.obj().child_added(&paintable_clone, "paintable");
}
});
}
paintable.to_value()
}
_ => unimplemented!(),
}
@ -163,53 +181,99 @@ impl ElementImpl for PaintableSink {
{
let caps = caps.get_mut().unwrap();
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
{
for features in [
[
gst_allocators::CAPS_FEATURE_MEMORY_DMABUF,
gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
]
.as_slice(),
[gst_allocators::CAPS_FEATURE_MEMORY_DMABUF].as_slice(),
] {
let c = gst_video::VideoCapsBuilder::new()
.format(gst_video::VideoFormat::DmaDrm)
.features(features.iter().copied())
.build();
caps.append(c);
}
}
for features in [
None,
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
Some(gst::CapsFeatures::new([
"memory:GLMemory",
"meta:GstVideoOverlayComposition",
gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY,
gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
])),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
Some(gst::CapsFeatures::new(["memory:GLMemory"])),
Some(gst::CapsFeatures::new([
gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY,
])),
Some(gst::CapsFeatures::new([
"memory:SystemMemory",
"meta:GstVideoOverlayComposition",
gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
])),
Some(gst::CapsFeatures::new(["meta:GstVideoOverlayComposition"])),
Some(gst::CapsFeatures::new([
gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
])),
None,
] {
const GL_FORMATS: &[gst_video::VideoFormat] =
&[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb];
const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[
gst_video::VideoFormat::Bgra,
gst_video::VideoFormat::Argb,
gst_video::VideoFormat::Rgba,
gst_video::VideoFormat::Abgr,
gst_video::VideoFormat::Rgb,
gst_video::VideoFormat::Bgr,
];
let formats = if features
.as_ref()
.is_some_and(|features| features.contains("memory:GLMemory"))
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
{
GL_FORMATS
} else {
NON_GL_FORMATS
};
const GL_FORMATS: &[gst_video::VideoFormat] =
&[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb];
const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[
gst_video::VideoFormat::Bgra,
gst_video::VideoFormat::Argb,
gst_video::VideoFormat::Rgba,
gst_video::VideoFormat::Abgr,
gst_video::VideoFormat::Rgb,
gst_video::VideoFormat::Bgr,
];
let mut c = gst_video::video_make_raw_caps(formats).build();
let formats = if features.as_ref().is_some_and(|features| {
features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY)
}) {
GL_FORMATS
} else {
NON_GL_FORMATS
};
if let Some(features) = features {
let c = c.get_mut().unwrap();
let mut c = gst_video::video_make_raw_caps(formats).build();
if features.contains("memory:GLMemory") {
c.set("texture-target", "2D")
if let Some(features) = features {
let c = c.get_mut().unwrap();
if features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) {
c.set("texture-target", "2D")
}
c.set_features_simple(Some(features));
}
c.set_features_simple(Some(features));
caps.append(c);
}
#[cfg(not(any(
target_os = "macos",
target_os = "windows",
feature = "gst-gl"
)))]
{
const FORMATS: &[gst_video::VideoFormat] = &[
gst_video::VideoFormat::Bgra,
gst_video::VideoFormat::Argb,
gst_video::VideoFormat::Rgba,
gst_video::VideoFormat::Abgr,
gst_video::VideoFormat::Rgb,
gst_video::VideoFormat::Bgr,
];
caps.append(c);
let mut c = gst_video::video_make_raw_caps(FORMATS).build();
if let Some(features) = features {
let c = c.get_mut().unwrap();
c.set_features_simple(Some(features));
}
caps.append(c);
}
}
}
@ -244,18 +308,31 @@ impl ElementImpl for PaintableSink {
}
}
let mut paintable = self.paintable.lock().unwrap();
if paintable.is_none() {
self.create_paintable(&mut paintable);
let mut paintable_guard = self.paintable.lock().unwrap();
let mut created = false;
if paintable_guard.is_none() {
created = true;
self.create_paintable(&mut paintable_guard);
}
if paintable.is_none() {
if paintable_guard.is_none() {
gst::error!(CAT, imp: self, "Failed to create paintable");
return Err(gst::StateChangeError);
}
drop(paintable);
drop(paintable_guard);
if created {
let self_ = self.to_owned();
glib::MainContext::default().invoke(move || {
let paintable_guard = self_.paintable.lock().unwrap();
if let Some(paintable) = &*paintable_guard {
let paintable_clone = paintable.get_ref().clone();
drop(paintable_guard);
self_.obj().child_added(&paintable_clone, "paintable");
}
});
}
// Notify the pipeline about the GL display and wrapped context so that any other
// elements in the pipeline ideally use the same / create GL contexts that are
@ -361,8 +438,21 @@ impl BaseSinkImpl for PaintableSink {
fn set_caps(&self, caps: &gst::Caps) -> Result<(), gst::LoggableError> {
gst::debug!(CAT, imp: self, "Setting caps {caps:?}");
let video_info = gst_video::VideoInfo::from_caps(caps)
.map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))?;
#[allow(unused_mut)]
let mut video_info = None;
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
{
if let Ok(info) = gst_video::VideoInfoDmaDrm::from_caps(caps) {
video_info = Some(info.into());
}
}
let video_info = match video_info {
Some(info) => info,
None => gst_video::VideoInfo::from_caps(caps)
.map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))?
.into(),
};
self.info.lock().unwrap().replace(video_info);
@ -516,10 +606,11 @@ impl PaintableSink {
match action {
SinkEvent::FrameChanged => {
let Some(frame) = self.pending_frame() else {
return glib::ControlFlow::Continue;
};
gst::trace!(CAT, imp: self, "Frame changed");
paintable
.get_ref()
.handle_frame_changed(self.pending_frame())
paintable.get_ref().handle_frame_changed(&self.obj(), frame);
}
}
@ -530,13 +621,59 @@ impl PaintableSink {
#[allow(unused_mut)]
let mut tmp_caps = Self::pad_templates()[0].caps().clone();
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
{
let formats = utils::invoke_on_main_thread(move || {
let Some(display) = gdk::Display::default() else {
return vec![];
};
let dmabuf_formats = display.dmabuf_formats();
let mut formats = vec![];
let n_formats = dmabuf_formats.n_formats();
for i in 0..n_formats {
let (fourcc, modifier) = dmabuf_formats.format(i);
if fourcc == 0 || modifier == (u64::MAX >> 8) {
continue;
}
formats.push(gst_video::dma_drm_fourcc_to_string(fourcc, modifier));
}
formats
});
if formats.is_empty() {
// Filter out dmabufs caps from the template pads if we have no supported formats
if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) {
tmp_caps = tmp_caps
.iter_with_features()
.filter(|(_, features)| {
!features.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF)
})
.map(|(s, c)| (s.to_owned(), c.to_owned()))
.collect::<gst::Caps>();
}
} else {
let tmp_caps = tmp_caps.make_mut();
for (s, f) in tmp_caps.iter_with_features_mut() {
if f.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF) {
s.set("drm-format", gst::List::new(&formats));
}
}
}
}
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
{
// Filter out GL caps from the template pads if we have no context
if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) {
tmp_caps = tmp_caps
.iter_with_features()
.filter(|(_, features)| !features.contains("memory:GLMemory"))
.filter(|(_, features)| {
!features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY)
})
.map(|(s, c)| (s.to_owned(), c.to_owned()))
.collect::<gst::Caps>();
}
@ -564,7 +701,17 @@ impl PaintableSink {
let window = gtk::Window::new();
let picture = gtk::Picture::new();
picture.set_paintable(Some(&paintable));
window.set_child(Some(&picture));
#[cfg(feature = "gtk_v4_14")]
{
let offload = gtk::GraphicsOffload::new(Some(&picture));
offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled);
window.set_child(Some(&offload));
}
#[cfg(not(feature = "gtk_v4_14"))]
{
window.set_child(Some(&picture));
}
window.set_default_size(640, 480);
window.connect_close_request({
@ -1073,3 +1220,33 @@ impl PaintableSink {
}
}
}
impl ChildProxyImpl for PaintableSink {
fn child_by_index(&self, index: u32) -> Option<glib::Object> {
if index != 0 {
return None;
}
let paintable = self.paintable.lock().unwrap();
paintable
.as_ref()
.filter(|p| p.is_owner())
.map(|p| p.get_ref().upcast_ref::<glib::Object>().clone())
}
fn child_by_name(&self, name: &str) -> Option<glib::Object> {
if name == "paintable" {
return self.child_by_index(0);
}
None
}
fn children_count(&self) -> u32 {
let paintable = self.paintable.lock().unwrap();
if paintable.is_some() {
1
} else {
0
}
}
}

3
video/gtk4/src/sink/mod.rs
View file

@ -22,7 +22,8 @@ enum SinkEvent {
glib::wrapper! {
pub struct PaintableSink(ObjectSubclass<imp::PaintableSink>)
@extends gst_video::VideoSink, gst_base::BaseSink, gst::Element, gst::Object;
@extends gst_video::VideoSink, gst_base::BaseSink, gst::Element, gst::Object,
@implements gst::ChildProxy;
}
impl PaintableSink {

339
video/gtk4/src/sink/paintable/imp.rs
View file

@ -31,12 +31,13 @@ static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
#[derive(Debug)]
pub struct Paintable {
paintables: RefCell<Vec<Texture>>,
cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
cached_textures: RefCell<HashMap<super::super::frame::TextureCacheId, gdk::Texture>>,
gl_context: RefCell<Option<gdk::GLContext>>,
background_color: Cell<gdk::RGBA>,
#[cfg(feature = "gtk_v4_10")]
scaling_filter: Cell<gsk::ScalingFilter>,
use_scaling_filter: Cell<bool>,
force_aspect_ratio: Cell<bool>,
#[cfg(not(feature = "gtk_v4_10"))]
premult_shader: gsk::GLShader,
}
@ -51,6 +52,7 @@ impl Default for Paintable {
#[cfg(feature = "gtk_v4_10")]
scaling_filter: Cell::new(gsk::ScalingFilter::Linear),
use_scaling_filter: Cell::new(false),
force_aspect_ratio: Cell::new(false),
#[cfg(not(feature = "gtk_v4_10"))]
premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
"premult.glsl"
@ -94,6 +96,11 @@ impl ObjectImpl for Paintable {
.blurb("Use selected scaling filter or GTK default for rendering")
.default_value(false)
.build(),
glib::ParamSpecBoolean::builder("force-aspect-ratio")
.nick("Force Aspect Ratio")
.blurb("When enabled, scaling will respect original aspect ratio")
.default_value(true)
.build(),
]
});
@ -117,6 +124,7 @@ impl ObjectImpl for Paintable {
"scaling-filter" => self.scaling_filter.get().to_value(),
#[cfg(feature = "gtk_v4_10")]
"use-scaling-filter" => self.use_scaling_filter.get().to_value(),
"force-aspect-ratio" => self.force_aspect_ratio.get().to_value(),
_ => unimplemented!(),
}
}
@ -139,6 +147,7 @@ impl ObjectImpl for Paintable {
"scaling-filter" => self.scaling_filter.set(value.get().unwrap()),
#[cfg(feature = "gtk_v4_10")]
"use-scaling-filter" => self.use_scaling_filter.set(value.get().unwrap()),
"force-aspect-ratio" => self.force_aspect_ratio.set(value.get().unwrap()),
_ => unimplemented!(),
}
}
@ -173,40 +182,66 @@ impl PaintableImpl for Paintable {
let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap();
let background_color = self.background_color.get();
let force_aspect_ratio = self.force_aspect_ratio.get();
let paintables = self.paintables.borrow();
if !paintables.is_empty() {
gst::trace!(CAT, imp: self, "Snapshotting frame");
let (frame_width, frame_height) =
paintables.first().map(|p| (p.width, p.height)).unwrap();
let mut scale_x = width / frame_width as f64;
let mut scale_y = height / frame_height as f64;
let mut trans_x = 0.0;
let mut trans_y = 0.0;
// TODO: Property for keeping aspect ratio or not
if (scale_x - scale_y).abs() > f64::EPSILON {
if scale_x > scale_y {
trans_x =
((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0;
scale_x = scale_y;
} else {
trans_y =
((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0;
scale_y = scale_x;
}
}
let Some(first_paintable) = paintables.first() else {
gst::trace!(CAT, imp: self, "Snapshotting black frame");
snapshot.append_color(
&background_color,
&graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
);
snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32));
return;
};
for Texture {
gst::trace!(CAT, imp: self, "Snapshotting frame");
// The first paintable is the actual video frame and defines the overall size.
//
// Based on its size relative to the snapshot width/height, all other paintables are
// scaled accordingly.
let (frame_width, frame_height) = (first_paintable.width, first_paintable.height);
let mut scale_x = width / frame_width as f64;
let mut scale_y = height / frame_height as f64;
// Usually the caller makes sure that the aspect ratio is preserved. To enforce this here
// optionally, we scale the frame equally in both directions and center it. In addition the
// background color is drawn behind the frame to fill the gaps.
//
// This is not done by default for performance reasons and usually would draw a <1px
// background.
if force_aspect_ratio {
let mut trans_x = 0.0;
let mut trans_y = 0.0;
if (scale_x - scale_y).abs() > f64::EPSILON {
if scale_x > scale_y {
trans_x = (width - (frame_width as f64 * scale_y)) / 2.0;
scale_x = scale_y;
} else {
trans_y = (height - (frame_height as f64 * scale_x)) / 2.0;
scale_y = scale_x;
}
}
if !background_color.is_clear() && (trans_x > f64::EPSILON || trans_y > f64::EPSILON) {
snapshot.append_color(
&background_color,
&graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
);
}
snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32));
}
// Make immutable
let scale_x = scale_x;
let scale_y = scale_y;
for (
idx,
Texture {
texture,
x,
y,
@ -214,151 +249,159 @@ impl PaintableImpl for Paintable {
height: paintable_height,
global_alpha,
has_alpha,
} in &*paintables
},
) in paintables.iter().enumerate()
{
snapshot.push_opacity(*global_alpha as f64);
let bounds = if !force_aspect_ratio && idx == 0 {
// While this should end up with width again, be explicit in this case to avoid
// rounding errors and fill the whole area with the video frame.
graphene::Rect::new(0.0, 0.0, width as f32, height as f32)
} else {
// Scale texture position and size with the same scale factor as the main video
// frame, and make sure to not render outside (0, 0, width, height).
let x = f32::clamp(*x * scale_x as f32, 0.0, width as f32);
let y = f32::clamp(*y * scale_y as f32, 0.0, height as f32);
let texture_width = f32::min(*paintable_width * scale_x as f32, width as f32);
let texture_height = f32::min(*paintable_height * scale_y as f32, height as f32);
graphene::Rect::new(x, y, texture_width, texture_height)
};
// Only premultiply GL textures that expect to be in premultiplied RGBA format.
//
// For GTK 4.14 or newer we use the correct format directly when building the
// texture, but only if a GLES3+ context is used. In that case the NGL renderer is
// used by GTK, which supports non-premultiplied formats correctly and fast.
//
// For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
// self-mask to pre-multiply the alpha.
//
// For GTK before 4.10, we use a GL shader and hope that it works.
#[cfg(feature = "gtk_v4_10")]
{
snapshot.push_opacity(*global_alpha as f64);
let texture_width = *paintable_width * scale_x as f32;
let texture_height = *paintable_height * scale_y as f32;
let x = *x * scale_x as f32;
let y = *y * scale_y as f32;
let bounds = graphene::Rect::new(x, y, texture_width, texture_height);
// Only premultiply GL textures that expect to be in premultiplied RGBA format.
//
// For GTK 4.14 or newer we use the correct format directly when building the
// texture, but only if a GLES3+ context is used. In that case the NGL renderer is
// used by GTK, which supports non-premultiplied formats correctly and fast.
//
// For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
// self-mask to pre-multiply the alpha.
//
// For GTK before 4.10, we use a GL shader and hope that it works.
#[cfg(feature = "gtk_v4_10")]
{
let context_requires_premult = {
#[cfg(feature = "gtk_v4_14")]
{
self.gl_context.borrow().as_ref().map_or(false, |context| {
context.api() != gdk::GLAPI::GLES || context.version().0 < 3
})
}
#[cfg(not(feature = "gtk_v4_14"))]
{
true
}
};
let do_premult =
context_requires_premult && texture.is::<gdk::GLTexture>() && *has_alpha;
if do_premult {
snapshot.push_mask(gsk::MaskMode::Alpha);
if self.use_scaling_filter.get() {
#[cfg(feature = "gtk_v4_10")]
snapshot.append_scaled_texture(
texture,
self.scaling_filter.get(),
&bounds,
);
} else {
snapshot.append_texture(texture, &bounds);
}
snapshot.pop(); // pop mask
// color matrix to set alpha of the source to 1.0 as it was
// already applied via the mask just above.
snapshot.push_color_matrix(
&graphene::Matrix::from_float({
[
1.0, 0.0, 0.0, 0.0, //
0.0, 1.0, 0.0, 0.0, //
0.0, 0.0, 1.0, 0.0, //
0.0, 0.0, 0.0, 0.0,
]
}),
&graphene::Vec4::new(0.0, 0.0, 0.0, 1.0),
);
let context_requires_premult = {
#[cfg(feature = "gtk_v4_14")]
{
self.gl_context.borrow().as_ref().map_or(false, |context| {
context.api() != gdk::GLAPI::GLES || context.version().0 < 3
})
}
#[cfg(not(feature = "gtk_v4_14"))]
{
true
}
};
let do_premult =
context_requires_premult && texture.is::<gdk::GLTexture>() && *has_alpha;
if do_premult {
snapshot.push_mask(gsk::MaskMode::Alpha);
if self.use_scaling_filter.get() {
#[cfg(feature = "gtk_v4_10")]
snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
} else {
snapshot.append_texture(texture, &bounds);
}
snapshot.pop(); // pop mask
if do_premult {
snapshot.pop(); // pop color matrix
snapshot.pop(); // pop mask 2
}
}
#[cfg(not(feature = "gtk_v4_10"))]
{
let do_premult =
texture.is::<gdk::GLTexture>() && *has_alpha && gtk::micro_version() < 13;
if do_premult {
snapshot.push_gl_shader(
&self.premult_shader,
&bounds,
gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
);
}
if self.use_scaling_filter.get() {
#[cfg(feature = "gtk_v4_10")]
snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
} else {
snapshot.append_texture(texture, &bounds);
}
if do_premult {
snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
snapshot.pop(); // pop shader
}
// color matrix to set alpha of the source to 1.0 as it was
// already applied via the mask just above.
snapshot.push_color_matrix(
&graphene::Matrix::from_float({
[
1.0, 0.0, 0.0, 0.0, //
0.0, 1.0, 0.0, 0.0, //
0.0, 0.0, 1.0, 0.0, //
0.0, 0.0, 0.0, 0.0,
]
}),
&graphene::Vec4::new(0.0, 0.0, 0.0, 1.0),
);
}
snapshot.pop(); // pop opacity
if self.use_scaling_filter.get() {
#[cfg(feature = "gtk_v4_10")]
snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
} else {
snapshot.append_texture(texture, &bounds);
}
if do_premult {
snapshot.pop(); // pop color matrix
snapshot.pop(); // pop mask 2
}
}
} else {
gst::trace!(CAT, imp: self, "Snapshotting black frame");
snapshot.append_color(
&background_color,
&graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
);
#[cfg(not(feature = "gtk_v4_10"))]
{
let do_premult =
texture.is::<gdk::GLTexture>() && *has_alpha && gtk::micro_version() < 13;
if do_premult {
snapshot.push_gl_shader(
&self.premult_shader,
&bounds,
gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
);
}
if self.use_scaling_filter.get() {
#[cfg(feature = "gtk_v4_10")]
snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
} else {
snapshot.append_texture(texture, &bounds);
}
if do_premult {
snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
snapshot.pop(); // pop shader
}
}
snapshot.pop(); // pop opacity
}
}
}
impl Paintable {
pub(super) fn handle_frame_changed(&self, frame: Option<Frame>) {
pub(super) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) {
let context = self.gl_context.borrow();
if let Some(frame) = frame {
gst::trace!(CAT, imp: self, "Received new frame");
let new_paintables =
frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut());
let new_size = new_paintables
.first()
.map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32))
.unwrap();
gst::trace!(CAT, imp: self, "Received new frame");
let old_paintables = self.paintables.replace(new_paintables);
let old_size = old_paintables
.first()
.map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32));
let new_paintables =
match frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()) {
Ok(textures) => textures,
Err(err) => {
gst::element_error!(
sink,
gst::ResourceError::Failed,
["Failed to transform frame into textures: {err}"]
);
return;
}
};
if Some(new_size) != old_size {
gst::debug!(
CAT,
imp: self,
"Size changed from {old_size:?} to {new_size:?}",
);
self.obj().invalidate_size();
}
let new_size = new_paintables
.first()
.map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32))
.unwrap();
self.obj().invalidate_contents();
let old_paintables = self.paintables.replace(new_paintables);
let old_size = old_paintables
.first()
.map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32));
if Some(new_size) != old_size {
gst::debug!(
CAT,
imp: self,
"Size changed from {old_size:?} to {new_size:?}",
);
self.obj().invalidate_size();
}
self.obj().invalidate_contents();
}
pub(super) fn handle_flush_frames(&self) {

4
video/gtk4/src/sink/paintable/mod.rs
View file

@ -30,8 +30,8 @@ impl Paintable {
}
impl Paintable {
pub(crate) fn handle_frame_changed(&self, frame: Option<Frame>) {
self.imp().handle_frame_changed(frame);
pub(crate) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) {
self.imp().handle_frame_changed(sink, frame);
}
pub(crate) fn handle_flush_frames(&self) {