mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-06-12 11:09:37 +00:00
Code Issues 1 Projects Releases Wiki Activity

vulkan: add vulkan overlay compositor element

Browse source 
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2470>
This commit is contained in:
Matthew Waters 2022-05-23 11:41:17 +10:00 committed by GStreamer Marge Bot
parent efe8e41b61
commit 9df7a21ec9
5 changed files with 862 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json
View file

@ -229776,6 +229776,34 @@
"properties": {},
"rank": "none"
},
"vulkanoverlaycompositor": {
"author": "Matthew Waters <matthew@centricular.com>",
"description": "Vulkan Overlay Composition element",
"hierarchy": [
"GstVulkanOverlayCompositor",
"GstVulkanVideoFilter",
"GstBaseTransform",
"GstElement",
"GstObject",
"GInitiallyUnowned",
"GObject"
],
"klass": "Filter/Video",
"long-name": "Vulkan Overlay Compositor",
"pad-templates": {
"sink": {
"caps": "video/x-raw(memory:VulkanImage, meta:GstVideoOverlayComposition):\n format: { BGRA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:VulkanImage):\n format: { BGRA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(ANY):\n format: { BGRA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
"direction": "sink",
"presence": "always"
},
"src": {
"caps": "video/x-raw(memory:VulkanImage, meta:GstVideoOverlayComposition):\n format: { BGRA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:VulkanImage):\n format: { BGRA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(ANY):\n format: { BGRA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
"direction": "src",
"presence": "always"
}
},
"rank": "none"
},
"vulkanshaderspv": {
"author": "Martin Reboredo <yakoyoku@gmail.com>",
"description": "Performs operations with SPIRV shaders in Vulkan",

3
subprojects/gst-plugins-bad/ext/vulkan/gstvulkan.c
View file

@ -38,6 +38,7 @@
#include "vkviewconvert.h"
#include "vkdeviceprovider.h"
#include "gstvulkanelements.h"
#include "vkoverlaycompositor.h"
static gboolean
@ -61,6 +62,8 @@ plugin_init (GstPlugin * plugin)
ret |= GST_ELEMENT_REGISTER (vulkanviewconvert, plugin);
ret |= GST_ELEMENT_REGISTER (vulkanoverlaycompositor, plugin);
return ret;
}

1
subprojects/gst-plugins-bad/ext/vulkan/meson.build
View file

@ -26,6 +26,7 @@ vulkan_sources = [
'vksink.c',
'vkupload.c',
'vkviewconvert.c',
'vkoverlaycompositor.c',
]
vulkan_plugin_enum_headers = [

794
subprojects/gst-plugins-bad/ext/vulkan/vkoverlaycompositor.c Normal file
View file

@ -0,0 +1,794 @@
/*
* GStreamer
* Copyright (C) 2022 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-vulkanoverlaycompositor
* @title: vulkanoverlaycompositor
*
* `vulkanoverlaycompositor` overlays upstream `GstVideoOverlayCompositonMeta`
* onto the video stream.
*
* Since: 1.22
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <string.h>
#include "gstvulkanelements.h"
#include "vkoverlaycompositor.h"
#include "shaders/identity.vert.h"
#include "shaders/swizzle.frag.h"
GST_DEBUG_CATEGORY (gst_debug_vulkan_overlay_compositor);
#define GST_CAT_DEFAULT gst_debug_vulkan_overlay_compositor
struct vk_overlay
{
GstBuffer *buffer;
GstVideoOverlayComposition *composition;
GstVideoOverlayRectangle *rectangle;
GstVulkanFullScreenQuad *quad;
};
static void
vk_overlay_clear (struct vk_overlay *overlay)
{
gst_clear_buffer (&overlay->buffer);
overlay->rectangle = NULL;
if (overlay->composition)
gst_video_overlay_composition_unref (overlay->composition);
overlay->composition = NULL;
gst_clear_object (&overlay->quad);
}
static void
vk_overlay_init (struct vk_overlay *overlay, GstVulkanQueue * queue,
GstBuffer * buffer, GstVideoOverlayComposition * comp,
GstVideoOverlayRectangle * rectangle, GstVulkanHandle * vert,
GstVulkanHandle * frag)
{
GstVideoOverlayFormatFlags flags;
memset (overlay, 0, sizeof (*overlay));
flags = gst_video_overlay_rectangle_get_flags (rectangle);
overlay->buffer = gst_buffer_ref (buffer);
overlay->composition = gst_video_overlay_composition_ref (comp);
overlay->rectangle = rectangle;
overlay->quad = gst_vulkan_full_screen_quad_new (queue);
gst_vulkan_full_screen_quad_enable_clear (overlay->quad, FALSE);
gst_vulkan_full_screen_quad_set_shaders (overlay->quad, vert, frag);
gst_vulkan_full_screen_quad_enable_blend (overlay->quad, TRUE);
gst_vulkan_full_screen_quad_set_blend_operation (overlay->quad,
VK_BLEND_OP_ADD, VK_BLEND_OP_ADD);
if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA) {
gst_vulkan_full_screen_quad_set_blend_factors (overlay->quad,
VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA,
VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA);
} else {
gst_vulkan_full_screen_quad_set_blend_factors (overlay->quad,
VK_BLEND_FACTOR_SRC_ALPHA, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA,
VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA);
}
}
struct Vertex
{
float x, y, z;
float s, t;
};
struct swizzle_uniforms
{
int in_reorder_index[4];
int out_reorder_index[4];
};
static gboolean
vk_overlay_upload (struct vk_overlay *overlay, GstVideoInfo * out_info,
GError ** error)
{
GstBuffer *overlay_buffer, *vk_gst_buffer = NULL;
GstVideoMeta *vmeta;
GstVideoInfo vinfo;
GstVideoFrame vframe;
GstVulkanBufferMemory *buf_mem;
GstVulkanImageMemory *img_mem;
GstMemory *vkbuffer = NULL, *vkimage = NULL, *vkvertices = NULL;
GstMemory *vkuniforms = NULL;
VkFormat vk_format;
GstMapInfo map_info;
GstVulkanFence *fence = NULL;
GstVulkanCommandBuffer *cmd_buf = NULL;
VkBufferMemoryBarrier buffer_memory_barrier;
VkImageMemoryBarrier image_memory_barrier;
VkBufferImageCopy region;
VkResult err;
struct Vertex vertices[4];
struct swizzle_uniforms uniforms;
overlay_buffer =
gst_video_overlay_rectangle_get_pixels_unscaled_argb
(overlay->rectangle, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
vmeta = gst_buffer_get_video_meta (overlay_buffer);
gst_video_info_set_format (&vinfo, vmeta->format, vmeta->width,
vmeta->height);
vinfo.stride[0] = vmeta->stride[0];
if (!gst_vulkan_full_screen_quad_set_info (overlay->quad, out_info, out_info))
goto error;
if (!gst_video_frame_map (&vframe, &vinfo, overlay_buffer, GST_MAP_READ)) {
g_set_error_literal (error, GST_TYPE_RESOURCE_ERROR,
GST_RESOURCE_ERROR_READ, "Cannot map overlay buffer for reading");
return FALSE;
}
vkbuffer =
gst_vulkan_buffer_memory_alloc (overlay->quad->queue->device,
GST_VIDEO_INFO_COMP_STRIDE (&vinfo, 0) *
GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
buf_mem = (GstVulkanBufferMemory *) vkbuffer;
if (!gst_memory_map (vkbuffer, &map_info, GST_MAP_WRITE)) {
g_set_error_literal (error, GST_TYPE_RESOURCE_ERROR,
GST_RESOURCE_ERROR_WRITE,
"Cannot map staging vulkan buffer for writing");
gst_video_frame_unmap (&vframe);
goto error;
}
memcpy (map_info.data, vframe.data[0], vframe.info.size);
gst_memory_unmap (vkbuffer, &map_info);
gst_video_frame_unmap (&vframe);
vk_format = gst_vulkan_format_from_video_info (&vinfo, 0);
vkimage =
gst_vulkan_image_memory_alloc (overlay->quad->queue->device, vk_format,
GST_VIDEO_INFO_COMP_WIDTH (&vinfo, 0),
GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
VK_IMAGE_TILING_OPTIMAL,
VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
VK_IMAGE_USAGE_SAMPLED_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
img_mem = (GstVulkanImageMemory *) vkimage;
/* *INDENT-OFF* */
region = (VkBufferImageCopy) {
.bufferOffset = 0,
.bufferRowLength = GST_VIDEO_INFO_COMP_WIDTH (&vinfo, 0),
.bufferImageHeight = GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
.imageSubresource = {
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.mipLevel = 0,
.baseArrayLayer = 0,
.layerCount = 1,
},
.imageOffset = { .x = 0, .y = 0, .z = 0, },
.imageExtent = {
.width = GST_VIDEO_INFO_COMP_WIDTH (&vinfo, 0),
.height = GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
.depth = 1,
}
};
buffer_memory_barrier = (VkBufferMemoryBarrier) {
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = buf_mem->barrier.parent.access_flags,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
/* FIXME: implement exclusive transfers */
.srcQueueFamilyIndex = 0,
.dstQueueFamilyIndex = 0,
.buffer = buf_mem->buffer,
.offset = region.bufferOffset,
.size = region.bufferRowLength * region.bufferImageHeight,
};
image_memory_barrier = (VkImageMemoryBarrier) {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = img_mem->barrier.parent.access_flags,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = img_mem->barrier.image_layout,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
/* FIXME: implement exclusive transfers */
.srcQueueFamilyIndex = 0,
.dstQueueFamilyIndex = 0,
.image = img_mem->image,
.subresourceRange = img_mem->barrier.subresource_range,
};
/* *INDENT-ON* */
if (!(cmd_buf =
gst_vulkan_command_pool_create (overlay->quad->cmd_pool, error)))
goto error;
{
/* *INDENT-OFF* */
VkCommandBufferBeginInfo cmd_buf_info = {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.pNext = NULL,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
.pInheritanceInfo = NULL
};
/* *INDENT-ON* */
gst_vulkan_command_buffer_lock (cmd_buf);
err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
if (gst_vulkan_error_to_g_error (err, error, "vkBeginCommandBuffer") < 0) {
gst_vulkan_command_buffer_unlock (cmd_buf);
goto error;
}
}
vkCmdPipelineBarrier (cmd_buf->cmd,
buf_mem->barrier.parent.pipeline_stages | img_mem->barrier.
parent.pipeline_stages, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 1,
&buffer_memory_barrier, 1, &image_memory_barrier);
buf_mem->barrier.parent.pipeline_stages = VK_PIPELINE_STAGE_TRANSFER_BIT;
buf_mem->barrier.parent.access_flags = buffer_memory_barrier.dstAccessMask;
img_mem->barrier.parent.pipeline_stages = VK_PIPELINE_STAGE_TRANSFER_BIT;
img_mem->barrier.parent.access_flags = image_memory_barrier.dstAccessMask;
img_mem->barrier.image_layout = image_memory_barrier.newLayout;
vkCmdCopyBufferToImage (cmd_buf->cmd, buf_mem->buffer, img_mem->image,
img_mem->barrier.image_layout, 1, &region);
err = vkEndCommandBuffer (cmd_buf->cmd);
gst_vulkan_command_buffer_unlock (cmd_buf);
if (gst_vulkan_error_to_g_error (err, error, "vkEndCommandBuffer") < 0) {
goto error;
}
{
VkSubmitInfo submit_info = { 0, };
/* *INDENT-OFF* */
submit_info = (VkSubmitInfo) {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.pNext = NULL,
.waitSemaphoreCount = 0,
.pWaitSemaphores = NULL,
.pWaitDstStageMask = NULL,
.commandBufferCount = 1,
.pCommandBuffers = &cmd_buf->cmd,
.signalSemaphoreCount = 0,
.pSignalSemaphores = NULL,
};
/* *INDENT-ON* */
fence =
gst_vulkan_device_create_fence (overlay->quad->queue->device, error);
if (!fence)
goto error;
gst_vulkan_queue_submit_lock (overlay->quad->queue);
err =
vkQueueSubmit (overlay->quad->queue->queue, 1, &submit_info,
GST_VULKAN_FENCE_FENCE (fence));
gst_vulkan_queue_submit_unlock (overlay->quad->queue);
if (gst_vulkan_error_to_g_error (err, error, "vkQueueSubmit") < 0)
goto error;
gst_vulkan_trash_list_add (overlay->quad->trash_list,
gst_vulkan_trash_list_acquire (overlay->quad->trash_list, fence,
gst_vulkan_trash_mini_object_unref,
GST_MINI_OBJECT_CAST (cmd_buf)));
cmd_buf = NULL;
gst_vulkan_trash_list_add (overlay->quad->trash_list,
gst_vulkan_trash_list_acquire (overlay->quad->trash_list, fence,
gst_vulkan_trash_mini_object_unref,
GST_MINI_OBJECT_CAST (vkbuffer)));
vkbuffer = NULL;
gst_vulkan_trash_list_add (overlay->quad->trash_list,
gst_vulkan_trash_list_acquire (overlay->quad->trash_list, fence,
gst_vulkan_trash_mini_object_unref,
GST_MINI_OBJECT_CAST (gst_memory_ref (vkimage))));
gst_vulkan_trash_list_gc (overlay->quad->trash_list);
gst_vulkan_fence_unref (fence);
fence = NULL;
}
vk_gst_buffer = gst_buffer_new ();
gst_buffer_append_memory (vk_gst_buffer, vkimage);
vkimage = NULL;
if (!gst_vulkan_full_screen_quad_set_input_buffer (overlay->quad,
vk_gst_buffer, error))
goto error;
gst_clear_buffer (&vk_gst_buffer);
{
int xpos, ypos;
guint width, height, out_width, out_height;
float xl, xr, yt, yb;
if (!gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle,
&xpos, &ypos, &width, &height))
goto error;
out_width = GST_VIDEO_INFO_WIDTH (out_info);
out_height = GST_VIDEO_INFO_HEIGHT (out_info);
xl = 2.0 * (float) xpos / (float) out_width - 1.0;
yt = 2.0 * (float) ypos / (float) out_height - 1.0;
xr = xl + 2.0 * (float) width / (float) out_width;
yb = yt + 2.0 * (float) height / (float) out_height;
GST_LOG_OBJECT (overlay->quad, "rectangle %ux%u+%d,%d placed in %ux%u at "
"%fx%f+%f,%f", width, height, xpos, ypos, out_width, out_height,
xr - xl, yb - yt, xl, yt);
/* top-left */
vertices[0].x = xl;
vertices[0].y = yt;
vertices[0].z = 0.0;
vertices[0].s = 0.0;
vertices[0].t = 0.0;
/* top-right */
vertices[1].x = xr;
vertices[1].y = yt;
vertices[1].z = 0.0;
vertices[1].s = 1.0;
vertices[1].t = 0.0;
/* bottom-right */
vertices[2].x = xr;
vertices[2].y = yb;
vertices[2].z = 0.0;
vertices[2].s = 1.0;
vertices[2].t = 1.0;
/* bottom-left */
vertices[3].x = xl;
vertices[3].y = yb;
vertices[3].z = 0.0;
vertices[3].s = 0.0;
vertices[3].t = 1.0;
}
vkvertices =
gst_vulkan_buffer_memory_alloc (overlay->quad->queue->device,
sizeof (vertices),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (!gst_memory_map (vkvertices, &map_info, GST_MAP_WRITE))
goto error;
memcpy (map_info.data, vertices, sizeof (vertices));
gst_memory_unmap (vkvertices, &map_info);
if (!gst_vulkan_full_screen_quad_set_vertex_buffer (overlay->quad,
vkvertices, error))
goto error;
gst_clear_mini_object ((GstMiniObject **) & vkvertices);
uniforms.in_reorder_index[0] = 0;
uniforms.in_reorder_index[1] = 1;
uniforms.in_reorder_index[2] = 2;
uniforms.in_reorder_index[3] = 3;
uniforms.out_reorder_index[0] = 0;
uniforms.out_reorder_index[1] = 1;
uniforms.out_reorder_index[2] = 2;
uniforms.out_reorder_index[3] = 3;
vkuniforms =
gst_vulkan_buffer_memory_alloc (overlay->quad->queue->device,
sizeof (uniforms),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (!gst_memory_map (vkuniforms, &map_info, GST_MAP_WRITE))
goto error;
memcpy (map_info.data, &uniforms, sizeof (uniforms));
gst_memory_unmap (vkuniforms, &map_info);
if (!gst_vulkan_full_screen_quad_set_uniform_buffer (overlay->quad,
vkuniforms, error))
goto error;
gst_clear_mini_object ((GstMiniObject **) & vkuniforms);
return TRUE;
error:
gst_clear_mini_object ((GstMiniObject **) & vkimage);
gst_clear_mini_object ((GstMiniObject **) & vkbuffer);
gst_clear_mini_object ((GstMiniObject **) & vkvertices);
gst_clear_mini_object ((GstMiniObject **) & vkuniforms);
if (cmd_buf)
gst_vulkan_command_buffer_unref (cmd_buf);
if (fence)
gst_vulkan_fence_unref (fence);
gst_clear_buffer (&vk_gst_buffer);
gst_clear_buffer (&overlay_buffer);
return FALSE;
}
static gboolean gst_vulkan_overlay_compositor_start (GstBaseTransform * bt);
static gboolean gst_vulkan_overlay_compositor_stop (GstBaseTransform * bt);
static GstCaps *gst_vulkan_overlay_compositor_transform_caps (GstBaseTransform *
bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter);
static gboolean gst_vulkan_overlay_compositor_set_caps (GstBaseTransform *
bt, GstCaps * incaps, GstCaps * outcaps);
static GstFlowReturn
gst_vulkan_overlay_compositor_transform_ip (GstBaseTransform * bt,
GstBuffer * inbuf);
#define IMAGE_FORMATS " { BGRA }"
static GstStaticPadTemplate gst_vulkan_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE ","
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
IMAGE_FORMATS) "; "
GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
IMAGE_FORMATS) "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
IMAGE_FORMATS)));
static GstStaticPadTemplate gst_vulkan_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE ","
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
IMAGE_FORMATS) "; "
GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
IMAGE_FORMATS) "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
IMAGE_FORMATS)));
enum
{
PROP_0,
};
typedef struct _GstVulkanOverlayCompositor GstVulkanOverlayCompositor;
struct _GstVulkanOverlayCompositor
{
GstVulkanVideoFilter parent;
GstVulkanHandle *vert;
GstVulkanHandle *frag;
GArray *overlays;
gboolean render_overlays;
};
#define gst_vulkan_overlay_compositor_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanOverlayCompositor,
gst_vulkan_overlay_compositor, GST_TYPE_VULKAN_VIDEO_FILTER,
GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_overlay_compositor,
"vulkanoverlaycompositor", 0, "Vulkan Overlay Compositor"));
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vulkanoverlaycompositor,
"vulkanoverlaycompositor", GST_RANK_NONE,
GST_TYPE_VULKAN_OVERLAY_COMPOSITOR, vulkan_element_init (plugin));
static void
gst_vulkan_overlay_compositor_class_init (GstVulkanOverlayCompositorClass *
klass)
{
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
gst_element_class_set_metadata (gstelement_class, "Vulkan Overlay Compositor",
"Filter/Video", "Vulkan Overlay Composition element",
"Matthew Waters <matthew@centricular.com>");
gst_element_class_add_static_pad_template (gstelement_class,
&gst_vulkan_sink_template);
gst_element_class_add_static_pad_template (gstelement_class,
&gst_vulkan_src_template);
gstbasetransform_class->start =
GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_start);
gstbasetransform_class->stop =
GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_stop);
gstbasetransform_class->transform_caps =
GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_transform_caps);
gstbasetransform_class->set_caps =
GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_set_caps);
gstbasetransform_class->transform_ip =
GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_transform_ip);
}
static void
gst_vulkan_overlay_compositor_init (GstVulkanOverlayCompositor * vk_overlay)
{
}
static gboolean
gst_vulkan_overlay_compositor_start (GstBaseTransform * bt)
{
GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (vk_overlay);
GError *error = NULL;
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
return FALSE;
if (!(vk_overlay->vert =
gst_vulkan_create_shader (vfilter->device, identity_vert,
identity_vert_size, &error)))
goto error;
if (!(vk_overlay->frag =
gst_vulkan_create_shader (vfilter->device, swizzle_frag,
swizzle_frag_size, &error))) {
gst_clear_vulkan_handle (&vk_overlay->vert);
goto error;
}
vk_overlay->overlays = g_array_new (FALSE, TRUE, sizeof (struct vk_overlay));
g_array_set_clear_func (vk_overlay->overlays,
(GDestroyNotify) vk_overlay_clear);
return TRUE;
error:
GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL));
return FALSE;
}
static gboolean
gst_vulkan_overlay_compositor_stop (GstBaseTransform * bt)
{
GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
if (vk_overlay->overlays) {
g_array_set_size (vk_overlay->overlays, 0);
g_array_unref (vk_overlay->overlays);
}
vk_overlay->overlays = NULL;
gst_clear_vulkan_handle (&vk_overlay->vert);
gst_clear_vulkan_handle (&vk_overlay->frag);
return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
}
static struct vk_overlay *
find_by_rectangle (GstVulkanOverlayCompositor * vk_overlay,
GstVideoOverlayRectangle * rectangle)
{
int i;
for (i = 0; i < vk_overlay->overlays->len; i++) {
struct vk_overlay *over =
&g_array_index (vk_overlay->overlays, struct vk_overlay, i);
if (over->rectangle == rectangle)
return over;
}
return NULL;
}
static gboolean
overlay_in_rectangles (struct vk_overlay *over,
GstVideoOverlayComposition * composition)
{
int i, n;
n = gst_video_overlay_composition_n_rectangles (composition);
for (i = 0; i < n; i++) {
GstVideoOverlayRectangle *rect;
rect = gst_video_overlay_composition_get_rectangle (composition, i);
if (over->rectangle == rect)
return TRUE;
}
return FALSE;
}
static GstCaps *
gst_vulkan_overlay_compositor_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter)
{
GstCaps *ret;
/* add/remove the composition overlay meta as necessary */
if (direction == GST_PAD_SRC) {
GstCaps *composition_caps;
int i;
composition_caps = gst_caps_copy (caps);
for (i = 0; i < gst_caps_get_size (composition_caps); i++) {
GstCapsFeatures *f = gst_caps_get_features (composition_caps, i);
if (!gst_caps_features_is_any (f))
gst_caps_features_add (f,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
}
ret = gst_caps_merge (composition_caps, gst_caps_copy (caps));
} else {
guint i, n;
GstCaps *removed;
ret = gst_caps_copy (caps);
removed = gst_caps_copy (caps);
n = gst_caps_get_size (removed);
for (i = 0; i < n; i++) {
GstCapsFeatures *feat = gst_caps_get_features (removed, i);
if (feat && gst_caps_features_contains (feat,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
feat = gst_caps_features_copy (feat);
/* prefer the passthrough case */
gst_caps_features_remove (feat,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
gst_caps_set_features (removed, i, feat);
}
}
ret = gst_caps_merge (ret, removed);
}
if (filter) {
GstCaps *tmp = gst_caps_intersect (ret, filter);
gst_clear_caps (&ret);
ret = tmp;
}
return ret;
}
static gboolean
gst_vulkan_overlay_compositor_set_caps (GstBaseTransform * bt, GstCaps * incaps,
GstCaps * outcaps)
{
GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
GstCapsFeatures *in_features, *out_features;
GST_DEBUG_OBJECT (bt, " incaps %" GST_PTR_FORMAT, incaps);
GST_DEBUG_OBJECT (bt, "outcaps %" GST_PTR_FORMAT, outcaps);
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, incaps, outcaps))
return FALSE;
in_features = gst_caps_get_features (incaps, 0);
out_features = gst_caps_get_features (outcaps, 0);
if (gst_caps_features_contains (in_features,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION) &&
!gst_caps_features_contains (out_features,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
GST_INFO_OBJECT (bt, "caps say to render GstVideoOverlayCompositionMeta");
vk_overlay->render_overlays = TRUE;
} else {
GST_INFO_OBJECT (bt,
"caps say to not render GstVideoOverlayCompositionMeta");
vk_overlay->render_overlays = FALSE;
}
return TRUE;
}
static GstFlowReturn
gst_vulkan_overlay_compositor_transform_ip (GstBaseTransform * bt,
GstBuffer * buffer)
{
GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
GstVideoOverlayCompositionMeta *ometa;
GstVideoOverlayComposition *comp = NULL;
GError *error = NULL;
int i, n;
if (!vk_overlay->render_overlays) {
GST_LOG_OBJECT (bt,
"caps don't say to render GstVideoOverlayCompositionMeta, passthrough");
return GST_FLOW_OK;
}
ometa = gst_buffer_get_video_overlay_composition_meta (buffer);
if (!ometa) {
GST_LOG_OBJECT (bt,
"no GstVideoOverlayCompositionMeta on buffer, passthrough");
return GST_FLOW_OK;
}
comp = gst_video_overlay_composition_ref (ometa->overlay);
gst_buffer_remove_meta (buffer, (GstMeta *) ometa);
n = gst_video_overlay_composition_n_rectangles (comp);
if (n == 0) {
GST_LOG_OBJECT (bt,
"GstVideoOverlayCompositionMeta has 0 rectangles, passthrough");
return GST_FLOW_OK;
}
GST_LOG_OBJECT (bt,
"rendering GstVideoOverlayCompositionMeta with %u rectangles", n);
for (i = 0; i < n; i++) {
GstVideoOverlayRectangle *rectangle;
struct vk_overlay *over;
rectangle = gst_video_overlay_composition_get_rectangle (comp, i);
over = find_by_rectangle (vk_overlay, rectangle);
if (!over) {
struct vk_overlay new_overlay = { 0, };
vk_overlay_init (&new_overlay, vk_overlay->parent.queue, buffer, comp,
rectangle, vk_overlay->vert, vk_overlay->frag);
if (!vk_overlay_upload (&new_overlay, &vk_overlay->parent.out_info,
&error))
goto error;
g_array_append_val (vk_overlay->overlays, new_overlay);
}
}
n = vk_overlay->overlays->len;
for (i = 0; i < n;) {
struct vk_overlay *over =
&g_array_index (vk_overlay->overlays, struct vk_overlay, i);
if (!overlay_in_rectangles (over, ometa->overlay)) {
g_array_remove_index (vk_overlay->overlays, i);
continue;
}
if (!gst_vulkan_full_screen_quad_set_output_buffer (over->quad, buffer,
&error))
goto error;
if (!gst_vulkan_full_screen_quad_draw (over->quad, &error))
goto error;
i++;
}
if (comp)
gst_video_overlay_composition_unref (comp);
return GST_FLOW_OK;
error:
GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
g_clear_error (&error);
if (comp)
gst_video_overlay_composition_unref (comp);
return GST_FLOW_ERROR;
}

36
subprojects/gst-plugins-bad/ext/vulkan/vkoverlaycompositor.h Normal file
View file

@ -0,0 +1,36 @@
/*
* GStreamer
* Copyright (C) 2022 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef _VK_OVERLAY_COMPOSITOR_H_
#define _VK_OVERLAY_COMPOSITOR_H_
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/vulkan/vulkan.h>
G_BEGIN_DECLS
#define GST_TYPE_VULKAN_OVERLAY_COMPOSITOR (gst_vulkan_overlay_compositor_get_type())
G_DECLARE_FINAL_TYPE(GstVulkanOverlayCompositor, gst_vulkan_overlay_compositor, GST, VULKAN_OVERLAY_COMPOSITOR, GstVulkanVideoFilter);
GST_ELEMENT_REGISTER_DECLARE (vulkanoverlaycompositor);
G_END_DECLS
#endif