va: Lock wayland display dmabuf formats against concurrent access

If there are multiple Wayland event listeners in different threads we
get the formats and modifiers pushed concurrently which leads to
segfault from GArray methods. This patch protects the array.

The problem occurs e.g. when using vaapipostproc together with Qt
qmlglsink, QtWayland will get the events as well as VAAPI.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5280>
This commit is contained in:
Jochen Henneberg 2023-09-05 12:46:52 +02:00 committed by GStreamer Marge Bot
parent 85adc2daee
commit c0f44e90c5
3 changed files with 32 additions and 9 deletions

View file

@ -132,7 +132,9 @@ dmabuf_modifier (void *data, struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf,
format, gst_video_format_to_string (gst_vaapi_video_format_from_drm_format
(format)), drm_format.modifier);
g_mutex_lock (&priv->dmabuf_formats_lock);
g_array_append_val (priv->dmabuf_formats, drm_format);
g_mutex_unlock (&priv->dmabuf_formats_lock);
}
static const struct zwp_linux_dmabuf_v1_listener dmabuf_listener = {
@ -261,7 +263,9 @@ gst_vaapi_display_wayland_close_display (GstVaapiDisplay * display)
g_clear_pointer (&priv->compositor, wl_compositor_destroy);
g_clear_pointer (&priv->registry, wl_registry_destroy);
g_mutex_lock (&priv->dmabuf_formats_lock);
g_array_unref (priv->dmabuf_formats);
g_mutex_unlock (&priv->dmabuf_formats_lock);
if (priv->wl_display) {
if (!priv->use_foreign_display)
@ -341,6 +345,7 @@ gst_vaapi_display_wayland_init (GstVaapiDisplayWayland * display)
display->priv = priv;
priv->event_fd = -1;
priv->dmabuf_formats = g_array_new (FALSE, FALSE, sizeof (GstDRMFormat));
g_mutex_init (&priv->dmabuf_formats_lock);
}
static void

View file

@ -77,6 +77,7 @@ struct _GstVaapiDisplayWaylandPrivate
struct zwp_linux_dmabuf_v1 *dmabuf;
struct wl_registry *registry;
GArray *dmabuf_formats;
GMutex dmabuf_formats_lock;
guint width;
guint height;
guint phys_width;

View file

@ -616,26 +616,36 @@ static GstVaapiDmabufStatus
dmabuf_format_supported (GstVaapiDisplayWaylandPrivate * const priv_display,
guint format, guint64 modifier)
{
GArray *formats = priv_display->dmabuf_formats;
GArray *formats;
gboolean linear = FALSE;
gboolean dontcare = FALSE;
gint i;
g_mutex_lock (&priv_display->dmabuf_formats_lock);
formats = priv_display->dmabuf_formats;
for (i = 0; i < formats->len; i++) {
GstDRMFormat fmt = g_array_index (formats, GstDRMFormat, i);
if (fmt.format == format && (fmt.modifier == modifier ||
(fmt.modifier == DRM_FORMAT_MOD_INVALID && modifier == 0)))
return GST_VAAPI_DMABUF_SUCCESS;
(fmt.modifier == DRM_FORMAT_MOD_INVALID && modifier == 0))) {
dontcare = TRUE;
break;
}
if (fmt.format == format && (fmt.modifier == 0 ||
fmt.modifier == DRM_FORMAT_MOD_INVALID))
linear = TRUE;
}
g_mutex_unlock (&priv_display->dmabuf_formats_lock);
if (dontcare)
return GST_VAAPI_DMABUF_SUCCESS;
if (linear)
return GST_VAAPI_DMABUF_BAD_MODIFIER;
else
return GST_VAAPI_DMABUF_BAD_FORMAT;
}
GstVideoFormat
static GstVideoFormat
check_format (GstVaapiDisplay * const display, gint index,
GstVideoFormat expect)
{
@ -662,22 +672,25 @@ check_format (GstVaapiDisplay * const display, gint index,
return format;
}
GstVideoFormat
static GstVideoFormat
choose_next_format (GstVaapiDisplay * const display, gint * next_index)
{
GstVaapiDisplayWaylandPrivate *const priv_display =
GST_VAAPI_DISPLAY_WAYLAND_GET_PRIVATE (display);
GArray *formats = priv_display->dmabuf_formats;
GArray *formats;
GstVideoFormat format;
gint i;
g_mutex_lock (&priv_display->dmabuf_formats_lock);
formats = priv_display->dmabuf_formats;
if (*next_index < 0) {
*next_index = 0;
/* try GST_VIDEO_FORMAT_RGBA first */
for (i = 0; i < formats->len; i++) {
format = check_format (display, i, GST_VIDEO_FORMAT_RGBA);
if (format != GST_VIDEO_FORMAT_UNKNOWN)
return format;
goto out;
}
}
@ -685,11 +698,15 @@ choose_next_format (GstVaapiDisplay * const display, gint * next_index)
format = check_format (display, i, GST_VIDEO_FORMAT_UNKNOWN);
if (format != GST_VIDEO_FORMAT_UNKNOWN) {
*next_index = i + 1;
return format;
goto out;
}
}
*next_index = formats->len;
return GST_VIDEO_FORMAT_UNKNOWN;
format = GST_VIDEO_FORMAT_UNKNOWN;
out:
g_mutex_unlock (&priv_display->dmabuf_formats_lock);
return format;
}
static GstVaapiDmabufStatus