waylandsink: Move buffer commits to the display thread

Syncrhonizing buffer commits to the streaming thread can lead to
dropped frames when frame callbacks are not processed before the
next frame is ready for rendering.  Depending on the drift between
the wayland compositor and buffer source timings, this can lead to
periods of significant frame drop, especially when the media frame
rate is close to the display frame rate.

Cache buffers in the streaming thread and peform commits on the
display thread to eliminate the buffer commit racing.

The implementation is the same for both waylandsink and gtkwaylandsink,
so move it to the common wayland library under gst-lib.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6133>

subprojects/gst-plugins-bad/ext/gtk/gstgtkwaylandsink.c

@@ -107,15 +107,12 @@ typedef struct _GstGtkWaylandSinkPrivate
   GstVideoInfoDmaDrm drm_info;
   GstCaps *caps;
 
-  gboolean redraw_pending;
   GMutex render_lock;
 
   GstVideoOrientationMethod sink_rotate_method;
   GstVideoOrientationMethod tag_rotate_method;
   GstVideoOrientationMethod current_rotate_method;
 
-  struct wl_callback *callback;
-
   gchar *drm_device;
   gboolean skip_dumb_buffer_copy;
 } GstGtkWaylandSinkPrivate;
@@ -761,14 +758,6 @@ gst_gtk_wayland_sink_change_state (GstElement * element,
         /* remove buffer from surface, show nothing */
         gst_wl_window_render (priv->wl_window, NULL, NULL);
       }
-
-      g_mutex_lock (&priv->render_lock);
-      if (priv->callback) {
-        wl_callback_destroy (priv->callback);
-        priv->callback = NULL;
-      }
-      priv->redraw_pending = FALSE;
-      g_mutex_unlock (&priv->render_lock);
       break;
     default:
       break;
@@ -1075,56 +1064,25 @@ gst_gtk_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
   return TRUE;
 }
 
-static void
-frame_redraw_callback (void *data, struct wl_callback *callback, uint32_t time)
-{
-  GstGtkWaylandSink *self = data;
-  GstGtkWaylandSinkPrivate *priv =
-      gst_gtk_wayland_sink_get_instance_private (self);
-
-  GST_LOG_OBJECT (self, "frame_redraw_cb");
-
-  g_mutex_lock (&priv->render_lock);
-  priv->redraw_pending = FALSE;
-
-  if (priv->callback) {
-    wl_callback_destroy (callback);
-    priv->callback = NULL;
-  }
-  g_mutex_unlock (&priv->render_lock);
-}
-
-static const struct wl_callback_listener frame_callback_listener = {
-  frame_redraw_callback
-};
-
 /* must be called with the render lock */
-static void
+static gboolean
 render_last_buffer (GstGtkWaylandSink * self, gboolean redraw)
 {
   GstGtkWaylandSinkPrivate *priv =
       gst_gtk_wayland_sink_get_instance_private (self);
   GstWlBuffer *wlbuffer;
   const GstVideoInfo *info = NULL;
-  struct wl_surface *surface;
-  struct wl_callback *callback;
 
   if (!priv->wl_window)
-    return;
+    return FALSE;
 
   wlbuffer = gst_buffer_get_wl_buffer (priv->display, priv->last_buffer);
-  surface = gst_wl_window_get_wl_surface (priv->wl_window);
-
-  priv->redraw_pending = TRUE;
-  callback = wl_surface_frame (surface);
-  priv->callback = callback;
-  wl_callback_add_listener (callback, &frame_callback_listener, self);
 
   if (G_UNLIKELY (priv->video_info_changed && !redraw)) {
     info = &priv->video_info;
     priv->video_info_changed = FALSE;
   }
-  gst_wl_window_render (priv->wl_window, wlbuffer, info);
+  return gst_wl_window_render (priv->wl_window, wlbuffer, info);
 }
 
 static GstFlowReturn
@@ -1151,14 +1109,6 @@ gst_gtk_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
     goto done;
   }
 
-  /* drop buffers until we get a frame callback */
-  if (priv->redraw_pending) {
-    GST_LOG_OBJECT (self, "buffer %" GST_PTR_FORMAT " dropped (redraw pending)",
-        buffer);
-    ret = GST_BASE_SINK_FLOW_DROPPED;
-    goto done;
-  }
-
   /* make sure that the application has called set_render_rectangle() */
   if (G_UNLIKELY (gst_wl_window_get_render_rectangle (priv->wl_window)->w == 0))
     goto no_window_size;
@@ -1319,7 +1269,8 @@ render:
   }
 
   gst_buffer_replace (&priv->last_buffer, to_render);
-  render_last_buffer (self, FALSE);
+  if (!render_last_buffer (self, FALSE))
+    ret = GST_BASE_SINK_FLOW_DROPPED;
 
   if (buffer != to_render)
     gst_buffer_unref (to_render);

subprojects/gst-plugins-bad/ext/wayland/gstwaylandsink.c

@@ -448,13 +448,6 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition)
         }
       }
 
-      g_mutex_lock (&self->render_lock);
-      if (self->callback) {
-        wl_callback_destroy (self->callback);
-        self->callback = NULL;
-      }
-      self->redraw_pending = FALSE;
-      g_mutex_unlock (&self->render_lock);
       break;
     case GST_STATE_CHANGE_READY_TO_NULL:
       g_mutex_lock (&self->display_lock);
@@ -795,49 +788,20 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
   return TRUE;
 }
 
-static void
-frame_redraw_callback (void *data, struct wl_callback *callback, uint32_t time)
-{
-  GstWaylandSink *self = data;
-
-  GST_LOG_OBJECT (self, "frame_redraw_cb");
-
-  g_mutex_lock (&self->render_lock);
-  self->redraw_pending = FALSE;
-
-  if (self->callback) {
-    wl_callback_destroy (callback);
-    self->callback = NULL;
-  }
-  g_mutex_unlock (&self->render_lock);
-}
-
-static const struct wl_callback_listener frame_callback_listener = {
-  frame_redraw_callback
-};
-
 /* must be called with the render lock */
-static void
+static gboolean
 render_last_buffer (GstWaylandSink * self, gboolean redraw)
 {
   GstWlBuffer *wlbuffer;
   const GstVideoInfo *info = NULL;
-  struct wl_surface *surface;
-  struct wl_callback *callback;
 
   wlbuffer = gst_buffer_get_wl_buffer (self->display, self->last_buffer);
-  surface = gst_wl_window_get_wl_surface (self->window);
-
-  self->redraw_pending = TRUE;
-  callback = wl_surface_frame (surface);
-  self->callback = callback;
-  wl_callback_add_listener (callback, &frame_callback_listener, self);
 
   if (G_UNLIKELY (self->video_info_changed && !redraw)) {
     info = &self->video_info;
     self->video_info_changed = FALSE;
   }
-  gst_wl_window_render (self->window, wlbuffer, info);
+  return gst_wl_window_render (self->window, wlbuffer, info);
 }
 
 static void
@@ -883,14 +847,6 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer)
     }
   }
 
-  /* drop buffers until we get a frame callback */
-  if (self->redraw_pending) {
-    GST_LOG_OBJECT (self, "buffer %" GST_PTR_FORMAT " dropped (redraw pending)",
-        buffer);
-    ret = GST_BASE_SINK_FLOW_DROPPED;
-    goto done;
-  }
-
   /* make sure that the application has called set_render_rectangle() */
   if (G_UNLIKELY (gst_wl_window_get_render_rectangle (self->window)->w == 0))
     goto no_window_size;
@@ -1050,7 +1006,8 @@ render:
   }
 
   gst_buffer_replace (&self->last_buffer, to_render);
-  render_last_buffer (self, FALSE);
+  if (!render_last_buffer (self, FALSE))
+    ret = GST_BASE_SINK_FLOW_DROPPED;
 
   if (buffer != to_render)
     gst_buffer_unref (to_render);
@@ -1194,7 +1151,7 @@ gst_wayland_sink_expose (GstVideoOverlay * overlay)
   GST_DEBUG_OBJECT (self, "expose");
 
   g_mutex_lock (&self->render_lock);
-  if (self->last_buffer && !self->redraw_pending) {
+  if (self->last_buffer) {
     GST_DEBUG_OBJECT (self, "redrawing last buffer");
     render_last_buffer (self, TRUE);
   }

subprojects/gst-plugins-bad/ext/wayland/gstwaylandsink.h

@@ -60,7 +60,6 @@ struct _GstWaylandSink
 
   gchar *display_name;
 
-  gboolean redraw_pending;
   GMutex render_lock;
   GstBuffer *last_buffer;
 
@@ -68,8 +67,6 @@ struct _GstWaylandSink
   GstVideoOrientationMethod tag_rotate_method;
   GstVideoOrientationMethod current_rotate_method;
 
-  struct wl_callback *callback;
-
   gchar *drm_device;
   gboolean skip_dumb_buffer_copy;
 };

subprojects/gst-plugins-bad/gst-libs/gst/wayland/gstwlwindow.c

@@ -72,6 +72,14 @@ typedef struct _GstWlWindowPrivate
   /* when this is not set both the area_surface and the video_surface are not
    * visible and certain steps should be skipped */
   gboolean is_area_surface_mapped;
+
+  GMutex window_lock;
+  GstWlBuffer *next_buffer;
+  GstVideoInfo *next_video_info;
+  GstWlBuffer *staged_buffer;
+  gboolean clear_window;
+  struct wl_callback *frame_callback;
+  struct wl_callback *commit_callback;
 } GstWlWindowPrivate;
 
 G_DEFINE_TYPE_WITH_CODE (GstWlWindow, gst_wl_window, G_TYPE_OBJECT,
@@ -93,6 +101,9 @@ static void gst_wl_window_finalize (GObject * gobject);
 
 static void gst_wl_window_update_borders (GstWlWindow * self);
 
+static void gst_wl_window_commit_buffer (GstWlWindow * self,
+    GstWlBuffer * buffer);
+
 static void
 handle_xdg_toplevel_close (void *data, struct xdg_toplevel *xdg_toplevel)
 {
@@ -173,6 +184,7 @@ gst_wl_window_init (GstWlWindow * self)
   priv->configured = TRUE;
   g_cond_init (&priv->configure_cond);
   g_mutex_init (&priv->configure_mutex);
+  g_mutex_init (&priv->window_lock);
 }
 
 static void
@@ -181,6 +193,9 @@ gst_wl_window_finalize (GObject * gobject)
   GstWlWindow *self = GST_WL_WINDOW (gobject);
   GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self);
 
+  gst_wl_display_callback_destroy (priv->display, &priv->frame_callback);
+  gst_wl_display_callback_destroy (priv->display, &priv->commit_callback);
+
   if (priv->xdg_toplevel)
     xdg_toplevel_destroy (priv->xdg_toplevel);
   if (priv->xdg_surface)
@@ -499,11 +514,40 @@ gst_wl_window_set_opaque (GstWlWindow * self, const GstVideoInfo * info)
   }
 }
 
-void
-gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
-    const GstVideoInfo * info)
+static void
+frame_redraw_callback (void *data, struct wl_callback *callback, uint32_t time)
+{
+  GstWlWindow *self = data;
+  GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self);
+  GstWlBuffer *next_buffer;
+
+  GST_INFO ("frame_redraw_cb ");
+
+  wl_callback_destroy (callback);
+  priv->frame_callback = NULL;
+
+  g_mutex_lock (&priv->window_lock);
+  next_buffer = priv->next_buffer = priv->staged_buffer;
+  priv->staged_buffer = NULL;
+  g_mutex_unlock (&priv->window_lock);
+
+  if (next_buffer || priv->clear_window)
+    gst_wl_window_commit_buffer (self, next_buffer);
+
+  if (next_buffer)
+    gst_wl_buffer_unref_buffer (next_buffer);
+}
+
+static const struct wl_callback_listener frame_callback_listener = {
+  frame_redraw_callback
+};
+
+static void
+gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer)
 {
   GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self);
+  GstVideoInfo *info = priv->next_video_info;
+  struct wl_callback *callback;
 
   if (G_UNLIKELY (info)) {
     priv->scaled_width =
@@ -517,6 +561,9 @@ gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
   }
 
   if (G_LIKELY (buffer)) {
+    callback = wl_surface_frame (priv->video_surface_wrapper);
+    priv->frame_callback = callback;
+    wl_callback_add_listener (callback, &frame_callback_listener, self);
     gst_wl_buffer_attach (buffer, priv->video_surface_wrapper);
     wl_surface_damage_buffer (priv->video_surface_wrapper, 0, 0, G_MAXINT32,
         G_MAXINT32);
@@ -535,6 +582,7 @@ gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
     wl_surface_attach (priv->area_surface_wrapper, NULL, 0, 0);
     wl_surface_commit (priv->area_surface_wrapper);
     priv->is_area_surface_mapped = FALSE;
+    priv->clear_window = FALSE;
   }
 
   if (G_UNLIKELY (info)) {
@@ -542,9 +590,70 @@ gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
      * the position of the video_subsurface */
     wl_surface_commit (priv->area_surface_wrapper);
     wl_subsurface_set_desync (priv->video_subsurface);
+    gst_video_info_free (priv->next_video_info);
+    priv->next_video_info = NULL;
   }
 
-  wl_display_flush (gst_wl_display_get_display (priv->display));
+}
+
+static void
+commit_callback (void *data, struct wl_callback *callback, uint32_t serial)
+{
+  GstWlWindow *self = data;
+  GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self);
+  GstWlBuffer *next_buffer;
+
+  wl_callback_destroy (callback);
+  priv->commit_callback = NULL;
+
+  g_mutex_lock (&priv->window_lock);
+  next_buffer = priv->next_buffer;
+  g_mutex_unlock (&priv->window_lock);
+
+  gst_wl_window_commit_buffer (self, next_buffer);
+
+  if (next_buffer)
+    gst_wl_buffer_unref_buffer (next_buffer);
+}
+
+static const struct wl_callback_listener commit_listener = {
+  commit_callback
+};
+
+gboolean
+gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
+    const GstVideoInfo * info)
+{
+  GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self);
+  gboolean ret = TRUE;
+
+  if (G_LIKELY (buffer))
+    gst_wl_buffer_ref_gst_buffer (buffer);
+
+  g_mutex_lock (&priv->window_lock);
+  if (G_UNLIKELY (info))
+    priv->next_video_info = gst_video_info_copy (info);
+
+  if (priv->next_buffer && priv->staged_buffer) {
+    GST_LOG_OBJECT (self, "buffer %p dropped (replaced)", priv->staged_buffer);
+    gst_wl_buffer_unref_buffer (priv->staged_buffer);
+    ret = FALSE;
+  }
+
+  if (!priv->next_buffer) {
+    priv->next_buffer = buffer;
+    priv->commit_callback =
+        gst_wl_display_sync (priv->display, &commit_listener, self);
+    wl_display_flush (gst_wl_display_get_display (priv->display));
+  } else {
+    priv->staged_buffer = buffer;
+  }
+  if (!buffer)
+    priv->clear_window = TRUE;
+
+  g_mutex_unlock (&priv->window_lock);
+
+  return ret;
 }
 
 /* Update the buffer used to draw black borders. When we have viewporter

subprojects/gst-plugins-bad/gst-libs/gst/wayland/gstwlwindow.h

@@ -60,7 +60,7 @@ GST_WL_API
 gboolean gst_wl_window_is_toplevel (GstWlWindow * self);
 
 GST_WL_API
-void gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
+gboolean gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer,
         const GstVideoInfo * info);
 
 GST_WL_API