ges: Implement a gesvideoscale bin

This allows user to be in control of when scaling happens in the pipeline.
It can be plugged as an effect and scaling will be forced to happen in
the effect instead of in the compositor.

Without this, it would not be possible for users to, for example,
crop a video source before scaling to the target source size.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5735>

girs/GES-1.0.gir

@@ -3536,10 +3536,23 @@ and [clip management](http://pitivi.org/manual/usingclips.html).</doc>
       </function>
     </enumeration>
     <class name="Effect" c:symbol-prefix="effect" c:type="GESEffect" parent="BaseEffect" glib:type-name="GESEffect" glib:get-type="ges_effect_get_type" glib:type-struct="EffectClass">
-      <doc xml:space="preserve" filename="../subprojects/gst-editing-services/ges/ges-effect.c">Currently we only support effects with N sinkpads and one single srcpad.
-Apart from `gesaudiomixer` and `gescompositor` which can be used as effects
-and where sinkpads will be requested as needed based on the timeline topology
-GES will always request at most one sinkpad per effect (when required).
+      <doc xml:space="preserve" filename="../subprojects/gst-editing-services/ges/ges-effect.c">Any GStreamer filter can be used as effects in GES. The only restriction we
+have is that effects element should have a single [sinkpad](GST_PAD_SINK)
+(which will be requested if necessary) and a single [srcpad](GST_PAD_SRC).
+
+Note that `gesaudiomixer` and `gescompositor` can be used as effects even
+though they can have several sinkpads.
+
+## GES specific effects:
+
+* **`gesvideoscale`**: GES implements a specific scaling bin that allows
+  specifying where scaling will happen inside the chain of effects. By
+  default scaling can happen either in the source (if the source doesn't have
+  a specific size, like `videotestsrc` or [mixing](ges_track_set_mixing) has
+  been disabled) or in the mixing element otherwise, when adding that element
+  as an effect, GES guarantees that the scaling will happen in it. This can
+  be useful for example if you want to crop the video before scaling or apply
+  rounding corners to the video after scaling, etc...
 
 &gt; Note: GES always adds converters (`audioconvert ! audioresample !
 &gt; audioconvert` for audio effects and `videoconvert` for video effects) to

subprojects/gst-editing-services/ges/ges-clip.c

@@ -250,6 +250,7 @@ struct _GESClipPrivate
 
   gboolean allow_any_remove;
 
+  gint nb_scale_effects;
   gboolean use_effect_priority;
   guint32 effect_priority;
   GError *add_error;
@@ -1638,6 +1639,7 @@ _add_child (GESContainer * container, GESTimelineElement * element)
   GESTimeline *timeline = GES_TIMELINE_ELEMENT_TIMELINE (container);
   GESClipPrivate *priv = self->priv;
   GESAsset *asset, *creator_asset;
+  gboolean adding_scale_effect = FALSE;
   gboolean prev_prevent = priv->prevent_duration_limit_update;
   gboolean prev_prevent_outpoint = priv->prevent_children_outpoint_update;
   GList *tmp;
@@ -1775,6 +1777,14 @@ _add_child (GESContainer * container, GESTimelineElement * element)
           new_prio = MAX (new_prio, _PRIORITY (tmp->data) + 1);
       }
     }
+
+    if (GES_IS_EFFECT (element)) {
+      GESAsset *asset = ges_extractable_get_asset (GES_EXTRACTABLE (element));
+      const gchar *bindesc = ges_asset_get_id (asset);
+
+      adding_scale_effect = !strstr (bindesc, "gesvideoscale");
+    }
+
     /* make sure higher than core */
     for (tmp = container->children; tmp; tmp = tmp->next) {
       if (_IS_CORE_CHILD (tmp->data))
@@ -1817,9 +1827,16 @@ _add_child (GESContainer * container, GESTimelineElement * element)
     _update_active_for_track (self, track_el);
 
     priv->nb_effects++;
+
     GST_DEBUG_OBJECT (self, "Adding %ith effect: %" GES_FORMAT
         " Priority %i", priv->nb_effects, GES_ARGS (element), new_prio);
 
+    if (adding_scale_effect) {
+      GST_DEBUG_OBJECT (self, "Adding scaling effect to clip "
+          "%" GES_FORMAT, GES_ARGS (self));
+      priv->nb_scale_effects += 1;
+    }
+
     /* changing priorities, and updating their offset */
     priv->prevent_resort = TRUE;
     priv->setting_priority = TRUE;
@@ -1900,6 +1917,12 @@ ges_clip_set_remove_error (GESClip * clip, GError * error)
   priv->remove_error = error;
 }
 
+gboolean
+ges_clip_has_scale_effect (GESClip * clip)
+{
+  return clip->priv->nb_scale_effects > 0;
+}
+
 static gboolean
 _remove_child (GESContainer * container, GESTimelineElement * element)
 {
@@ -1961,6 +1984,17 @@ _remove_child (GESContainer * container, GESTimelineElement * element)
      * relative priorities */
     /* height may have changed */
     _compute_height (container);
+
+    if (GES_IS_EFFECT (element)) {
+      GESAsset *asset = ges_extractable_get_asset (GES_EXTRACTABLE (element));
+      const gchar *bindesc = ges_asset_get_id (asset);
+
+      if (bindesc && !strstr (bindesc, "gesvideoscale")) {
+        GST_DEBUG_OBJECT (self, "Removing scaling effect to clip "
+            "%" GES_FORMAT, GES_ARGS (self));
+        priv->nb_scale_effects -= 1;
+      }
+    }
   }
   /* duration-limit updated in _child_removed */
   return TRUE;

subprojects/gst-editing-services/ges/ges-effect.c

@@ -23,10 +23,23 @@
  * @short_description: adds an effect build from a parse-launch style bin
  * description to a stream in a GESSourceClip or a GESLayer
  *
- * Currently we only support effects with N sinkpads and one single srcpad.
- * Apart from `gesaudiomixer` and `gescompositor` which can be used as effects
- * and where sinkpads will be requested as needed based on the timeline topology
- * GES will always request at most one sinkpad per effect (when required).
+ * Any GStreamer filter can be used as effects in GES. The only restriction we
+ * have is that effects element should have a single [sinkpad](GST_PAD_SINK)
+ * (which will be requested if necessary) and a single [srcpad](GST_PAD_SRC).
+ *
+ * Note that `gesaudiomixer` and `gescompositor` can be used as effects even
+ * though they can have several sinkpads.
+ *
+ * ## GES specific effects:
+ *
+ * * **`gesvideoscale`**: GES implements a specific scaling bin that allows
+ *   specifying where scaling will happen inside the chain of effects. By
+ *   default scaling can happen either in the source (if the source doesn't have
+ *   a specific size, like `videotestsrc` or [mixing](ges_track_set_mixing) has
+ *   been disabled) or in the mixing element otherwise, when adding that element
+ *   as an effect, GES guarantees that the scaling will happen in it. This can
+ *   be useful for example if you want to crop the video before scaling or apply
+ *   rounding corners to the video after scaling, etc...
  *
  * > Note: GES always adds converters (`audioconvert ! audioresample !
  * > audioconvert` for audio effects and `videoconvert` for video effects) to

subprojects/gst-editing-services/ges/ges-internal.h

@@ -462,6 +462,7 @@ G_GNUC_INTERNAL void              ges_clip_set_add_error          (GESClip * cli
 G_GNUC_INTERNAL void              ges_clip_take_add_error         (GESClip * clip, GError ** error);
 G_GNUC_INTERNAL void              ges_clip_set_remove_error       (GESClip * clip, GError * error);
 G_GNUC_INTERNAL void              ges_clip_take_remove_error      (GESClip * clip, GError ** error);
+G_GNUC_INTERNAL gboolean          ges_clip_has_scale_effect       (GESClip * clip);
 
 /****************************************************
  *              GESLayer                            *

subprojects/gst-editing-services/ges/ges-smart-video-mixer.c

@@ -240,8 +240,13 @@ set_pad_properties_from_composition_meta (GstPad * mixer_pad,
     g_object_set (mixer_pad, "alpha", meta->alpha * transalpha, NULL);
   }
 
-  g_object_set (mixer_pad, "xpos", meta->posx, "ypos",
-      meta->posy, "width", meta->width, "height", meta->height, NULL);
+  g_object_set (mixer_pad, "xpos", meta->posx, "ypos", meta->posy, NULL);
+
+  if (meta->width >= 0)
+    g_object_set (mixer_pad, "width", meta->width, NULL);
+
+  if (meta->height >= 0)
+    g_object_set (mixer_pad, "height", meta->height, NULL);
 
   if (self->ABI.abi.has_operator)
     g_object_set (mixer_pad, "operator", meta->operator, NULL);

subprojects/gst-editing-services/ges/ges.c

@@ -54,6 +54,7 @@ G_LOCK_DEFINE_STATIC (init_lock);
  * between init/deinit
  */
 static GThread *initialized_thread = NULL;
+extern GType ges_video_scale_get_type (void);
 
 #ifndef GST_DISABLE_GST_DEBUG
 static gpointer
@@ -96,9 +97,10 @@ ges_init_pre (GOptionContext * context, GOptionGroup * group, gpointer data,
   return TRUE;
 }
 
+
 static gboolean
-ges_init_post (GOptionContext * context, GOptionGroup * group, gpointer data,
-    GError ** error)
+ges_init_post (GOptionContext * context, GOptionGroup * group,
+    gpointer data, GError ** error)
 {
   GESUriClipAssetClass *uriasset_klass = NULL;
   GstElementFactory *nlecomposition_factory = NULL;
@@ -154,6 +156,7 @@ ges_init_post (GOptionContext * context, GOptionGroup * group, gpointer data,
 
   ges_asset_cache_init ();
 
+  gst_element_register (NULL, "gesvideoscale", 0, ges_video_scale_get_type ());
   gst_element_register (NULL, "gesaudiomixer", 0, GES_TYPE_SMART_ADDER);
   gst_element_register (NULL, "gescompositor", 0, GES_TYPE_SMART_MIXER);
   gst_element_register (NULL, "framepositioner", 0, GST_TYPE_FRAME_POSITIONNER);

subprojects/gst-editing-services/ges/gesvideoscale.c

@@ -0,0 +1,174 @@
+/* GStreamer
+ * Copyright (C) 2023 Thibault Saunier <tsaunier@igalia.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
+ * Boston, MA 02110-1335, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <gst/gst.h>
+
+#include "ges-frame-composition-meta.h"
+
+typedef struct _GESVideoScale GESVideoScale;
+typedef struct
+{
+  GstBinClass parent_class;
+} GESVideoScaleClass;
+
+struct _GESVideoScale
+{
+  GstBin parent;
+
+  GstPad *sink;
+  GstElement *capsfilter;
+
+  gint width, height;
+};
+
+/* *INDENT-OFF* */
+static GstStaticPadTemplate gst_video_scale_sink_template =
+    GST_STATIC_PAD_TEMPLATE ("sink",
+        GST_PAD_SINK,
+        GST_PAD_ALWAYS,
+      GST_STATIC_CAPS ("ANY")
+    );
+
+static GstStaticPadTemplate gst_video_scale_src_template =
+    GST_STATIC_PAD_TEMPLATE ("src",
+      GST_PAD_SRC,
+      GST_PAD_ALWAYS,
+      GST_STATIC_CAPS ("ANY")
+    );
+
+GES_DECLARE_TYPE (VideoScale, video_scale, VIDEO_SCALE)
+G_DEFINE_TYPE (GESVideoScale, ges_video_scale, GST_TYPE_BIN);
+/* *INDENT-ON* */
+
+static void
+set_dimension (GESVideoScale * self, gint width, gint height)
+{
+  GstCaps *caps = gst_caps_new_simple ("video/x-raw",
+      "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
+      NULL);
+
+  if (width >= 0)
+    gst_caps_set_simple (caps, "width", G_TYPE_INT, width, NULL);
+  if (height >= 0)
+    gst_caps_set_simple (caps, "height", G_TYPE_INT, height, NULL);
+
+  gst_caps_set_features (caps, 0, gst_caps_features_new_any ());
+  g_object_set (self->capsfilter, "caps", caps, NULL);
+  gst_caps_unref (caps);
+
+  GST_OBJECT_LOCK (self);
+  self->width = width;
+  self->height = height;
+  GST_OBJECT_UNLOCK (self);
+}
+
+static GstFlowReturn
+chain (GstPad * pad, GESVideoScale * self, GstBuffer * buffer)
+{
+  GESFrameCompositionMeta *meta;
+
+  meta =
+      (GESFrameCompositionMeta *) gst_buffer_get_meta (buffer,
+      ges_frame_composition_meta_api_get_type ());
+
+  if (meta) {
+    GST_OBJECT_LOCK (self);
+    if (meta->height != self->height || meta->width != self->width) {
+      GST_OBJECT_UNLOCK (self);
+
+      set_dimension (self, meta->width, meta->height);
+    } else {
+      GST_OBJECT_UNLOCK (self);
+    }
+
+    meta->height = meta->width = -1;
+  }
+
+  return gst_proxy_pad_chain_default (pad, GST_OBJECT (self), buffer);
+}
+
+static GstStateChangeReturn
+change_state (GstElement * element, GstStateChange transition)
+{
+  GESVideoScale *self = GES_VIDEO_SCALE (element);
+  GstStateChangeReturn res =
+      ((GstElementClass *) ges_video_scale_parent_class)->change_state (element,
+      transition);
+
+  if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) {
+    GST_OBJECT_LOCK (self);
+    self->width = 0;
+    self->height = 0;
+    GST_OBJECT_UNLOCK (self);
+  }
+
+  return res;
+}
+
+static void
+ges_video_scale_init (GESVideoScale * self)
+{
+  GstPad *pad;
+  GstElement *scale;
+  GstPadTemplate *template =
+      gst_static_pad_template_get (&gst_video_scale_sink_template);
+
+  scale = gst_element_factory_make ("videoscale", NULL);
+  g_object_set (scale, "add-borders", FALSE, NULL);
+  self->capsfilter = gst_element_factory_make ("capsfilter", NULL);
+
+  gst_bin_add_many (GST_BIN (self), scale, self->capsfilter, NULL);
+  gst_element_link (scale, self->capsfilter);
+
+  self->sink =
+      gst_ghost_pad_new_from_template ("sink", scale->sinkpads->data, template);
+  gst_pad_set_chain_function (self->sink, (GstPadChainFunction) chain);
+  gst_element_add_pad (GST_ELEMENT (self), self->sink);
+  gst_object_unref (template);
+
+  template = gst_static_pad_template_get (&gst_video_scale_src_template);
+  pad =
+      gst_ghost_pad_new_from_template ("src", self->capsfilter->srcpads->data,
+      template);
+  gst_element_add_pad (GST_ELEMENT (self), pad);
+  gst_object_unref (template);
+}
+
+static void
+ges_video_scale_class_init (GESVideoScaleClass * klass)
+{
+  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
+
+  gst_element_class_set_static_metadata (element_class,
+      "VideoScale",
+      "Video/Filter",
+      "Scaling element usable as a GES effect",
+      "Thibault Saunier <tsaunier@igalia.com>");
+
+  gst_element_class_add_static_pad_template (element_class,
+      &gst_video_scale_sink_template);
+  gst_element_class_add_static_pad_template (element_class,
+      &gst_video_scale_src_template);
+
+  element_class->change_state = change_state;
+}

subprojects/gst-editing-services/ges/gstframepositioner.c

@@ -113,6 +113,24 @@ gst_compositor_operator_get_type_and_default_value (int *default_operator_value)
   return operator_gtype;
 }
 
+static gboolean
+scales_downstream (GstFramePositioner * self)
+{
+  if (self->scale_in_compositor)
+    return TRUE;
+
+  if (!self->track_source)
+    return self->scale_in_compositor;
+
+  GESTimelineElement *parent = GES_TIMELINE_ELEMENT_PARENT (self->track_source);
+
+  if (!parent || !GES_IS_CLIP (parent)) {
+    return self->scale_in_compositor;
+  }
+
+  return ges_clip_has_scale_effect (GES_CLIP (parent));
+}
+
 static void
 _weak_notify_cb (GstFramePositioner * pos, GObject * old)
 {
@@ -274,7 +292,7 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos,
   caps = gst_caps_from_string ("video/x-raw(ANY)");
 
   if (pos->track_width && pos->track_height &&
-      (!track_mixing || !pos->scale_in_compositor)) {
+      (!track_mixing || !scales_downstream (pos))) {
     gst_caps_set_simple (caps, "width", G_TYPE_INT,
         pos->track_width, "height", G_TYPE_INT, pos->track_height, NULL);
   }
@@ -321,6 +339,13 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos,
   reposition_properties (pos, old_track_width, old_track_height);
 
 done:
+  if (scales_downstream (pos) && pos->natural_width && pos->natural_height) {
+    GST_DEBUG_OBJECT (pos,
+        "Forcing natural width in source make downstream scaling work");
+    gst_caps_set_simple (caps, "width", G_TYPE_INT, pos->natural_width,
+        "height", G_TYPE_INT, pos->natural_height, NULL);
+  }
+
   GST_DEBUG_OBJECT (pos, "setting caps %" GST_PTR_FORMAT, caps);
 
   g_object_set (pos->capsfilter, "caps", caps, NULL);

subprojects/gst-editing-services/ges/meson.build

@@ -67,6 +67,7 @@ ges_sources = files([
     'ges-structure-parser.c',
     'ges-marker-list.c',
     'ges-discoverer-manager.c',
+    'gesvideoscale.c',
     'gstframepositioner.c'
 ])
 

subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest

@@ -0,0 +1,37 @@
+set-globals, media_file="$(test_dir)/../../medias/defaults/matroska/timed_frames_video_only_1fps.mkv"
+meta,
+    tool = "ges-launch-$(gst_api_version)",
+    handles-states=true,
+    seek=true,
+    needs_preroll=true,
+    args = {
+        --track-types, video,
+        --video-caps, "video/x-raw, format=(string)I420, width=(int)1080, height=(int)720, framerate=(fraction)1/1",
+        --videosink, "$(videosink) name=videosink",
+    }
+
+# Add a clip and check that the first frame is displayed
+add-clip, name=clip, asset-id="file://$(media_file)", layer-priority=0, type=GESUriClip, name=(string)theclip
+set-child-properties, width=100, height=100, posx=10, posy=10, element-name=theclip
+container-add-child, container-name=theclip, asset-id="videocrop name=videocrop", child-type=(string)GESEffect, child-name=crop;
+set-child-properties, bottom=200, element-name=crop
+container-add-child, container-name=theclip, asset-id="gesvideoscale name=videoscale", child-type=(string)GESEffect;
+
+pause
+
+# Checking that the 'framepositioner' is forcing caps to the clip "natual size"
+check-properties, gesvideourisource0-capsfilter::caps=(GstCaps)"video/x-raw(ANY),framerate=(fraction)1/1,width=(int)1080,height=(int)720"
+check-properties, videoscale::parent::parent::priority=3
+check-properties, videocrop::parent::parent::priority=4
+check-current-pad-caps, target-element-name=videocrop, pad=sink, expected-caps=[video/x-raw,width=1080,height=720]
+check-current-pad-caps, target-element-name=videocrop, pad=src, expected-caps=[video/x-raw,width=1080,height=520]
+check-current-pad-caps, target-element-name=videoscale, pad=sink, expected-caps=[video/x-raw,width=1080,height=520]
+check-current-pad-caps, target-element-name=videoscale, pad=src, expected-caps=[video/x-raw,width=100,height=100]
+
+check-properties,
+    gessmartmixer0-compositor.sink_0::xpos=10,
+    gessmartmixer0-compositor.sink_0::ypos=10,
+    gessmartmixer0-compositor.sink_0::width=-1,
+    gessmartmixer0-compositor.sink_0::height=-1
+
+stop

subprojects/gst-integration-testsuites/testsuites/ges.testslist

@@ -760,3 +760,4 @@ ges.test.edit_deeply_nested_timeline_too_short
 ges.test.play_deeply_nested_back_to_back
 ges.test.play_two_nested_back_to_back
 ges.test.seek_on_stack_change_on_internal_subtimeline
+ges.test.videoscale_effect