Merge branch 'VMAFPlugin20Api' into 'main'

vmaf: create new VMAF plugin

See merge request gstreamer/gstreamer!1250

subprojects/gst-plugins-bad/ext/meson.build

@@ -65,6 +65,7 @@ subdir('svtav1')
 subdir('svthevcenc')
 subdir('teletextdec')
 subdir('ttml')
+subdir('vmaf')
 subdir('voaacenc')
 subdir('voamrwbenc')
 subdir('vulkan')

subprojects/gst-plugins-bad/ext/vmaf/gstvmafconvert.c

@@ -0,0 +1,115 @@
+/* VMAF plugin
+ * Copyright (C) 2021 Hudl
+ *   @author: Casey Bateman <Casey.Bateman@hudl.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#include "gstvmafconvert.h"
+#include "gstvmafenums.h"
+
+enum VmafOutputFormat
+vmaf_map_log_fmt (GstVmafLogFormats log_fmt)
+{
+  if (log_fmt == OUTPUT_FORMAT_CSV)
+    return VMAF_OUTPUT_FORMAT_CSV;
+  if (log_fmt == OUTPUT_FORMAT_XML)
+    return VMAF_OUTPUT_FORMAT_XML;
+  if (log_fmt == OUTPUT_FORMAT_JSON)
+    return VMAF_OUTPUT_FORMAT_JSON;
+  return VMAF_OUTPUT_FORMAT_NONE;
+}
+
+gint
+vmaf_map_pix_fmt (const char *fmt)
+{
+  if (fmt) {
+    if (!strcmp (fmt, "I420"))
+      return VMAF_PIX_FMT_YUV420P;
+    if (!strcmp (fmt, "NV12"))
+      return VMAF_PIX_FMT_YUV420P;
+    if (!strcmp (fmt, "YV12"))
+      return VMAF_PIX_FMT_YUV420P;
+    if (!strcmp (fmt, "Y42B"))
+      return VMAF_PIX_FMT_YUV422P;
+    if (!strcmp (fmt, "Y444"))
+      return VMAF_PIX_FMT_YUV444P;
+    if (!strcmp (fmt, "I420_10LE"))
+      return VMAF_PIX_FMT_YUV420P;
+    if (!strcmp (fmt, "I422_10LE"))
+      return VMAF_PIX_FMT_YUV422P;
+    if (!strcmp (fmt, "Y444_10LE"))
+      return VMAF_PIX_FMT_YUV444P;
+  }
+  return VMAF_PIX_FMT_UNKNOWN;
+}
+
+gint
+vmaf_map_bit_depth (const char *fmt)
+{
+  if (fmt) {
+    if (!strcmp (fmt, "I420_10LE"))
+      return 10;
+    if (!strcmp (fmt, "I422_10LE"))
+      return 10;
+    if (!strcmp (fmt, "Y444_10LE"))
+      return 10;
+  }
+  return 8;
+}
+
+enum VmafPoolingMethod
+vmaf_map_pooling_method (GstVmafPoolMethodEnum pool_method)
+{
+  if (pool_method == POOL_METHOD_MAX)
+    return VMAF_POOL_METHOD_MAX;
+  if (pool_method == POOL_METHOD_MIN)
+    return VMAF_POOL_METHOD_MIN;
+  if (pool_method == POOL_METHOD_MEAN)
+    return VMAF_POOL_METHOD_MEAN;
+  if (pool_method == POOL_METHOD_HARMONIC_MEAN)
+    return VMAF_POOL_METHOD_HARMONIC_MEAN;
+  return VMAF_POOL_METHOD_UNKNOWN;
+}
+
+void
+fill_vmaf_picture_buffer (float *src, VmafPicture * dst, unsigned width,
+    unsigned height, int src_stride)
+{
+  float *a = src;
+  uint8_t *b = dst->data[0];
+  for (unsigned i = 0; i < height; i++) {
+    for (unsigned j = 0; j < width; j++) {
+      b[j] = a[j];
+    }
+    a += src_stride / sizeof (float);
+    b += dst->stride[0];
+  }
+}
+
+void
+fill_vmaf_picture_buffer_hbd (float *src, VmafPicture * dst, unsigned width,
+    unsigned height, int src_stride, unsigned bpc)
+{
+  float *a = src;
+  uint16_t *b = dst->data[0];
+  for (unsigned i = 0; i < height; i++) {
+    for (unsigned j = 0; j < width; j++) {
+      b[j] = a[j] * (1 << (bpc - 8));
+    }
+    a += src_stride / sizeof (float);
+    b += dst->stride[0] / sizeof (uint16_t);
+  }
+}

subprojects/gst-plugins-bad/ext/vmaf/gstvmafconvert.h

@@ -0,0 +1,36 @@
+/* VMAF plugin
+ * Copyright (C) 2021 Hudl
+ *   @author: Casey Bateman <Casey.Bateman@hudl.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifndef __GST_VMAFCONVERT_H__
+#define __GST_VMAFCONVERT_H__
+
+#include <libvmaf.h>
+#include <gst/gst.h>
+#include "gstvmafenums.h"
+
+G_BEGIN_DECLS 
+enum VmafOutputFormat vmaf_map_log_fmt (GstVmafLogFormats log_fmt);
+gint vmaf_map_pix_fmt (const char *fmt);
+gint vmaf_map_bit_depth (const char *fmt);
+enum VmafPoolingMethod vmaf_map_pooling_method(GstVmafPoolMethodEnum pool_method);
+void fill_vmaf_picture_buffer (float *src, VmafPicture * dst, unsigned width, unsigned height, int src_stride);
+void fill_vmaf_picture_buffer_hbd (float *src, VmafPicture * dst, unsigned width, unsigned height, int src_stride, unsigned bpc);
+
+G_END_DECLS
+#endif /* __GST_VMAFCONVERT_H__ */

subprojects/gst-plugins-bad/ext/vmaf/gstvmafelements.h

@@ -0,0 +1,105 @@
+/* VMAF plugin
+ * Copyright (C) 2021 Hudl
+ *   @author: Casey Bateman <Casey.Bateman@hudl.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifndef __GST_VMAFELEMENT_H__
+#define __GST_VMAFELEMENT_H__
+
+#include <libvmaf.h>
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/video/gstvideoaggregator.h>
+#include "gstvmafenums.h"
+
+G_BEGIN_DECLS
+#define GST_TYPE_VMAF (gst_vmaf_get_type())
+#define GST_VMAF(obj) \
+        (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VMAF, GstVmaf))
+#define GST_VMAF_CLASS(klass) \
+        (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VMAF, GstVmafClass))
+#define GST_IS_VMAF(obj) \
+        (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VMAF))
+#define GST_IS_VMAF_CLASS(klass) \
+        (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VMAF))
+typedef struct _GstVmaf GstVmaf;
+typedef struct _GstVmafClass GstVmafClass;
+
+typedef struct
+{
+  void *ref_ptr;
+  void *dist_ptr;
+  gint frame_index;
+} GstVmafQueueElem;
+
+typedef struct
+{
+  GstVmaf *gst_vmaf_p;
+  GstTask *vmaf_thread;
+  GRecMutex vmaf_thread_mutex;
+  GAsyncQueue *frame_queue;
+  GMutex check_thread_failure;
+  gboolean thread_eos;
+  gboolean thread_failure;
+  gint stream_index;
+  gint vmaf_pix_fmt;
+  gint frame_height;
+  gint frame_width;
+  gint frame_index;
+  gint last_frame_processed;
+  gint bpc;
+  gchar *padname;
+  VmafContext *vmaf_ctx;
+  VmafModel *vmaf_model;
+  VmafModelCollection *vmaf_model_collection;
+} GstVmafThreadHelper;
+
+struct _GstVmaf
+{
+  GstVideoAggregator videoaggregator;
+  // VMAF settings from cmd
+  GstVmafPoolMethodEnum vmaf_config_pool_method;
+  GstVmafLogFormats vmaf_config_log_format;
+  gchar *vmaf_config_model_filename;
+  gboolean vmaf_config_disable_clip;
+  gboolean vmaf_config_disable_avx;
+  gboolean vmaf_config_enable_transform;
+  gboolean vmaf_config_phone_model;
+  gboolean vmaf_config_psnr;
+  gboolean vmaf_config_ssim;
+  gboolean vmaf_config_ms_ssim;
+  guint vmaf_config_num_threads;
+  guint vmaf_config_subsample;
+  gboolean vmaf_config_conf_int;
+  gboolean vmaf_config_frame_messaging;
+  gchar *vmaf_config_log_filename;
+  // Thread helpers
+  GstVmafThreadHelper *helper_struct_pointer;
+  gint number_of_input_streams;
+  gboolean finish_threads;
+  GMutex finish_mutex;
+};
+
+struct _GstVmafClass
+{
+  GstVideoAggregatorClass parent_class;
+};
+
+GType gst_vmaf_get_type (void);
+
+G_END_DECLS
+#endif /* __GST_VMAFELEMENT_H__ */

subprojects/gst-plugins-bad/ext/vmaf/gstvmafenums.h

@@ -0,0 +1,81 @@
+/* VMAF plugin
+ * Copyright (C) 2021 Hudl
+ *   @author: Casey Bateman <Casey.Bateman@hudl.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifndef __GST_VMAFENUMS_H__
+#define __GST_VMAFENUMS_H__
+
+#include <gst/gst.h>
+
+G_BEGIN_DECLS 
+
+typedef enum _GstVmafPoolMethodEnum
+{
+  POOL_METHOD_UNKNOWN = 0,
+  POOL_METHOD_MIN = 1,
+  POOL_METHOD_MAX = 2,
+  POOL_METHOD_MEAN = 3,
+  POOL_METHOD_HARMONIC_MEAN = 4,
+  POOL_METHOD_NB = 5,
+} GstVmafPoolMethodEnum;
+
+typedef enum _GstReadFrameReturnCodes
+{
+  READING_SUCCESSFUL = 0,
+  READING_ERROR = 1,
+  READING_EOS = 2,
+} GstReadFrameReturnCodes;
+
+typedef enum _GstVmafPropertyTypes
+{
+  PROP_0,
+  PROP_MODEL_FILENAME,
+  PROP_DISABLE_CLIP,
+  PROP_DISABLE_AVX,
+  PROP_ENABLE_TRANSFORM,
+  PROP_PHONE_MODEL,
+  PROP_PSNR,
+  PROP_SSIM,
+  PROP_MS_SSIM,
+  PROP_NUM_THREADS,
+  PROP_SUBSAMPLE,
+  PROP_CONF_INT,
+  PROP_LAST,
+  PROP_POOL_METHOD,
+  PROP_FRAME_MESSAGING,
+  PROP_VMAF_LOG_FORMAT,
+  PROP_VMAF_LOG_FILENAME,
+} GstVmafPropertyTypes;
+
+typedef enum _GstVmafMessageBusScoreTypes
+{
+  MESSAGE_TYPE_FRAME = 0,
+  MESSAGE_TYPE_POOLED = 1,
+} GstVmafMessageBusScoreTypes;
+
+typedef enum _GstVmafLogFormats
+{
+  OUTPUT_FORMAT_NONE = 0,
+  OUTPUT_FORMAT_CSV = 1,
+  OUTPUT_FORMAT_XML = 2,
+  OUTPUT_FORMAT_JSON = 3,
+} GstVmafLogFormats;
+
+
+G_END_DECLS
+#endif /* __GST_VMAFENUMS_H__ */

subprojects/gst-plugins-bad/ext/vmaf/gstvmafplugin.c

@@ -0,0 +1,38 @@
+/* VMAF plugin
+ * Copyright (C) 2021 Hudl
+ *   @author: Casey Bateman <Casey.Bateman@hudl.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "gstvmafelements.h"
+
+static gboolean
+plugin_init (GstPlugin * plugin)
+{
+  gboolean result =
+      gst_element_register (plugin, "vmaf", GST_RANK_PRIMARY, GST_TYPE_VMAF);
+  return result;
+}
+
+GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
+    GST_VERSION_MINOR,
+    vmaf,
+    "Netflix VMAF quality metric plugin",
+    plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)

subprojects/gst-plugins-bad/ext/vmaf/meson.build

@@ -0,0 +1,20 @@
+vmaf_sources = [
+  'gstvmafconvert.c',
+  'gstvmafelement.c',
+  'gstvmafplugin.c',
+]
+
+libvmaf_dep = dependency('libvmaf', required : get_option('vmaf'))
+if libvmaf_dep.found()
+  gstvmaf = library('gstvmaf',
+    vmaf_sources,
+    c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API', '-DHAVE_LIBVMAF'],
+    include_directories : [configinc],
+    dependencies : [gstvideo_dep, gstbase_dep, gst_dep, libvmaf_dep],
+    install : true,
+    install_dir : plugins_install_dir,
+  )
+  pkgconfig.generate(gstvmaf, install_dir : plugins_pkgconfig_install_dir)
+  plugins += [gstvmaf]
+endif
+

subprojects/gst-plugins-bad/meson_options.txt

@@ -176,6 +176,7 @@ option('transcode', type : 'feature', value : 'auto', description : 'Transcode p
 option('ttml', type : 'feature', value : 'auto', description : 'TTML subtitle parser and renderer plugin')
 option('uvch264', type : 'feature', value : 'auto', description : 'UVC compliant H.264 camera source plugin')
 option('va', type : 'feature', value : 'auto', description: 'VA-API new plugin')
+option('vmaf', type : 'feature', value : 'auto', description : 'Netflix VMAF image quality assessment plugin')
 option('voaacenc', type : 'feature', value : 'auto', description : 'AAC audio encoder plugin')
 option('voamrwbenc', type : 'feature', value : 'auto', description : 'AMR-WB audio encoder plugin')
 option('vulkan', type : 'feature', value : 'auto', description : 'Vulkan video sink plugin')