From 184d94305cbe5d93832f7d5c58bb41e625e20363 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 10 Apr 2024 15:05:12 +1000 Subject: [PATCH] glcolorconvert: add support for planar yuv->planar yuv conversions Currently only supported by keeping the same colorimetry and is only a repacking operation. Part-of: --- .../gst-libs/gst/gl/gstglcolorconvert.c | 604 ++++++++++++------ 1 file changed, 402 insertions(+), 202 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c index 5fee6ad223..6003b20503 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c @@ -103,7 +103,7 @@ typedef struct "uniform int input_swizzle[4];\n" \ "uniform int output_swizzle[4];\n" -#define MAX_FUNCTIONS 4 +#define MAX_FUNCTIONS 5 #define glsl_OES_extension_string "#extension GL_OES_EGL_image_external : require \n" @@ -211,77 +211,204 @@ static const struct shader_templ templ_RGB_to_AYUV = GST_GL_TEXTURE_TARGET_2D }; +static const char glsl_func_planar_yuv_to_yuva[] = + "vec4 fetch_planar_yuv(sampler2D Ytex, sampler2D Utex, sampler2D Vtex, vec2 texcoord) {\n" + " vec4 yuva;\n" + " yuva.x = texture2D(Ytex, texcoord * tex_scale0).r * in_bitdepth_factor;\n" + " yuva.y = texture2D(Utex, texcoord * tex_scale1).r * in_bitdepth_factor;\n" + " yuva.z = texture2D(Vtex, texcoord * tex_scale2).r * in_bitdepth_factor;\n" + " yuva.a = 1.0;\n" + " return yuva;\n" + "}\n" + "vec4 fetch_planar_yuva(sampler2D Ytex, sampler2D Utex, sampler2D Vtex, sampler2D Atex, vec2 texcoord) {\n" + " vec4 yuva = fetch_planar_yuv(Ytex, Utex, Vtex, texcoord);\n" + " yuva.a = texture2D(Atex, texcoord * tex_scale3).r * in_bitdepth_factor;\n" + " return yuva;\n" + "}\n"; + /* YUV to RGB conversion */ static const gchar templ_PLANAR_YUV_to_RGB_BODY[] = /* FIXME: should get the sampling right... */ "vec4 yuva = fetch_planar_yuv(Ytex, Utex, Vtex, texcoord);\n" "yuva = swizzle(yuva, input_swizzle);\n" "vec4 rgba = color_matrix_apply(yuva, to_RGB_matrix);\n" - "%s" "gl_FragColor = swizzle(rgba, output_swizzle);\n"; static const struct shader_templ templ_PLANAR_YUV_to_RGB = { NULL, DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex;\n" "uniform float in_bitdepth_factor;\n", - { glsl_func_swizzle, glsl_func_color_matrix, NULL, }, + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_planar_yuv_to_yuva, NULL, }, GST_GL_TEXTURE_TARGET_2D }; +static const gchar templ_PLANAR_YUVA_to_RGB_BODY[] = + /* FIXME: should get the sampling right... */ + "vec4 yuva = fetch_planar_yuva(Ytex, Utex, Vtex, Atex, texcoord);\n" + "yuva = swizzle(yuva, input_swizzle);\n" + "vec4 rgba = color_matrix_apply(yuva, to_RGB_matrix);\n" + "gl_FragColor = swizzle(rgba, output_swizzle);\n"; static const struct shader_templ templ_A420_to_RGB = { NULL, /* 4th uniform is the alpha buffer */ DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex, Atex;\n" "uniform float in_bitdepth_factor;\n", - { glsl_func_swizzle, glsl_func_color_matrix, NULL, }, + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_planar_yuv_to_yuva, NULL, }, GST_GL_TEXTURE_TARGET_2D }; +static const char glsl_func_chroma_sample[] = + "vec4 chroma_sample(sampler2D tex, vec2 texcoord, vec2 unnormalization) {\n" + " vec4 uv_texel = vec4(0.0);\n" + /* One u and v sample can be generated by a nxm sized block given by + * @chroma_sampling. The result is the average of all the values in the + * block computed with a rolling average. + */ + /* scale for chroma size */ + " vec2 chroma_pos = texcoord * chroma_sampling * unnormalization;\n" + /* offset chroma to the center of the first texel in the block */ + " chroma_pos -= clamp(chroma_sampling * 0.5 - 0.5, vec2(0.0), chroma_sampling);\n" + " if (chroma_pos.x < width && chroma_pos.y < height) {\n" + " for (int i = 0; i < int(chroma_sampling.x); i++) {\n" + " vec2 delta = vec2 (float(i), 0.0);\n" + " for (int j = 0; j < int(chroma_sampling.y); j++) {\n" + " int n = (i+1)*(j+1);\n" + " delta.y = float(j);\n" + " vec4 s = swizzle(texture2D(tex, (chroma_pos + delta) / unnormalization), input_swizzle);\n" + /* rolling average */ + " uv_texel = (float(n-1) * uv_texel + s) / float(n);\n" + " }\n" + " }\n" + " }\n" + " return uv_texel;\n" + "}\n"; + +static const char glsl_func_write_planar_yuv[] = + "void write_planar_yuv(vec4 yuva) {\n" + " gl_FragData[0] = vec4(yuva.x, 0.0, 0.0, 1.0);\n" + " gl_FragData[1] = vec4(yuva.y, 0.0, 0.0, 1.0);\n" + " gl_FragData[2] = vec4(yuva.z, 0.0, 0.0, 1.0);\n" + "}\n"; +static const char glsl_func_write_planar_yuva[] = + "void write_planar_yuva(vec4 yuva) {\n" + " gl_FragData[0] = vec4(yuva.x, 0.0, 0.0, 1.0);\n" + " gl_FragData[1] = vec4(yuva.y, 0.0, 0.0, 1.0);\n" + " gl_FragData[2] = vec4(yuva.z, 0.0, 0.0, 1.0);\n" + " gl_FragData[3] = vec4(yuva.a, 0.0, 0.0, 1.0);\n" + "}\n"; + static const gchar templ_RGB_to_PLANAR_YUV_BODY[] = - "vec4 texel;\n" - "vec4 yuva;\n" - "texel = swizzle(texture2D(tex, texcoord), input_swizzle);\n" - /* FIXME: this is not quite correct yet */ - "vec4 uv_texel = vec4(0.0);\n" - /* One u and v sample can be generated by a nxm sized block given by */ - /* @chroma_sampling. The result is the average of all the values in the */ - /* block computed with a rolling average. */ + "vec4 texel = swizzle(texture2D(tex, texcoord), input_swizzle);\n" "vec2 unnormalization;\n" "if (texcoord.x == v_texcoord.x) {\n" " unnormalization = vec2(width, height);\n" "} else {\n" " unnormalization = vec2 (1.0);\n" "}\n" - /* scale for chroma size */ - "vec2 chroma_pos = texcoord * chroma_sampling * unnormalization;\n" - /* offset chroma to the center of the first texel in the block */ - "chroma_pos -= clamp(chroma_sampling * 0.5 - 0.5, vec2(0.0), chroma_sampling);\n" - "if (chroma_pos.x < width && chroma_pos.y < height) {\n" - " for (int i = 0; i < int(chroma_sampling.x); i++) {\n" - " vec2 delta = vec2 (float(i), 0.0);\n" - " for (int j = 0; j < int(chroma_sampling.y); j++) {\n" - " int n = (i+1)*(j+1);\n" - " delta.y = float(j);\n" - " vec4 s = swizzle(texture2D(tex, (chroma_pos + delta) / unnormalization), input_swizzle);\n" - /* rolling average */ - " uv_texel = (float(n-1) * uv_texel + s) / float(n);\n" - " }\n" - " }\n" - "}\n" + "vec4 uv_texel = chroma_sample(tex, texcoord, unnormalization);\n" + "vec4 yuva;\n" "yuva.x = color_matrix_apply(texel, to_YUV_matrix).x;\n" "yuva.yz = color_matrix_apply(uv_texel, to_YUV_matrix).yz;\n" "yuva.a = texel.a;\n" - "yuva = swizzle(yuva, output_swizzle);\n" - "yuva = yuva * out_bitdepth_factor;\n" - "gl_FragData[0] = vec4(yuva.x, 0.0, 0.0, 1.0);\n" - "gl_FragData[1] = vec4(yuva.y, 0.0, 0.0, 1.0);\n" - "gl_FragData[2] = vec4(yuva.z, 0.0, 0.0, 1.0);\n" - "%s"; + "yuva = swizzle(yuva, output_swizzle) * out_bitdepth_factor;\n" + "write_planar_yuv(yuva);\n"; static const struct shader_templ templ_RGB_to_PLANAR_YUV = { NULL, DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n" "uniform vec2 chroma_sampling;\n" "uniform float out_bitdepth_factor;\n", - { glsl_func_swizzle, glsl_func_color_matrix, NULL, }, + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_chroma_sample, glsl_func_write_planar_yuv, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + +static const gchar templ_RGB_to_PLANAR_YUVA_BODY[] = + "vec4 texel = swizzle(texture2D(tex, texcoord), input_swizzle);\n" + "vec2 unnormalization;\n" + "if (texcoord.x == v_texcoord.x) {\n" + " unnormalization = vec2(width, height);\n" + "} else {\n" + " unnormalization = vec2 (1.0);\n" + "}\n" + "vec4 uv_texel = chroma_sample(tex, texcoord, unnormalization);\n" + "vec4 yuva;\n" + "yuva.x = color_matrix_apply(texel, to_YUV_matrix).x;\n" + "yuva.yz = color_matrix_apply(uv_texel, to_YUV_matrix).yz;\n" + "yuva.a = texel.a;\n" + "yuva = swizzle(yuva, output_swizzle) * out_bitdepth_factor;\n" + "write_planar_yuva(yuva);\n"; + +static const struct shader_templ templ_RGB_to_PLANAR_YUVA = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n" + "uniform vec2 chroma_sampling;\n" "uniform float out_bitdepth_factor;\n", + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_chroma_sample, glsl_func_write_planar_yuva, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + +static const gchar templ_PLANAR_YUV_to_PLANAR_YUV_BODY[] = + "vec4 yuva;\n" + "yuva.x = swizzle(texture2D(Ytex, texcoord * tex_scale0), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.y = swizzle(texture2D(Utex, texcoord * tex_scale1 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.z = swizzle(texture2D(Vtex, texcoord * tex_scale2 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.a = 1.0;\n" + "yuva = swizzle(yuva, output_swizzle) * out_bitdepth_factor;\n" + "write_planar_yuv(yuva);\n"; + +static const struct shader_templ templ_PLANAR_YUV_to_PLANAR_YUV = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex;\n" + "uniform vec2 chroma_sampling;\n" "uniform float in_bitdepth_factor;\n" "uniform float out_bitdepth_factor;\n", + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_write_planar_yuv, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + +static const gchar templ_PLANAR_YUV_to_PLANAR_YUVA_BODY[] = + "vec4 yuva;\n" + "yuva.x = swizzle(texture2D(Ytex, texcoord * tex_scale0), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.y = swizzle(texture2D(Utex, texcoord * tex_scale1 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.z = swizzle(texture2D(Vtex, texcoord * tex_scale2 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.a = 1.0;\n" + "yuva = swizzle(yuva, output_swizzle) * out_bitdepth_factor;\n" + "write_planar_yuva(yuva);\n"; + +static const struct shader_templ templ_PLANAR_YUV_to_PLANAR_YUVA = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex;\n" + "uniform vec2 chroma_sampling;\n" "uniform float in_bitdepth_factor;\n" "uniform float out_bitdepth_factor;\n", + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_write_planar_yuva, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + +static const gchar templ_PLANAR_YUVA_to_PLANAR_YUVA_BODY[] = + "vec4 yuva;\n" + "yuva.x = swizzle(texture2D(Ytex, texcoord * tex_scale0), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.y = swizzle(texture2D(Utex, texcoord * tex_scale1 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.z = swizzle(texture2D(Vtex, texcoord * tex_scale2 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.a = swizzle(texture2D(Atex, texcoord * tex_scale3), input_swizzle).r;\n" + "yuva = swizzle(yuva, output_swizzle) * out_bitdepth_factor;\n" + "write_planar_yuva(yuva);\n"; + +static const struct shader_templ templ_PLANAR_YUVA_to_PLANAR_YUVA = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex, Atex;\n" + "uniform vec2 chroma_sampling;\n" "uniform float in_bitdepth_factor;\n" "uniform float out_bitdepth_factor;\n", + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_write_planar_yuv, glsl_func_write_planar_yuva, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + +static const gchar templ_PLANAR_YUVA_to_PLANAR_YUV_BODY[] = + "vec4 yuva;\n" + "yuva.x = swizzle(texture2D(Ytex, texcoord * tex_scale0), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.y = swizzle(texture2D(Utex, texcoord * tex_scale1 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.z = swizzle(texture2D(Vtex, texcoord * tex_scale2 * chroma_sampling), input_swizzle).r * in_bitdepth_factor;\n" + "yuva.a = swizzle(texture2D(Atex, texcoord * tex_scale3), input_swizzle).r;\n" + "yuva = swizzle(yuva, output_swizzle) * out_bitdepth_factor;\n" + "write_planar_yuv(yuva);\n"; + +static const struct shader_templ templ_PLANAR_YUVA_to_PLANAR_YUV = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex, Atex;\n" + "uniform vec2 chroma_sampling;\n" "uniform float in_bitdepth_factor;\n" "uniform float out_bitdepth_factor;\n", + { glsl_func_swizzle, glsl_func_color_matrix, glsl_func_chroma_sample, glsl_func_write_planar_yuv, NULL, }, GST_GL_TEXTURE_TARGET_2D }; @@ -986,6 +1113,29 @@ _gst_gl_color_convert_can_passthrough_info (const GstVideoInfo * in, return TRUE; } +static gboolean +conversion_formats_are_supported (const GstVideoFormatInfo * in_finfo, + const GstVideoFormatInfo * out_finfo) +{ + gboolean input_yuv_planar = GST_VIDEO_FORMAT_INFO_IS_YUV (in_finfo) + && GST_VIDEO_FORMAT_INFO_N_PLANES (in_finfo) == + GST_VIDEO_FORMAT_INFO_N_COMPONENTS (in_finfo); + gboolean output_yuv_planar = GST_VIDEO_FORMAT_INFO_IS_YUV (out_finfo) + && GST_VIDEO_FORMAT_INFO_N_PLANES (out_finfo) == + GST_VIDEO_FORMAT_INFO_N_COMPONENTS (out_finfo); + + /* GRAY/YUV -> GRAY/YUV is not supported for non-passthrough */ + if (GST_VIDEO_FORMAT_INFO_IS_RGB (in_finfo)) + return TRUE; + if (GST_VIDEO_FORMAT_INFO_IS_RGB (out_finfo)) + return TRUE; + + if (input_yuv_planar && output_yuv_planar) + return TRUE; + + return FALSE; +} + static gboolean _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert, GstCaps * in_caps, GstCaps * out_caps) @@ -1066,18 +1216,9 @@ _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert, && to_target != GST_GL_TEXTURE_TARGET_RECTANGLE) return FALSE; - { - guint yuv_gray_flags, in_flags, out_flags; - - in_flags = GST_VIDEO_FORMAT_INFO_FLAGS (in_info.finfo); - out_flags = GST_VIDEO_FORMAT_INFO_FLAGS (out_info.finfo); - yuv_gray_flags = GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY; - - /* GRAY/YUV -> GRAY/YUV is not supported for non-passthrough */ - if (!passthrough && (in_flags & yuv_gray_flags) != 0 - && (out_flags & yuv_gray_flags) != 0) - return FALSE; - } + if (!passthrough && + !conversion_formats_are_supported (in_info.finfo, out_info.finfo)) + return FALSE; gst_gl_color_convert_reset (convert); convert->in_info = in_info; @@ -1350,6 +1491,8 @@ gst_gl_color_convert_caps_transform_format_info (GstGLContext * context, GValue supported_formats = G_VALUE_INIT; GValue rgb_formats = G_VALUE_INIT; GValue supported_rgb_formats = G_VALUE_INIT; + GValue planar_yuv_formats = G_VALUE_INIT; + GValue supported_planar_yuv_formats = G_VALUE_INIT; /* There are effectively two modes here with the RGB/YUV transition: * 1. There is a RGB-like format as input and we can transform to YUV or, @@ -1363,9 +1506,17 @@ gst_gl_color_convert_caps_transform_format_info (GstGLContext * context, "xRGB", "BGRx", "xBGR", "RGB", "BGR", "ARGB64", "BGR10A2_LE", "RGB10A2_LE", "RGBA64_LE", "RGBA64_BE", "RBGA", "GBRA", "GBR", "RGBP", "BGRP", "RGB16", "BGR16", NULL); + _init_value_string_list (&planar_yuv_formats, "Y444", "I420", "Y42B", "Y41B", + "A420", "A444", "A422", "A420_10LE", "A422_10LE", "A444_10LE", + "A444_12LE", "A422_12LE", "A420_12LE", "A444_16LE", "A422_16LE", + "A420_16LE", "I420_12LE", "I420_10LE", "A420_10BE", "A422_10BE", + "A444_10BE", "A444_12BE", "A422_12BE", "A420_12BE", "A444_16BE", + "A422_16BE", "A420_16BE", "I420_12BE", "I420_10BE", NULL); _init_supported_formats (context, output, &supported_formats); gst_value_intersect (&supported_rgb_formats, &rgb_formats, &supported_formats); + gst_value_intersect (&supported_planar_yuv_formats, &planar_yuv_formats, + &supported_formats); res = gst_caps_new_empty (); @@ -1389,6 +1540,7 @@ gst_gl_color_convert_caps_transform_format_info (GstGLContext * context, gst_caps_features_contains (f, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { if (GST_VALUE_HOLDS_LIST (format)) { gboolean have_rgb_formats = FALSE; + gboolean have_planar_yuv_formats = FALSE; GValue passthrough_formats = G_VALUE_INIT; gint j, len; @@ -1410,36 +1562,63 @@ gst_gl_color_convert_caps_transform_format_info (GstGLContext * context, GST_VIDEO_FORMAT_FLAG_RGB) { have_rgb_formats = TRUE; break; + } else if (GST_VIDEO_FORMAT_INFO_IS_YUV (t_info) + && GST_VIDEO_FORMAT_INFO_N_PLANES (t_info) == + GST_VIDEO_FORMAT_INFO_N_COMPONENTS (t_info)) { + have_planar_yuv_formats = TRUE; } } } if (have_rgb_formats) { gst_structure_set_value (st, "format", &supported_formats); + gst_structure_remove_fields (st, "colorimetry", "chroma-site", + "texture-target", NULL); } else { - /* add passthrough structure, then the rgb conversion structure */ + /* add passthrough structure */ gst_structure_set_value (st, "format", &passthrough_formats); + /* then optional planar yuv conversion structure */ + if (have_planar_yuv_formats) { + gst_caps_append_structure_full (res, gst_structure_copy (st), + gst_caps_features_copy (f)); + gst_structure_set_value (st, "format", + &supported_planar_yuv_formats); + gst_structure_remove_fields (st, "texture-target", NULL); + } + /* then the rgb conversion structure */ gst_caps_append_structure_full (res, gst_structure_copy (st), gst_caps_features_copy (f)); gst_structure_set_value (st, "format", &supported_rgb_formats); + gst_structure_remove_fields (st, "colorimetry", "chroma-site", + "texture-target", NULL); } g_value_unset (&passthrough_formats); } else if (G_VALUE_HOLDS_STRING (format)) { const gchar *format_str = g_value_get_string (format); GstVideoFormat v_format = gst_video_format_from_string (format_str); const GstVideoFormatInfo *t_info = gst_video_format_get_info (v_format); - if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) & (GST_VIDEO_FORMAT_FLAG_YUV | - GST_VIDEO_FORMAT_FLAG_GRAY)) { + if (GST_VIDEO_FORMAT_INFO_IS_RGB (t_info)) { + gst_structure_set_value (st, "format", &supported_formats); + gst_structure_remove_fields (st, "colorimetry", "chroma-site", + "texture-target", NULL); + } else { /* add passthrough structure, then the rgb conversion structure */ gst_structure_set_value (st, "format", format); + if (GST_VIDEO_FORMAT_INFO_IS_YUV (t_info) + && GST_VIDEO_FORMAT_INFO_N_PLANES (t_info) == + GST_VIDEO_FORMAT_INFO_N_COMPONENTS (t_info)) { + gst_caps_append_structure_full (res, gst_structure_copy (st), + gst_caps_features_copy (f)); + gst_structure_set_value (st, "format", + &supported_planar_yuv_formats); + gst_structure_remove_fields (st, "texture-target", NULL); + } gst_caps_append_structure_full (res, gst_structure_copy (st), gst_caps_features_copy (f)); gst_structure_set_value (st, "format", &supported_rgb_formats); - } else { /* RGB */ - gst_structure_set_value (st, "format", &supported_formats); + gst_structure_remove_fields (st, "colorimetry", "chroma-site", + "texture-target", NULL); } } - gst_structure_remove_fields (st, "colorimetry", "chroma-site", - "texture-target", NULL); } gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); @@ -1448,6 +1627,8 @@ gst_gl_color_convert_caps_transform_format_info (GstGLContext * context, g_value_unset (&supported_formats); g_value_unset (&rgb_formats); g_value_unset (&supported_rgb_formats); + g_value_unset (&planar_yuv_formats); + g_value_unset (&supported_planar_yuv_formats); return res; } @@ -1560,10 +1741,7 @@ score_format_target (const GstVideoFormatInfo * in_info, guint targets_mask, t_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX; t_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK; - /* GRAY/YUV -> GRAY/YUV is not supported */ - if ((in_flags & (GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY)) != 0 - && (t_flags & (GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY)) != - 0) + if (!conversion_formats_are_supported (in_info, t_info)) return; if ((t_flags & PALETTE_MASK) != (in_flags & PALETTE_MASK)) { @@ -1895,6 +2073,14 @@ _get_n_textures (GstVideoFormat v_format) return finfo->n_planes; } +static gboolean +format_is_planar (GstVideoFormat v_format) +{ + const GstVideoFormatInfo *finfo = gst_video_format_get_info (v_format); + + return finfo->n_planes == finfo->n_components; +} + static void _PLANAR_RGB_to_PLANAR_RGB (GstGLColorConvert * convert) { @@ -2074,6 +2260,21 @@ _YUV_to_RGB (GstGLColorConvert * convert) info->templ = &templ_REORDER; info->frag_body = g_strdup (templ_REORDER_BODY); info->shader_tex_names[0] = "tex"; + } else if (in_finfo->n_planes >= 3 && format_is_planar (in_format)) { + info->shader_tex_names[0] = "Ytex"; + info->shader_tex_names[1] = "Utex"; + info->shader_tex_names[2] = "Vtex"; + info->in_bitdepth_factor = + (float) ((1 << GST_ROUND_UP_8 (in_finfo->bits)) - + 1) / (float) ((1 << in_finfo->bits) - 1); + if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (in_finfo)) { + info->templ = &templ_A420_to_RGB; + info->frag_body = g_strdup_printf (templ_PLANAR_YUVA_to_RGB_BODY); + info->shader_tex_names[3] = "Atex"; + } else { + info->templ = &templ_PLANAR_YUV_to_RGB; + info->frag_body = g_strdup (templ_PLANAR_YUV_to_RGB_BODY); + } } else { switch (in_format) { case GST_VIDEO_FORMAT_AYUV: @@ -2085,58 +2286,6 @@ _YUV_to_RGB (GstGLColorConvert * convert) info->frag_body = g_strdup (templ_AYUV_to_RGB_BODY); info->shader_tex_names[0] = "tex"; break; - case GST_VIDEO_FORMAT_I420: - case GST_VIDEO_FORMAT_I420_10LE: - case GST_VIDEO_FORMAT_I420_10BE: - case GST_VIDEO_FORMAT_I420_12LE: - case GST_VIDEO_FORMAT_I420_12BE: - case GST_VIDEO_FORMAT_Y444: - case GST_VIDEO_FORMAT_Y42B: - case GST_VIDEO_FORMAT_Y41B: - case GST_VIDEO_FORMAT_YV12: - info->templ = &templ_PLANAR_YUV_to_RGB; - info->frag_body = - g_strdup_printf (templ_PLANAR_YUV_to_RGB_BODY, "yuva.a = 1.0;\n"); - info->shader_tex_names[0] = "Ytex"; - info->shader_tex_names[1] = "Utex"; - info->shader_tex_names[2] = "Vtex"; - info->in_bitdepth_factor = - (float) ((1 << GST_ROUND_UP_8 (in_finfo->bits)) - - 1) / (float) ((1 << in_finfo->bits) - 1); - break; - case GST_VIDEO_FORMAT_A420: - case GST_VIDEO_FORMAT_A420_10LE: - case GST_VIDEO_FORMAT_A420_10BE: - case GST_VIDEO_FORMAT_A420_12LE: - case GST_VIDEO_FORMAT_A420_12BE: - case GST_VIDEO_FORMAT_A420_16LE: - case GST_VIDEO_FORMAT_A420_16BE: - case GST_VIDEO_FORMAT_A422: - case GST_VIDEO_FORMAT_A422_10LE: - case GST_VIDEO_FORMAT_A422_10BE: - case GST_VIDEO_FORMAT_A422_12LE: - case GST_VIDEO_FORMAT_A422_12BE: - case GST_VIDEO_FORMAT_A422_16LE: - case GST_VIDEO_FORMAT_A422_16BE: - case GST_VIDEO_FORMAT_A444: - case GST_VIDEO_FORMAT_A444_10LE: - case GST_VIDEO_FORMAT_A444_10BE: - case GST_VIDEO_FORMAT_A444_12LE: - case GST_VIDEO_FORMAT_A444_12BE: - case GST_VIDEO_FORMAT_A444_16LE: - case GST_VIDEO_FORMAT_A444_16BE: - info->templ = &templ_A420_to_RGB; - info->frag_body = - g_strdup_printf (templ_PLANAR_YUV_to_RGB_BODY, - "yuva.a = texture2D(Atex, texcoord * tex_scale3).r;\n"); - info->shader_tex_names[0] = "Ytex"; - info->shader_tex_names[1] = "Utex"; - info->shader_tex_names[2] = "Vtex"; - info->shader_tex_names[3] = "Atex"; - info->in_bitdepth_factor = - (float) ((1 << GST_ROUND_UP_8 (in_finfo->bits)) - - 1) / (float) ((1 << in_finfo->bits) - 1); - break; case GST_VIDEO_FORMAT_YUY2: { char uv_val = @@ -2255,94 +2404,74 @@ _RGB_to_YUV (GstGLColorConvert * convert) calculate_reorder_indexes (in_format, out_format, info->input_swizzle, info->output_swizzle); - switch (out_format) { - case GST_VIDEO_FORMAT_AYUV: - alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; - info->templ = &templ_RGB_to_AYUV; - info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, alpha); - break; - case GST_VIDEO_FORMAT_VUYA: - alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; - info->templ = &templ_RGB_to_AYUV; - info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, alpha); - break; - case GST_VIDEO_FORMAT_Y410: - case GST_VIDEO_FORMAT_Y412_LE: - case GST_VIDEO_FORMAT_Y412_BE: - alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; - info->templ = &templ_RGB_to_AYUV; - info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, alpha); - break; - case GST_VIDEO_FORMAT_I420: - case GST_VIDEO_FORMAT_I420_10LE: - case GST_VIDEO_FORMAT_I420_10BE: - case GST_VIDEO_FORMAT_I420_12LE: - case GST_VIDEO_FORMAT_I420_12BE: - case GST_VIDEO_FORMAT_YV12: - case GST_VIDEO_FORMAT_Y444: - case GST_VIDEO_FORMAT_Y42B: - case GST_VIDEO_FORMAT_Y41B: - case GST_VIDEO_FORMAT_A420: - case GST_VIDEO_FORMAT_A420_10LE: - case GST_VIDEO_FORMAT_A420_10BE: - case GST_VIDEO_FORMAT_A420_12LE: - case GST_VIDEO_FORMAT_A420_12BE: - case GST_VIDEO_FORMAT_A420_16LE: - case GST_VIDEO_FORMAT_A420_16BE: - case GST_VIDEO_FORMAT_A422: - case GST_VIDEO_FORMAT_A422_10LE: - case GST_VIDEO_FORMAT_A422_10BE: - case GST_VIDEO_FORMAT_A422_12LE: - case GST_VIDEO_FORMAT_A422_12BE: - case GST_VIDEO_FORMAT_A422_16LE: - case GST_VIDEO_FORMAT_A422_16BE: - case GST_VIDEO_FORMAT_A444: - case GST_VIDEO_FORMAT_A444_10LE: - case GST_VIDEO_FORMAT_A444_10BE: - case GST_VIDEO_FORMAT_A444_12LE: - case GST_VIDEO_FORMAT_A444_12BE: - case GST_VIDEO_FORMAT_A444_16LE: - case GST_VIDEO_FORMAT_A444_16BE: + if (out_finfo->n_planes >= 3 && format_is_planar (out_format)) { + if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (out_finfo)) { + info->templ = &templ_RGB_to_PLANAR_YUVA; + info->frag_body = g_strdup (templ_RGB_to_PLANAR_YUVA_BODY); + } else { info->templ = &templ_RGB_to_PLANAR_YUV; - if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (out_finfo)) { - alpha = "gl_FragData[3] = vec4(yuva.a, 0.0, 0.0, 1.0);\n"; - } else { - alpha = ""; - } - info->frag_body = g_strdup_printf (templ_RGB_to_PLANAR_YUV_BODY, alpha); - info->chroma_sampling[0] = (float) (1 << out_finfo->w_sub[1]); - info->chroma_sampling[1] = (float) (1 << out_finfo->h_sub[1]); - info->out_bitdepth_factor = - (float) ((1 << out_finfo->bits) - - 1) / (float) ((1 << GST_ROUND_UP_8 (out_finfo->bits)) - 1); - break; - case GST_VIDEO_FORMAT_YUY2: - case GST_VIDEO_FORMAT_Y210: - case GST_VIDEO_FORMAT_Y212_LE: - case GST_VIDEO_FORMAT_Y212_BE: - info->templ = &templ_RGB_to_YUY2_UYVY; - info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY, - 'x', 'y', 'x', 'z'); - break; - case GST_VIDEO_FORMAT_UYVY: - info->templ = &templ_RGB_to_YUY2_UYVY, - info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY, - 'y', 'x', 'z', 'x'); - break; - case GST_VIDEO_FORMAT_NV12: - case GST_VIDEO_FORMAT_NV16: - info->templ = &templ_RGB_to_SEMI_PLANAR_YUV; - info->frag_body = g_strdup_printf (templ_RGB_to_SEMI_PLANAR_YUV_BODY, ""); - if (out_format == GST_VIDEO_FORMAT_NV16) { - info->chroma_sampling[0] = 2.0f; - info->chroma_sampling[1] = 1.0f; - } else { + info->frag_body = g_strdup (templ_RGB_to_PLANAR_YUV_BODY); + } + info->chroma_sampling[0] = (float) (1 << out_finfo->w_sub[1]); + info->chroma_sampling[1] = (float) (1 << out_finfo->h_sub[1]); + info->out_bitdepth_factor = + (float) ((1 << out_finfo->bits) - + 1) / (float) ((1 << GST_ROUND_UP_8 (out_finfo->bits)) - 1); + } else { + switch (out_format) { + case GST_VIDEO_FORMAT_AYUV: + alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; + info->templ = &templ_RGB_to_AYUV; + info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, alpha); + break; + case GST_VIDEO_FORMAT_VUYA: + alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; + info->templ = &templ_RGB_to_AYUV; + info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, alpha); + break; + case GST_VIDEO_FORMAT_Y410: + case GST_VIDEO_FORMAT_Y412_LE: + case GST_VIDEO_FORMAT_Y412_BE: + alpha = _is_RGBx (in_format) ? "1.0" : "texel.a"; + info->templ = &templ_RGB_to_AYUV; + info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, alpha); + break; + case GST_VIDEO_FORMAT_YUY2: + case GST_VIDEO_FORMAT_Y210: + case GST_VIDEO_FORMAT_Y212_LE: + case GST_VIDEO_FORMAT_Y212_BE: + info->templ = &templ_RGB_to_YUY2_UYVY; + info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY, + 'x', 'y', 'x', 'z'); + break; + case GST_VIDEO_FORMAT_UYVY: + info->templ = &templ_RGB_to_YUY2_UYVY; + info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY, + 'y', 'x', 'z', 'x'); + break; + case GST_VIDEO_FORMAT_NV12: + case GST_VIDEO_FORMAT_NV16: + info->templ = &templ_RGB_to_SEMI_PLANAR_YUV; + info->frag_body = + g_strdup_printf (templ_RGB_to_SEMI_PLANAR_YUV_BODY, ""); + if (out_format == GST_VIDEO_FORMAT_NV16) { + info->chroma_sampling[0] = 2.0f; + info->chroma_sampling[1] = 1.0f; + } else { + info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f; + } + break; + case GST_VIDEO_FORMAT_AV12: + info->templ = &templ_RGB_to_SEMI_PLANAR_YUV; + info->frag_body = g_strdup_printf (templ_RGB_to_SEMI_PLANAR_YUV_BODY, + "gl_FragData[2] = vec4(yuva.a, 0.0, 0.0, 1.0);\n"); info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f; break; case GST_VIDEO_FORMAT_NV21: case GST_VIDEO_FORMAT_NV61: info->templ = &templ_RGB_to_SEMI_PLANAR_YUV; - info->frag_body = g_strdup_printf (templ_RGB_to_SEMI_PLANAR_YUV_BODY, ""); + info->frag_body = + g_strdup_printf (templ_RGB_to_SEMI_PLANAR_YUV_BODY, ""); if (out_format == GST_VIDEO_FORMAT_NV61) { info->chroma_sampling[0] = 2.0f; info->chroma_sampling[1] = 1.0f; @@ -2350,21 +2479,86 @@ _RGB_to_YUV (GstGLColorConvert * convert) info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f; } break; - } - break; - case GST_VIDEO_FORMAT_AV12: - info->templ = &templ_RGB_to_SEMI_PLANAR_YUV, - info->frag_body = g_strdup_printf (templ_RGB_to_SEMI_PLANAR_YUV_BODY, - "gl_FragData[2] = vec4(yuva.a, 0.0, 0.0, 1.0);\n"); - info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f; - break; - default: - break; + default: + break; + } } convert_to_YUV (info, &convert->out_info); } +static void +_YUV_to_YUV (GstGLColorConvert * convert) +{ + struct ConvertInfo *info = &convert->priv->convert_info; + GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info); + const GstVideoFormatInfo *in_finfo = gst_video_format_get_info (in_format); + GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info); + const GstVideoFormatInfo *out_finfo = gst_video_format_get_info (out_format); + gboolean apple_ycbcr = gst_gl_context_check_feature (convert->context, + "GL_APPLE_ycbcr_422"); + gboolean in_tex_rectangular = FALSE; + gboolean input_planar = format_is_planar (in_format); + gboolean output_planar = format_is_planar (out_format); + +#if GST_GL_HAVE_OPENGL + GstMemory *memory = gst_buffer_peek_memory (convert->inbuf, 0); + if (gst_is_gl_memory (memory) && (USING_OPENGL (convert->context) + || USING_OPENGL3 (convert->context))) { + in_tex_rectangular = + convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE; + } +#endif + + calculate_reorder_indexes (in_format, out_format, info->input_swizzle, + info->output_swizzle); + + if (in_tex_rectangular && apple_ycbcr + && gst_buffer_n_memory (convert->inbuf) == 1) { + /* FIXME: We should probably also check if tex_target actually is using + * the Apple YCbCr422 extension. It could also be a normal UYVY texture + * with RB or Lum/Alpha + */ + /* The mangling will change this to the correct texture2DRect, sampler2DRect + * for us */ + info->templ = &templ_REORDER; + info->frag_body = g_strdup (templ_REORDER_BODY); + info->shader_tex_names[0] = "tex"; + } else if (input_planar && output_planar) { + info->chroma_sampling[0] = (float) (1 << out_finfo->w_sub[1]); + info->chroma_sampling[1] = (float) (1 << out_finfo->h_sub[1]); + info->shader_tex_names[0] = "Ytex"; + info->shader_tex_names[1] = "Utex"; + info->shader_tex_names[2] = "Vtex"; + info->in_bitdepth_factor = + (float) ((1 << GST_ROUND_UP_8 (in_finfo->bits)) - + 1) / (float) ((1 << in_finfo->bits) - 1); + info->out_bitdepth_factor = + (float) ((1 << out_finfo->bits) - + 1) / (float) ((1 << GST_ROUND_UP_8 (out_finfo->bits)) - 1); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (in_finfo)) { + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (out_finfo)) { + info->templ = &templ_PLANAR_YUV_to_PLANAR_YUV; + info->frag_body = g_strdup (templ_PLANAR_YUV_to_PLANAR_YUV_BODY); + } else { + info->templ = &templ_PLANAR_YUV_to_PLANAR_YUVA; + info->frag_body = g_strdup (templ_PLANAR_YUV_to_PLANAR_YUVA_BODY); + } + } else { + info->shader_tex_names[3] = "Atex"; + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (out_finfo)) { + info->templ = &templ_PLANAR_YUVA_to_PLANAR_YUV; + info->frag_body = g_strdup (templ_PLANAR_YUVA_to_PLANAR_YUV_BODY); + } else { + info->templ = &templ_PLANAR_YUVA_to_PLANAR_YUVA; + info->frag_body = g_strdup (templ_PLANAR_YUVA_to_PLANAR_YUVA_BODY); + } + } + } else { + g_assert_not_reached (); + } +} + static void _RGB_to_GRAY (GstGLColorConvert * convert) { @@ -2698,6 +2892,12 @@ _init_convert (GstGLColorConvert * convert) } } + if (GST_VIDEO_INFO_IS_YUV (&convert->in_info)) { + if (GST_VIDEO_INFO_IS_YUV (&convert->out_info)) { + _YUV_to_YUV (convert); + } + } + if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) { if (GST_VIDEO_INFO_IS_GRAY (&convert->out_info)) { _RGB_to_GRAY (convert);