[Cogl] [PATCH] cogl-gst-video-sink: Premultiply the colors coming from the video
Robert Bragg
robert at sixbynine.org
Thu Jan 30 16:34:38 PST 2014
Ah, good catch!
I think it could be good to extend the gtk-doc comment about sampling
the video via cogl_gst_sample_video0() to explicitly note that this
function will always return you a premultiplied alpha value.
Apart from that this looks to land to me:
Reviewed-by: Robert Bragg <robert at linux.intel.com>
Thanks,
Robert
On Wed, Jan 22, 2014 at 3:22 PM, Neil Roberts <neil at linux.intel.com> wrote:
> Commit 99a53c82e9ab0a1e5 removed the internal format argument when
> uploading a video frame to a texture so that the format will just be
> determined automatically from the image format. However this also
> leaves the premultiplied state at the default and the default is TRUE.
> That means that when we upload RGBA data Cogl will do a premultiplied
> conversion on the CPU. We probably don't want to be putting a CPU
> conversion in the way of video frames so this patch changes it to set
> the premultiplied state to FALSE on the textures and then do the
> premultiplied conversion in the shader.
>
> This is particularly important for AYUV which uses the alpha channel
> for the V component so doing a premultiplied conversion on the CPU
> just creates garbage and messes up the image.
>
> The RGB and RGBA renderers have each been split into two; one that
> uses GLSL and one that uses a regular pipeline. The RGBA pipeline
> without GLSL is then changed to use 2 layers so we that we can do the
> premultiplied conversion in the second layer with a special layer
> combine string.
> ---
> cogl-gst/cogl-gst-video-sink.c | 159 +++++++++++++++++++++++++++++++----------
> 1 file changed, 122 insertions(+), 37 deletions(-)
>
> diff --git a/cogl-gst/cogl-gst-video-sink.c b/cogl-gst/cogl-gst-video-sink.c
> index f9887aa..243d9ae 100644
> --- a/cogl-gst/cogl-gst-video-sink.c
> +++ b/cogl-gst/cogl-gst-video-sink.c
> @@ -417,40 +417,6 @@ clear_frame_textures (CoglGstVideoSink *sink)
> priv->frame_dirty = TRUE;
> }
>
> -static void
> -cogl_gst_rgb_setup_pipeline (CoglGstVideoSink *sink,
> - CoglPipeline *pipeline)
> -{
> - CoglGstVideoSinkPrivate *priv = sink->priv;
> -
> - if (cogl_has_feature (priv->ctx, COGL_FEATURE_ID_GLSL))
> - {
> - static SnippetCache snippet_cache;
> - SnippetCacheEntry *entry = get_cache_entry (sink, &snippet_cache);
> -
> - if (entry == NULL)
> - {
> - char *source;
> -
> - source =
> - g_strdup_printf ("vec4\n"
> - "cogl_gst_sample_video%i (vec2 UV)\n"
> - "{\n"
> - " return texture2D (cogl_sampler%i, UV);\n"
> - "}\n",
> - priv->custom_start,
> - priv->custom_start);
> -
> - entry = add_cache_entry (sink, &snippet_cache, source);
> - g_free (source);
> - }
> -
> - setup_pipeline_from_cache_entry (sink, pipeline, entry, 1);
> - }
> - else
> - setup_pipeline_from_cache_entry (sink, pipeline, NULL, 1);
> -}
> -
> static inline CoglBool
> is_pot (unsigned int number)
> {
> @@ -504,9 +470,46 @@ video_texture_new_from_data (CoglContext *ctx,
>
> cogl_object_unref (bitmap);
>
> + cogl_texture_set_premultiplied (tex, FALSE);
> +
> return tex;
> }
>
> +static void
> +cogl_gst_rgb24_glsl_setup_pipeline (CoglGstVideoSink *sink,
> + CoglPipeline *pipeline)
> +{
> + CoglGstVideoSinkPrivate *priv = sink->priv;
> + static SnippetCache snippet_cache;
> + SnippetCacheEntry *entry = get_cache_entry (sink, &snippet_cache);
> +
> + if (entry == NULL)
> + {
> + char *source;
> +
> + source =
> + g_strdup_printf ("vec4\n"
> + "cogl_gst_sample_video%i (vec2 UV)\n"
> + "{\n"
> + " return texture2D (cogl_sampler%i, UV);\n"
> + "}\n",
> + priv->custom_start,
> + priv->custom_start);
> +
> + entry = add_cache_entry (sink, &snippet_cache, source);
> + g_free (source);
> + }
> +
> + setup_pipeline_from_cache_entry (sink, pipeline, entry, 1);
> +}
> +
> +static void
> +cogl_gst_rgb24_setup_pipeline (CoglGstVideoSink *sink,
> + CoglPipeline *pipeline)
> +{
> + setup_pipeline_from_cache_entry (sink, pipeline, NULL, 1);
> +}
> +
> static CoglBool
> cogl_gst_rgb24_upload (CoglGstVideoSink *sink,
> GstBuffer *buffer)
> @@ -542,6 +545,17 @@ map_fail:
> }
> }
>
> +static CoglGstRenderer rgb24_glsl_renderer =
> +{
> + "RGB 24",
> + COGL_GST_RGB24,
> + COGL_GST_RENDERER_NEEDS_GLSL,
> + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGB, BGR }")),
> + 1, /* n_layers */
> + cogl_gst_rgb24_glsl_setup_pipeline,
> + cogl_gst_rgb24_upload,
> +};
> +
> static CoglGstRenderer rgb24_renderer =
> {
> "RGB 24",
> @@ -549,10 +563,61 @@ static CoglGstRenderer rgb24_renderer =
> 0,
> GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGB, BGR }")),
> 1, /* n_layers */
> - cogl_gst_rgb_setup_pipeline,
> + cogl_gst_rgb24_setup_pipeline,
> cogl_gst_rgb24_upload,
> };
>
> +static void
> +cogl_gst_rgb32_glsl_setup_pipeline (CoglGstVideoSink *sink,
> + CoglPipeline *pipeline)
> +{
> + CoglGstVideoSinkPrivate *priv = sink->priv;
> + static SnippetCache snippet_cache;
> + SnippetCacheEntry *entry = get_cache_entry (sink, &snippet_cache);
> +
> + if (entry == NULL)
> + {
> + char *source;
> +
> + source =
> + g_strdup_printf ("vec4\n"
> + "cogl_gst_sample_video%i (vec2 UV)\n"
> + "{\n"
> + " vec4 color = texture2D (cogl_sampler%i, UV);\n"
> + /* Premultiply the color */
> + " color.rgb *= color.a;\n"
> + " return color;\n"
> + "}\n",
> + priv->custom_start,
> + priv->custom_start);
> +
> + entry = add_cache_entry (sink, &snippet_cache, source);
> + g_free (source);
> + }
> +
> + setup_pipeline_from_cache_entry (sink, pipeline, entry, 1);
> +}
> +
> +static void
> +cogl_gst_rgb32_setup_pipeline (CoglGstVideoSink *sink,
> + CoglPipeline *pipeline)
> +{
> + CoglGstVideoSinkPrivate *priv = sink->priv;
> + char *layer_combine;
> +
> + setup_pipeline_from_cache_entry (sink, pipeline, NULL, 1);
> +
> + /* Premultiply the texture using the a special layer combine */
> + layer_combine = g_strdup_printf ("RGB=MODULATE(PREVIOUS, TEXTURE_%i[A])\n"
> + "A=REPLACE(PREVIOUS[A])",
> + priv->custom_start);
> + cogl_pipeline_set_layer_combine (pipeline,
> + priv->custom_start + 1,
> + layer_combine,
> + NULL);
> + g_free(layer_combine);
> +}
> +
> static CoglBool
> cogl_gst_rgb32_upload (CoglGstVideoSink *sink,
> GstBuffer *buffer)
> @@ -588,14 +653,25 @@ map_fail:
> }
> }
>
> +static CoglGstRenderer rgb32_glsl_renderer =
> +{
> + "RGB 32",
> + COGL_GST_RGB32,
> + COGL_GST_RENDERER_NEEDS_GLSL,
> + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGBA, BGRA }")),
> + 1, /* n_layers */
> + cogl_gst_rgb32_glsl_setup_pipeline,
> + cogl_gst_rgb32_upload,
> +};
> +
> static CoglGstRenderer rgb32_renderer =
> {
> "RGB 32",
> COGL_GST_RGB32,
> 0,
> GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGBA, BGRA }")),
> - 1, /* n_layers */
> - cogl_gst_rgb_setup_pipeline,
> + 2, /* n_layers */
> + cogl_gst_rgb32_setup_pipeline,
> cogl_gst_rgb32_upload,
> };
>
> @@ -778,6 +854,8 @@ cogl_gst_ayuv_glsl_setup_pipeline (CoglGstVideoSink *sink,
> " color.r = y + 1.59765625 * v;\n"
> " color.g = y - 0.390625 * u - 0.8125 * v;\n"
> " color.b = y + 2.015625 * u;\n"
> + /* Premultiply the color */
> + " color.rgb *= color.a;\n"
> " return color;\n"
> "}\n", priv->custom_start,
> priv->custom_start);
> @@ -838,8 +916,13 @@ cogl_gst_build_renderers_list (CoglContext *ctx)
> int i;
> static CoglGstRenderer *const renderers[] =
> {
> + /* These are in increasing order of priority so that the
> + * priv->renderers will be in decreasing order. That way the GLSL
> + * renderers will be preferred if they are available */
> &rgb24_renderer,
> &rgb32_renderer,
> + &rgb24_glsl_renderer,
> + &rgb32_glsl_renderer,
> &yv12_glsl_renderer,
> &i420_glsl_renderer,
> &ayuv_glsl_renderer,
> @@ -916,6 +999,8 @@ cogl_gst_find_renderer_by_format (CoglGstVideoSink *sink,
> CoglGstRenderer *renderer = NULL;
> GSList *element;
>
> + /* The renderers list is in decreasing order of priority so we'll
> + * pick the first one that matches */
> for (element = priv->renderers; element; element = g_slist_next (element))
> {
> CoglGstRenderer *candidate = (CoglGstRenderer *) element->data;
> --
> 1.8.4.2
>
> _______________________________________________
> Cogl mailing list
> Cogl at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/cogl
More information about the Cogl
mailing list