[Cogl] [PATCH] cogl-gst-video-sink: Premultiply the colors coming from the video

Robert Bragg robert at sixbynine.org
Thu Jan 30 16:34:38 PST 2014


Ah, good catch!

I think it could be good to extend the gtk-doc comment about sampling
the video via cogl_gst_sample_video0() to explicitly note that this
function will always return you a premultiplied alpha value.

Apart from that this looks to land to me:

Reviewed-by: Robert Bragg <robert at linux.intel.com>

Thanks,
Robert


On Wed, Jan 22, 2014 at 3:22 PM, Neil Roberts <neil at linux.intel.com> wrote:
> Commit 99a53c82e9ab0a1e5 removed the internal format argument when
> uploading a video frame to a texture so that the format will just be
> determined automatically from the image format. However this also
> leaves the premultiplied state at the default and the default is TRUE.
> That means that when we upload RGBA data Cogl will do a premultiplied
> conversion on the CPU. We probably don't want to be putting a CPU
> conversion in the way of video frames so this patch changes it to set
> the premultiplied state to FALSE on the textures and then do the
> premultiplied conversion in the shader.
>
> This is particularly important for AYUV which uses the alpha channel
> for the V component so doing a premultiplied conversion on the CPU
> just creates garbage and messes up the image.
>
> The RGB and RGBA renderers have each been split into two; one that
> uses GLSL and one that uses a regular pipeline. The RGBA pipeline
> without GLSL is then changed to use 2 layers so we that we can do the
> premultiplied conversion in the second layer with a special layer
> combine string.
> ---
>  cogl-gst/cogl-gst-video-sink.c | 159 +++++++++++++++++++++++++++++++----------
>  1 file changed, 122 insertions(+), 37 deletions(-)
>
> diff --git a/cogl-gst/cogl-gst-video-sink.c b/cogl-gst/cogl-gst-video-sink.c
> index f9887aa..243d9ae 100644
> --- a/cogl-gst/cogl-gst-video-sink.c
> +++ b/cogl-gst/cogl-gst-video-sink.c
> @@ -417,40 +417,6 @@ clear_frame_textures (CoglGstVideoSink *sink)
>    priv->frame_dirty = TRUE;
>  }
>
> -static void
> -cogl_gst_rgb_setup_pipeline (CoglGstVideoSink *sink,
> -                             CoglPipeline *pipeline)
> -{
> -  CoglGstVideoSinkPrivate *priv = sink->priv;
> -
> -  if (cogl_has_feature (priv->ctx, COGL_FEATURE_ID_GLSL))
> -    {
> -      static SnippetCache snippet_cache;
> -      SnippetCacheEntry *entry = get_cache_entry (sink, &snippet_cache);
> -
> -      if (entry == NULL)
> -        {
> -          char *source;
> -
> -          source =
> -            g_strdup_printf ("vec4\n"
> -                             "cogl_gst_sample_video%i (vec2 UV)\n"
> -                             "{\n"
> -                             "  return texture2D (cogl_sampler%i, UV);\n"
> -                             "}\n",
> -                             priv->custom_start,
> -                             priv->custom_start);
> -
> -          entry = add_cache_entry (sink, &snippet_cache, source);
> -          g_free (source);
> -        }
> -
> -      setup_pipeline_from_cache_entry (sink, pipeline, entry, 1);
> -    }
> -  else
> -    setup_pipeline_from_cache_entry (sink, pipeline, NULL, 1);
> -}
> -
>  static inline CoglBool
>  is_pot (unsigned int number)
>  {
> @@ -504,9 +470,46 @@ video_texture_new_from_data (CoglContext *ctx,
>
>    cogl_object_unref (bitmap);
>
> +  cogl_texture_set_premultiplied (tex, FALSE);
> +
>    return tex;
>  }
>
> +static void
> +cogl_gst_rgb24_glsl_setup_pipeline (CoglGstVideoSink *sink,
> +                                    CoglPipeline *pipeline)
> +{
> +  CoglGstVideoSinkPrivate *priv = sink->priv;
> +  static SnippetCache snippet_cache;
> +  SnippetCacheEntry *entry = get_cache_entry (sink, &snippet_cache);
> +
> +  if (entry == NULL)
> +    {
> +      char *source;
> +
> +      source =
> +        g_strdup_printf ("vec4\n"
> +                         "cogl_gst_sample_video%i (vec2 UV)\n"
> +                         "{\n"
> +                         "  return texture2D (cogl_sampler%i, UV);\n"
> +                         "}\n",
> +                         priv->custom_start,
> +                         priv->custom_start);
> +
> +      entry = add_cache_entry (sink, &snippet_cache, source);
> +      g_free (source);
> +    }
> +
> +  setup_pipeline_from_cache_entry (sink, pipeline, entry, 1);
> +}
> +
> +static void
> +cogl_gst_rgb24_setup_pipeline (CoglGstVideoSink *sink,
> +                               CoglPipeline *pipeline)
> +{
> +  setup_pipeline_from_cache_entry (sink, pipeline, NULL, 1);
> +}
> +
>  static CoglBool
>  cogl_gst_rgb24_upload (CoglGstVideoSink *sink,
>                         GstBuffer *buffer)
> @@ -542,6 +545,17 @@ map_fail:
>    }
>  }
>
> +static CoglGstRenderer rgb24_glsl_renderer =
> +{
> +  "RGB 24",
> +  COGL_GST_RGB24,
> +  COGL_GST_RENDERER_NEEDS_GLSL,
> +  GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGB, BGR }")),
> +  1, /* n_layers */
> +  cogl_gst_rgb24_glsl_setup_pipeline,
> +  cogl_gst_rgb24_upload,
> +};
> +
>  static CoglGstRenderer rgb24_renderer =
>  {
>    "RGB 24",
> @@ -549,10 +563,61 @@ static CoglGstRenderer rgb24_renderer =
>    0,
>    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGB, BGR }")),
>    1, /* n_layers */
> -  cogl_gst_rgb_setup_pipeline,
> +  cogl_gst_rgb24_setup_pipeline,
>    cogl_gst_rgb24_upload,
>  };
>
> +static void
> +cogl_gst_rgb32_glsl_setup_pipeline (CoglGstVideoSink *sink,
> +                                    CoglPipeline *pipeline)
> +{
> +  CoglGstVideoSinkPrivate *priv = sink->priv;
> +  static SnippetCache snippet_cache;
> +  SnippetCacheEntry *entry = get_cache_entry (sink, &snippet_cache);
> +
> +  if (entry == NULL)
> +    {
> +      char *source;
> +
> +      source =
> +        g_strdup_printf ("vec4\n"
> +                         "cogl_gst_sample_video%i (vec2 UV)\n"
> +                         "{\n"
> +                         "  vec4 color = texture2D (cogl_sampler%i, UV);\n"
> +                         /* Premultiply the color */
> +                         "  color.rgb *= color.a;\n"
> +                         "  return color;\n"
> +                         "}\n",
> +                         priv->custom_start,
> +                         priv->custom_start);
> +
> +      entry = add_cache_entry (sink, &snippet_cache, source);
> +      g_free (source);
> +    }
> +
> +  setup_pipeline_from_cache_entry (sink, pipeline, entry, 1);
> +}
> +
> +static void
> +cogl_gst_rgb32_setup_pipeline (CoglGstVideoSink *sink,
> +                               CoglPipeline *pipeline)
> +{
> +  CoglGstVideoSinkPrivate *priv = sink->priv;
> +  char *layer_combine;
> +
> +  setup_pipeline_from_cache_entry (sink, pipeline, NULL, 1);
> +
> +  /* Premultiply the texture using the a special layer combine */
> +  layer_combine = g_strdup_printf ("RGB=MODULATE(PREVIOUS, TEXTURE_%i[A])\n"
> +                                   "A=REPLACE(PREVIOUS[A])",
> +                                   priv->custom_start);
> +  cogl_pipeline_set_layer_combine (pipeline,
> +                                   priv->custom_start + 1,
> +                                   layer_combine,
> +                                   NULL);
> +  g_free(layer_combine);
> +}
> +
>  static CoglBool
>  cogl_gst_rgb32_upload (CoglGstVideoSink *sink,
>                         GstBuffer *buffer)
> @@ -588,14 +653,25 @@ map_fail:
>    }
>  }
>
> +static CoglGstRenderer rgb32_glsl_renderer =
> +{
> +  "RGB 32",
> +  COGL_GST_RGB32,
> +  COGL_GST_RENDERER_NEEDS_GLSL,
> +  GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGBA, BGRA }")),
> +  1, /* n_layers */
> +  cogl_gst_rgb32_glsl_setup_pipeline,
> +  cogl_gst_rgb32_upload,
> +};
> +
>  static CoglGstRenderer rgb32_renderer =
>  {
>    "RGB 32",
>    COGL_GST_RGB32,
>    0,
>    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ RGBA, BGRA }")),
> -  1, /* n_layers */
> -  cogl_gst_rgb_setup_pipeline,
> +  2, /* n_layers */
> +  cogl_gst_rgb32_setup_pipeline,
>    cogl_gst_rgb32_upload,
>  };
>
> @@ -778,6 +854,8 @@ cogl_gst_ayuv_glsl_setup_pipeline (CoglGstVideoSink *sink,
>                             "  color.r = y + 1.59765625 * v;\n"
>                             "  color.g = y - 0.390625 * u - 0.8125 * v;\n"
>                             "  color.b = y + 2.015625 * u;\n"
> +                           /* Premultiply the color */
> +                           "  color.rgb *= color.a;\n"
>                             "  return color;\n"
>                             "}\n", priv->custom_start,
>                             priv->custom_start);
> @@ -838,8 +916,13 @@ cogl_gst_build_renderers_list (CoglContext *ctx)
>    int i;
>    static CoglGstRenderer *const renderers[] =
>    {
> +    /* These are in increasing order of priority so that the
> +     * priv->renderers will be in decreasing order. That way the GLSL
> +     * renderers will be preferred if they are available */
>      &rgb24_renderer,
>      &rgb32_renderer,
> +    &rgb24_glsl_renderer,
> +    &rgb32_glsl_renderer,
>      &yv12_glsl_renderer,
>      &i420_glsl_renderer,
>      &ayuv_glsl_renderer,
> @@ -916,6 +999,8 @@ cogl_gst_find_renderer_by_format (CoglGstVideoSink *sink,
>    CoglGstRenderer *renderer = NULL;
>    GSList *element;
>
> +  /* The renderers list is in decreasing order of priority so we'll
> +   * pick the first one that matches */
>    for (element = priv->renderers; element; element = g_slist_next (element))
>      {
>        CoglGstRenderer *candidate = (CoglGstRenderer *) element->data;
> --
> 1.8.4.2
>
> _______________________________________________
> Cogl mailing list
> Cogl at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/cogl


More information about the Cogl mailing list