[Cogl] [PATCH 1/3] cogl-gst: video-sink: add NV12 support

[Neil Roberts]

This adds a cogl-gst renderer to decode NV12 data. NV12 is split into
two buffers, one for the luma component and another for the two
chrominance components at a quarter of the resolution. The second
buffer is uploaded to a two-component RG texture. RG-component
textures are only supported if COGL_FEATURE_ID_TEXTURE_RG is
advertised by Cogl so the NV12 cap is also only advertised when that
is available.

Based on a patch by Lionel Landwerlin which was in turn based on a
patch from Edward Hervey and Matthieu Bouron for Clutter-Gst:

https://bugzilla.gnome.org/show_bug.cgi?id=712832
---
 cogl-gst/cogl-gst-video-sink.c | 117 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 111 insertions(+), 6 deletions(-)

diff --git a/cogl-gst/cogl-gst-video-sink.c b/cogl-gst/cogl-gst-video-sink.c
index 1b83c9e..7708f59 100644
--- a/cogl-gst/cogl-gst-video-sink.c
+++ b/cogl-gst/cogl-gst-video-sink.c
@@ -51,7 +51,8 @@
                        "RGBA," \
                        "BGRA," \
                        "RGB," \
-                       "BGR }"
+                       "BGR," \
+                       "NV12 }"
 
 #define SINK_CAPS GST_VIDEO_CAPS_MAKE (BASE_SINK_CAPS)
 
@@ -99,12 +100,14 @@ typedef enum
   COGL_GST_AYUV,
   COGL_GST_YV12,
   COGL_GST_SURFACE,
-  COGL_GST_I420
+  COGL_GST_I420,
+  COGL_GST_NV12
 } CoglGstVideoFormat;
 
 typedef enum
 {
-  COGL_GST_RENDERER_NEEDS_GLSL = (1 << 0)
+  COGL_GST_RENDERER_NEEDS_GLSL = (1 << 0),
+  COGL_GST_RENDERER_NEEDS_TEXTURE_RG = (1 << 1)
 } CoglGstRendererFlag;
 
 /* We want to cache the snippets instead of recreating a new one every
@@ -828,11 +831,105 @@ static CoglGstRenderer ayuv_glsl_renderer =
   cogl_gst_ayuv_upload,
 };
 
+static void
+cogl_gst_nv12_glsl_setup_pipeline (CoglGstVideoSink *sink,
+                                   CoglPipeline *pipeline)
+{
+  CoglGstVideoSinkPrivate *priv = sink->priv;
+  static SnippetCache snippet_cache;
+  SnippetCacheEntry *entry;
+
+  entry = get_cache_entry (sink, &snippet_cache);
+
+  if (entry == NULL)
+    {
+      char *source;
+
+      source =
+        g_strdup_printf ("vec4\n"
+                         "cogl_gst_sample_video%i (vec2 UV)\n"
+                         "{\n"
+                         "  vec4 color;\n"
+                         "  float y = 1.1640625 *\n"
+                         "            (texture2D (cogl_sampler%i, UV).a -\n"
+                         "             0.0625);\n"
+                         "  vec2 uv = texture2D (cogl_sampler%i, UV).rg;\n"
+                         "  uv -= 0.5;\n"
+                         "  float u = uv.x;\n"
+                         "  float v = uv.y;\n"
+                         "  color.r = y + 1.59765625 * v;\n"
+                         "  color.g = y - 0.390625 * u - 0.8125 * v;\n"
+                         "  color.b = y + 2.015625 * u;\n"
+                         "  color.a = 1.0;\n"
+                         "  return color;\n"
+                         "}\n",
+                         priv->custom_start,
+                         priv->custom_start,
+                         priv->custom_start + 1);
+
+      entry = add_cache_entry (sink, &snippet_cache, source);
+      g_free (source);
+    }
+
+  setup_pipeline_from_cache_entry (sink, pipeline, entry, 2);
+}
+
+static CoglBool
+cogl_gst_nv12_upload (CoglGstVideoSink *sink,
+                      GstBuffer *buffer)
+{
+  CoglGstVideoSinkPrivate *priv = sink->priv;
+  GstVideoFrame frame;
+
+  if (!gst_video_frame_map (&frame, &priv->info, buffer, GST_MAP_READ))
+    goto map_fail;
+
+  clear_frame_textures (sink);
+
+  priv->frame[0] =
+    video_texture_new_from_data (priv->ctx,
+                                 GST_VIDEO_INFO_COMP_WIDTH (&priv->info, 0),
+                                 GST_VIDEO_INFO_COMP_HEIGHT (&priv->info, 0),
+                                 COGL_PIXEL_FORMAT_A_8,
+                                 priv->info.stride[0],
+                                 frame.data[0]);
+
+  priv->frame[1] =
+    video_texture_new_from_data (priv->ctx,
+                                 GST_VIDEO_INFO_COMP_WIDTH (&priv->info, 1),
+                                 GST_VIDEO_INFO_COMP_HEIGHT (&priv->info, 1),
+                                 COGL_PIXEL_FORMAT_RG_88,
+                                 priv->info.stride[1],
+                                 frame.data[1]);
+
+  gst_video_frame_unmap (&frame);
+
+  return TRUE;
+
+ map_fail:
+  {
+    GST_ERROR_OBJECT (sink, "Could not map incoming video frame");
+    return FALSE;
+  }
+}
+
+static CoglGstRenderer nv12_glsl_renderer =
+{
+  "NV12 glsl",
+  COGL_GST_NV12,
+  COGL_GST_RENDERER_NEEDS_GLSL | COGL_GST_RENDERER_NEEDS_TEXTURE_RG,
+  GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("memory:SystemMemory",
+                                                      "NV12")),
+  2, /* n_layers */
+  cogl_gst_nv12_glsl_setup_pipeline,
+  cogl_gst_nv12_upload,
+};
+
 static GSList*
 cogl_gst_build_renderers_list (CoglContext *ctx)
 {
   GSList *list = NULL;
-  CoglBool has_glsl;
+  CoglGstRendererFlag flags = 0;
   int i;
   static CoglGstRenderer *const renderers[] =
   {
@@ -841,13 +938,18 @@ cogl_gst_build_renderers_list (CoglContext *ctx)
     &yv12_glsl_renderer,
     &i420_glsl_renderer,
     &ayuv_glsl_renderer,
+    &nv12_glsl_renderer,
     NULL
   };
 
-  has_glsl = cogl_has_feature (ctx, COGL_FEATURE_ID_GLSL);
+  if (cogl_has_feature (ctx, COGL_FEATURE_ID_GLSL))
+    flags |= COGL_GST_RENDERER_NEEDS_GLSL;
+
+  if (cogl_has_feature (ctx, COGL_FEATURE_ID_TEXTURE_RG))
+    flags |= COGL_GST_RENDERER_NEEDS_TEXTURE_RG;
 
   for (i = 0; renderers[i]; i++)
-    if (has_glsl || !(renderers[i]->flags & COGL_GST_RENDERER_NEEDS_GLSL))
+    if ((renderers[i]->flags & flags) == renderers[i]->flags)
       list = g_slist_prepend (list, renderers[i]);
 
   return list;
@@ -969,6 +1071,9 @@ cogl_gst_video_sink_parse_caps (GstCaps *caps,
       format = COGL_GST_AYUV;
       bgr = FALSE;
       break;
+    case GST_VIDEO_FORMAT_NV12:
+      format = COGL_GST_NV12;
+      break;
     case GST_VIDEO_FORMAT_RGB:
       format = COGL_GST_RGB24;
       bgr = FALSE;
-- 
1.8.4.2