[Libva] Problem trying to make GStreamer vaapiupload element capable of passthrough

[Simon Farnsworth]

Hello,

I'm trying to adapt an existing in-house GStreamer player to use VA-API
instead of software decode wherever possible, while still continuing to use
software decode when VA-API can't handle a media format.

We have a user requirement to control deinterlacing programmatically, so our
current player doesn't quite use playbin2 on its own; instead, we set up a 
video output bin on playbin2's "video-sink" property, which looks like:

"queue ! deinterlace ! queue ! xvimagesink".

My current goal is to replace that with a suitable bin to use VA-API decoders - the obvious is to choose either:
"queue ! vaapipostproc ! queue ! vaapisink"
or
"queue ! vaapiupload ! vaapipostproc ! queue ! vaapisink"

depending on which decoder playbin2 chooses, but that doesn't seem to be
possible (playbin2 wants the video-sink upfront, before it's chosen a
decoder). Therefore, I'm trying to alter vaapiupload, so that I can use
"queue ! vaapiupload ! vaapipostproc ! queue ! vaapisink"
unconditionally, and have vaapiupload go into passthrough mode if there's a 
vaapidecoder feeding it.

I've got as far as getting the caps negotiation to work, but I'm being
tripped up by the buffer management. I'm testing with the following command
line:

gst-launch-0.10 -vvm udpsrc uri=udp://239.192.7.203:5000 ! tsdemux name=demux ! queue ! vaapidecode ! vaapiupload ! vaapipostproc ! vaapisink

I see that gst_vaapiupload_prepare_output_buffer is still being called, and
that I get an assertion failure from gst_vaapi_video_buffer_glx_new_from_pool:

** (gst-launch-0.10:4944): CRITICAL **: gst_vaapi_video_buffer_glx_new_from_pool: assertion `GST_VAAPI_IS_VIDEO_POOL (pool)' failed

I've made the following change to vaapiupload so far; what have I missed that
would be causing this problem? I'm currently working against the 0.3 branch,
but can shift to master if that's needed (and I'm happy to backport fixes from
master to 0.3).

diff --git a/gst/vaapi/gstvaapiupload.c b/gst/vaapi/gstvaapiupload.c
index 0762279..4ac7a5f 100644
--- a/gst/vaapi/gstvaapiupload.c
+++ b/gst/vaapi/gstvaapiupload.c
@@ -67,6 +67,12 @@ static const char gst_vaapiupload_yuv_caps_str[] =
     "width  = (int) [ 1, MAX ], "
     "height = (int) [ 1, MAX ]; ";
 
+static const char gst_vaapiupload_sink_caps_str[] =
+    "video/x-raw-yuv, "
+    "width  = (int) [ 1, MAX ], "
+    "height = (int) [ 1, MAX ]; "
+    GST_VAAPI_SURFACE_CAPS;
+
 static const char gst_vaapiupload_vaapi_caps_str[] =
     GST_VAAPI_SURFACE_CAPS;
 
@@ -75,7 +81,7 @@ static GstStaticPadTemplate gst_vaapiupload_sink_factory =
         "sink",
         GST_PAD_SINK,
         GST_PAD_ALWAYS,
-        GST_STATIC_CAPS(gst_vaapiupload_yuv_caps_str));
+        GST_STATIC_CAPS(gst_vaapiupload_sink_caps_str));
 
 static GstStaticPadTemplate gst_vaapiupload_src_factory =
     GST_STATIC_PAD_TEMPLATE(
@@ -297,6 +303,7 @@ gst_vaapiupload_class_init(GstVaapiUploadClass *klass)
     trans_class->set_caps       = gst_vaapiupload_set_caps;
     trans_class->get_unit_size  = gst_vaapiupload_get_unit_size;
     trans_class->prepare_output_buffer = gst_vaapiupload_prepare_output_buffer;
+    trans_class->passthrough_on_same_caps = TRUE;
 
     gst_element_class_set_details_simple(
         element_class,
@@ -476,17 +483,21 @@ gst_vaapiupload_transform_caps(
     structure = gst_caps_get_structure(caps, 0);
 
     if (direction == GST_PAD_SINK) {
-        if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
+        if (gst_structure_has_name(structure, "video/x-raw-yuv")) {
+            out_caps = gst_caps_from_string(gst_vaapiupload_vaapi_caps_str);
+
+            structure = gst_caps_get_structure(out_caps, 0);
+            gst_structure_set(
+                structure,
+                "type", G_TYPE_STRING, "vaapi",
+                "opengl", G_TYPE_BOOLEAN, USE_VAAPI_GLX,
+                NULL
+            );
+        }
+        else if(gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
+            out_caps = gst_caps_copy(caps);
+        else
             return NULL;
-        out_caps = gst_caps_from_string(gst_vaapiupload_vaapi_caps_str);
-
-        structure = gst_caps_get_structure(out_caps, 0);
-        gst_structure_set(
-            structure,
-            "type", G_TYPE_STRING, "vaapi",
-            "opengl", G_TYPE_BOOLEAN, USE_VAAPI_GLX,
-            NULL
-        );
     }
     else {
         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
@@ -502,12 +513,14 @@ gst_vaapiupload_transform_caps(
             gst_caps_unref(out_caps);
             out_caps = inter_caps;
         }
+        gst_caps_append(out_caps, gst_caps_from_string(gst_vaapiupload_vaapi_caps_str));
     }
 
     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
         gst_caps_unref(out_caps);
         return NULL;
     }
+
     return out_caps;
 }
 
@@ -655,6 +668,9 @@ gst_vaapiupload_set_caps(
 {
     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 
+    if (gst_caps_is_equal(incaps, outcaps))
+        return TRUE;
+
     if (!gst_vaapiupload_negotiate_buffers(upload, incaps, outcaps))
         return FALSE;
 


-- 
Simon Farnsworth
Software Engineer
ONELAN Ltd
http://www.onelan.com
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 490 bytes
Desc: This is a digitally signed message part.
URL: <http://lists.freedesktop.org/archives/libva/attachments/20120905/c2c3dbeb/attachment.pgp>