[Spice-devel] [PATCH spice-gtk 5/5] display-gst: Use Playbin for GStreamer 1.9.0 onwards

[Victor Toso]

From: Victor Toso <me at victortoso.com>

The Playbin can provide the full pipeline which reduces the
overall maintenance in the code as we don't need to track which
decoder can work with our stream type.

We need to maintain the GstCaps per SPICE_VIDEO_CODEC_TYPE in order to
tell Playbin the type of data we expect. This much should be covered
by spice-protocol and very likely we will need to extend it in the
future to allow more settings that might not possible to verify at
runtime.

This patch keeps previous code for compatibility reasons.

Note that we have to wait Playbin to emit "source-setup" in order to
configure GstAppSrc with the capabilities of input stream. If in the
unlikely event of frames arriving while GstAppSrc is not setup, we
will drop those frames.

Signed-off-by: Victor Toso <victortoso at redhat.com>
Signed-off-by: Victor Toso <me at victortoso.com>
---
 src/channel-display-gst.c | 88 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 86 insertions(+), 2 deletions(-)

diff --git a/src/channel-display-gst.c b/src/channel-display-gst.c
index 2a20763..2b14745 100644
--- a/src/channel-display-gst.c
+++ b/src/channel-display-gst.c
@@ -50,6 +50,9 @@ typedef struct SpiceGstDecoder {
     guint timer_id;
 } SpiceGstDecoder;
 
+/* FIXME: With gstreamer version 1.9.0 and higher, we are using playbin to
+ * create the pipeline for us and for that reason we don't need to keep track of
+ * decoder's name anymore. */
 static struct {
     const gchar *dec_name;
     const gchar *dec_caps;
@@ -256,10 +259,13 @@ static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_decoder)
             }
             l = l->next;
         }
+#if !GST_CHECK_VERSION(1,9,0)
+        /* We down own this sample when using playbin, no need to unref it */
         if (!l) {
             spice_warning("got an unexpected decoded buffer!");
             gst_sample_unref(sample);
         }
+#endif
 
         g_mutex_unlock(&decoder->queues_mutex);
         schedule_frame(decoder);
@@ -276,8 +282,11 @@ static void free_pipeline(SpiceGstDecoder *decoder)
     }
 
     gst_element_set_state(decoder->pipeline, GST_STATE_NULL);
+#if !GST_CHECK_VERSION(1,9,0)
+    /* Owned by playbin on 1.9.0 onwards */
     gst_object_unref(decoder->appsrc);
     gst_object_unref(decoder->appsink);
+#endif
     gst_object_unref(decoder->pipeline);
     gst_object_unref(decoder->clock);
     decoder->pipeline = NULL;
@@ -305,13 +314,79 @@ static gboolean handle_pipeline_message(GstBus *bus, GstMessage *msg, gpointer v
     return TRUE;
 }
 
+#if GST_CHECK_VERSION(1,9,0)
+static void app_source_setup(GstElement *pipeline, GstElement *source, SpiceGstDecoder *decoder)
+{
+    GstCaps *caps;
+
+    /* - We schedule the frame display ourselves so set sync=false on appsink
+     *   so the pipeline decodes them as fast as possible. This will also
+     *   minimize the risk of frames getting lost when we rebuild the
+     *   pipeline.
+     * - Set max-bytes=0 on appsrc so it does not drop frames that may be
+     *   needed by those that follow.
+     */
+    caps = gst_caps_from_string(gst_opts[decoder->base.codec_type].dec_caps);
+    g_object_set(source,
+                 "caps", caps,
+                 "is-live", TRUE,
+                 "format", GST_FORMAT_TIME,
+                 "max-bytes", 0,
+                 "block", TRUE,
+                 NULL);
+    gst_caps_unref(caps);
+    decoder->appsrc = GST_APP_SRC(source);
+}
+#endif
+
 static gboolean create_pipeline(SpiceGstDecoder *decoder)
 {
-    gchar *desc;
     GstAppSinkCallbacks appsink_cbs = { NULL };
-    GError *err = NULL;
     GstBus *bus;
+#if GST_CHECK_VERSION(1,9,0)
+    GstElement *playbin, *sink;
+    gint flags;
+    GstCaps *caps;
 
+    playbin = gst_element_factory_make("playbin", "playbin");
+    if (playbin == NULL) {
+        spice_warning("error upon creation of 'playbin' element");
+        return FALSE;
+    }
+
+    sink = gst_element_factory_make("appsink", "sink");
+    if (sink == NULL) {
+        spice_warning("error upon creation of 'appsink' element");
+        gst_object_unref(playbin);
+        return FALSE;
+    }
+
+    caps = gst_caps_from_string("video/x-raw,format=BGRx");
+    g_object_set(sink,
+                 "caps", caps,
+                 "sync", FALSE,
+                 "drop", FALSE,
+                 NULL);
+    gst_caps_unref(caps);
+
+    g_object_set(playbin,
+                 "uri", "appsrc://",
+                 "video-sink", sink,
+                 NULL);
+
+    g_signal_connect(playbin, "source-setup", G_CALLBACK(app_source_setup), decoder);
+
+    /* Disable audio in playbin */
+    g_object_get(playbin, "flags", &flags, NULL);
+    flags &= ~(1 << 1);
+    g_object_set(playbin, "flags", flags, NULL);
+
+    decoder->appsrc = NULL;
+    decoder->appsink = GST_APP_SINK(sink);
+    decoder->pipeline = playbin;
+#else
+    gchar *desc;
+    GError *err = NULL;
 
     /* - We schedule the frame display ourselves so set sync=false on appsink
      *   so the pipeline decodes them as fast as possible. This will also
@@ -337,6 +412,7 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
 
     decoder->appsrc = GST_APP_SRC(gst_bin_get_by_name(GST_BIN(decoder->pipeline), "src"));
     decoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(decoder->pipeline), "sink"));
+#endif
 
     appsink_cbs.new_sample = new_sample;
     gst_app_sink_set_callbacks(decoder->appsink, &appsink_cbs, decoder, NULL);
@@ -447,6 +523,14 @@ static gboolean spice_gst_decoder_queue_frame(VideoDecoder *video_decoder,
         return FALSE;
     }
 
+#if GST_CHECK_VERSION(1,9,0)
+    if (decoder->appsrc == NULL) {
+        spice_warning("Error: Playbin has not yet initialized the Appsrc element");
+        stream_dropped_frame_on_playback(decoder->base.stream);
+        return TRUE;
+    }
+#endif
+
     /* ref() the frame_msg for the buffer */
     spice_msg_in_ref(frame_msg);
     GstBuffer *buffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_PHYSICALLY_CONTIGUOUS,
-- 
2.12.2