[Spice-devel] [RFC spice-gtk v3 1/1] Gstreamer: Use GstVideoOverlay if possible
Snir Sheriber
ssheribe at redhat.com
Wed Apr 25 12:43:14 UTC 2018
Currently when gstreamer is used to decode a full-screen
stream sent from the server, the decoding frames are being
forced to RBGA format and pushed using appsink to be scaled
and rendered to screen.
Today most of the gstreamer sinks supports the GstVideoOverlay
interface which allows to render directly from the pipeline to
a window by a given windowing system id (i.e. xid). This patch
makes playbin to use this feature if possible.
Setting the DISABLE_GSTVIDEOOVERLAY environment variable will
make gstreamer to avoid of using the gstvideooverlay interface.
---
src/channel-display-gst.c | 92 +++++++++++++++++++++++++++++++++++-----------
src/channel-display-priv.h | 2 +
src/channel-display.c | 35 ++++++++++++++++++
src/channel-display.h | 3 ++
src/spice-marshal.txt | 1 +
src/spice-widget.c | 33 +++++++++++++++++
6 files changed, 145 insertions(+), 21 deletions(-)
diff --git a/src/channel-display-gst.c b/src/channel-display-gst.c
index 0d7aabb..8029e34 100644
--- a/src/channel-display-gst.c
+++ b/src/channel-display-gst.c
@@ -41,6 +41,8 @@ typedef struct SpiceGstDecoder {
GstElement *pipeline;
GstClock *clock;
+ guintptr win_handle;
+
/* ---------- Decoding and display queues ---------- */
uint32_t last_mm_time;
@@ -265,7 +267,8 @@ static void free_pipeline(SpiceGstDecoder *decoder)
gst_element_set_state(decoder->pipeline, GST_STATE_NULL);
gst_object_unref(decoder->appsrc);
- gst_object_unref(decoder->appsink);
+ if (decoder->appsink)
+ gst_object_unref(decoder->appsink);
gst_object_unref(decoder->pipeline);
gst_object_unref(decoder->clock);
decoder->pipeline = NULL;
@@ -306,6 +309,18 @@ static gboolean handle_pipeline_message(GstBus *bus, GstMessage *msg, gpointer v
g_free(filename);
break;
}
+ case GST_MESSAGE_ELEMENT: {
+ if (gst_is_video_overlay_prepare_window_handle_message(msg)) {
+ GstVideoOverlay *overlay;
+
+ SPICE_DEBUG("prepare-window-handle msg received (handle: %" PRIuPTR ")", decoder->win_handle);
+ if (decoder->win_handle != 0) {
+ overlay = GST_VIDEO_OVERLAY(GST_MESSAGE_SRC(msg));
+ gst_video_overlay_set_window_handle(overlay, decoder->win_handle);
+ }
+ }
+ break;
+ }
default:
/* not being handled */
break;
@@ -342,7 +357,6 @@ static void app_source_setup(GstElement *pipeline G_GNUC_UNUSED,
static gboolean create_pipeline(SpiceGstDecoder *decoder)
{
- GstAppSinkCallbacks appsink_cbs = { NULL };
GstBus *bus;
#if GST_CHECK_VERSION(1,9,0)
GstElement *playbin, *sink;
@@ -355,26 +369,56 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
return FALSE;
}
- sink = gst_element_factory_make("appsink", "sink");
- if (sink == NULL) {
- spice_warning("error upon creation of 'appsink' element");
- gst_object_unref(playbin);
- return FALSE;
- }
-
- caps = gst_caps_from_string("video/x-raw,format=BGRx");
- g_object_set(sink,
+ /* Will try to get window handle in order to apply the GstVideoOverlay
+ * interface, setting overlay to this window will happen only when
+ * prepare-window-handle message is received
+ */
+ decoder->win_handle = get_window_handle(decoder->base.stream);
+ SPICE_DEBUG("Creating Gstreamer pipline (handle for overlay %s)\n",
+ decoder->win_handle ? "received" : "not received");
+ if (decoder->win_handle == 0) {
+ sink = gst_element_factory_make("appsink", "sink");
+ if (sink == NULL) {
+ spice_warning("error upon creation of 'appsink' element");
+ gst_object_unref(playbin);
+ return FALSE;
+ }
+ caps = gst_caps_from_string("video/x-raw,format=BGRx");
+ g_object_set(sink,
"caps", caps,
"sync", FALSE,
"drop", FALSE,
NULL);
- gst_caps_unref(caps);
+ gst_caps_unref(caps);
+ g_object_set(playbin,
+ "video-sink", gst_object_ref(sink),
+ NULL);
+
+ decoder->appsink = GST_APP_SINK(sink);
+ } else {
+ /* handle has received, it means playbin will render directly into
+ * widget using the gstvideoooverlay interface instead of app-sink.
+ * Also avoid using vaapisink if exist since vaapisink could be
+ * buggy when it is combined with playbin. changing its rank to
+ * none will make playbin to avoid of using it.
+ */
+ GstRegistry *registry = NULL;
+ GstPluginFeature *vaapisink = NULL;
+
+ registry = gst_registry_get();
+ if (registry) {
+ vaapisink = gst_registry_lookup_feature(registry, "vaapisink");
+ }
+ if (vaapisink) {
+ gst_plugin_feature_set_rank(vaapisink, GST_RANK_NONE);
+ gst_object_unref(vaapisink);
+ }
+ }
g_signal_connect(playbin, "source-setup", G_CALLBACK(app_source_setup), decoder);
g_object_set(playbin,
"uri", "appsrc://",
- "video-sink", gst_object_ref(sink),
NULL);
/* Disable audio in playbin */
@@ -383,7 +427,6 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
g_object_set(playbin, "flags", flags, NULL);
g_warn_if_fail(decoder->appsrc == NULL);
- decoder->appsink = GST_APP_SINK(sink);
decoder->pipeline = playbin;
#else
gchar *desc;
@@ -415,8 +458,11 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
decoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(decoder->pipeline), "sink"));
#endif
- appsink_cbs.new_sample = new_sample;
- gst_app_sink_set_callbacks(decoder->appsink, &appsink_cbs, decoder, NULL);
+ if (decoder->appsink) {
+ GstAppSinkCallbacks appsink_cbs = { NULL };
+ appsink_cbs.new_sample = new_sample;
+ gst_app_sink_set_callbacks(decoder->appsink, &appsink_cbs, decoder, NULL);
+ }
bus = gst_pipeline_get_bus(GST_PIPELINE(decoder->pipeline));
gst_bus_add_watch(bus, handle_pipeline_message, decoder);
gst_object_unref(bus);
@@ -565,13 +611,17 @@ static gboolean spice_gst_decoder_queue_frame(VideoDecoder *video_decoder,
GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE;
GST_BUFFER_PTS(buffer) = gst_clock_get_time(decoder->clock) - gst_element_get_base_time(decoder->pipeline) + ((uint64_t)MAX(0, latency)) * 1000 * 1000;
- SpiceGstFrame *gst_frame = create_gst_frame(buffer, frame);
- g_mutex_lock(&decoder->queues_mutex);
- g_queue_push_tail(decoder->decoding_queue, gst_frame);
- g_mutex_unlock(&decoder->queues_mutex);
+ if (decoder->appsink != NULL) {
+ SpiceGstFrame *gst_frame = create_gst_frame(buffer, frame);
+ g_mutex_lock(&decoder->queues_mutex);
+ g_queue_push_tail(decoder->decoding_queue, gst_frame);
+ g_mutex_unlock(&decoder->queues_mutex);
+ } else {
+ frame->free(frame);
+ }
if (gst_app_src_push_buffer(decoder->appsrc, buffer) != GST_FLOW_OK) {
- SPICE_DEBUG("GStreamer error: unable to push frame of size %u", frame->size);
+ SPICE_DEBUG("GStreamer error: unable to push frame");
stream_dropped_frame_on_playback(decoder->base.stream);
}
return TRUE;
diff --git a/src/channel-display-priv.h b/src/channel-display-priv.h
index 6a90a78..76d4dd0 100644
--- a/src/channel-display-priv.h
+++ b/src/channel-display-priv.h
@@ -98,6 +98,7 @@ gboolean gstvideo_has_codec(int codec_type);
typedef struct display_surface {
guint32 surface_id;
bool primary;
+ bool streaming_mode;
enum SpiceSurfaceFmt format;
int width, height, stride, size;
uint8_t *data;
@@ -195,6 +196,7 @@ guint32 stream_get_time(display_stream *st);
void stream_dropped_frame_on_playback(display_stream *st);
#define SPICE_UNKNOWN_STRIDE 0
void stream_display_frame(display_stream *st, SpiceFrame *frame, uint32_t width, uint32_t height, int stride, uint8_t* data);
+guintptr get_window_handle(display_stream *st);
G_END_DECLS
diff --git a/src/channel-display.c b/src/channel-display.c
index ed003be..cc0a4a1 100644
--- a/src/channel-display.c
+++ b/src/channel-display.c
@@ -91,6 +91,7 @@ enum {
SPICE_DISPLAY_INVALIDATE,
SPICE_DISPLAY_MARK,
SPICE_DISPLAY_GL_DRAW,
+ SPICE_DISPLAY_STREAMING_MODE,
SPICE_DISPLAY_LAST_SIGNAL,
};
@@ -454,6 +455,30 @@ static void spice_display_channel_class_init(SpiceDisplayChannelClass *klass)
4,
G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT);
+ /**
+ * SpiceDisplayChannel::streaming-mode:
+ * @display: the #SpiceDisplayChannel that emitted the signal
+ * @streaming_mode: %TRUE when it's streaming mode
+ *
+ * Return: handle for the display window if possible
+ *
+ * The #SpiceDisplayChannel::streaming_mode signal is emitted when
+ * spice server is working in streaming mode.
+ *
+ * Since 0.35
+ **/
+ signals[SPICE_DISPLAY_STREAMING_MODE] =
+ g_signal_new("streaming-mode",
+ G_OBJECT_CLASS_TYPE(gobject_class),
+ 0,
+ G_STRUCT_OFFSET(SpiceDisplayChannelClass,
+ streaming_mode),
+ NULL, NULL,
+ g_cclosure_user_marshal_POINTER__BOOLEAN,
+ G_TYPE_POINTER,
+ 1,
+ G_TYPE_BOOLEAN);
+
g_type_class_add_private(klass, sizeof(SpiceDisplayChannelPrivate));
channel_set_handlers(SPICE_CHANNEL_CLASS(klass));
@@ -1370,6 +1395,15 @@ void stream_display_frame(display_stream *st, SpiceFrame *frame,
}
}
+guintptr get_window_handle(display_stream *st)
+{
+ void* handle = 0;
+
+ g_signal_emit(st->channel, signals[SPICE_DISPLAY_STREAMING_MODE], 0,
+ st->surface->streaming_mode, &handle);
+ return st->surface->streaming_mode ? (guintptr)handle : 0;
+}
+
/* after a sequence of 3 drops, push a report to the server, even
* if the report window is bigger */
#define STREAM_REPORT_DROP_SEQ_LEN_LIMIT 3
@@ -1775,6 +1809,7 @@ static void display_handle_surface_create(SpiceChannel *channel, SpiceMsgIn *in)
surface->height = create->height;
surface->stride = create->width * 4;
surface->size = surface->height * surface->stride;
+ surface->streaming_mode = !!(create->flags & SPICE_SURFACE_FLAGS_STREAMING_MODE);
if (create->flags & SPICE_SURFACE_FLAGS_PRIMARY) {
SPICE_DEBUG("surface flags: %x", create->flags);
diff --git a/src/channel-display.h b/src/channel-display.h
index 5b48d2f..9c51aa2 100644
--- a/src/channel-display.h
+++ b/src/channel-display.h
@@ -125,6 +125,7 @@ struct _SpiceDisplayChannel {
* @display_primary_destroy: Signal class handler for the #SpiceDisplayChannel::display-primary-destroy signal.
* @display_invalidate: Signal class handler for the #SpiceDisplayChannel::display-invalidate signal.
* @display_mark: Signal class handler for the #SpiceDisplayChannel::display-mark signal.
+ * @streaming_mode: Signal class handler for the #SpiceDisplayChannel::streaming-mode signal.
*
* Class structure for #SpiceDisplayChannel.
*/
@@ -140,6 +141,8 @@ struct _SpiceDisplayChannelClass {
gint x, gint y, gint w, gint h);
void (*display_mark)(SpiceChannel *channel,
gboolean mark);
+ void (*streaming_mode)(SpiceChannel *channel,
+ gboolean streaming_mode);
/*< private >*/
};
diff --git a/src/spice-marshal.txt b/src/spice-marshal.txt
index 86673bd..b3a8e69 100644
--- a/src/spice-marshal.txt
+++ b/src/spice-marshal.txt
@@ -13,3 +13,4 @@ BOOLEAN:UINT,POINTER,UINT
BOOLEAN:UINT,UINT
VOID:OBJECT,OBJECT
VOID:BOXED,BOXED
+POINTER:BOOLEAN
diff --git a/src/spice-widget.c b/src/spice-widget.c
index 73db593..251b29c 100644
--- a/src/spice-widget.c
+++ b/src/spice-widget.c
@@ -660,6 +660,10 @@ G_GNUC_BEGIN_IGNORE_DEPRECATIONS
G_GNUC_END_IGNORE_DEPRECATIONS
#endif
#endif
+ area = gtk_drawing_area_new();
+ gtk_stack_add_named(d->stack, area, "gst-area");
+ gtk_widget_set_double_buffered(area, true);
+
gtk_widget_show_all(widget);
g_signal_connect(display, "grab-broken-event", G_CALLBACK(grab_broken), NULL);
@@ -2589,6 +2593,33 @@ static void queue_draw_area(SpiceDisplay *display, gint x, gint y,
x, y, width, height);
}
+static void* prepare_streaming_mode(SpiceChannel *channel, bool streaming_mode, gpointer data)
+{
+#ifdef GDK_WINDOWING_X11
+ SpiceDisplay *display = data;
+ SpiceDisplayPrivate *d = display->priv;
+
+ /* GstVideoOverlay will currently be used only under x */
+ if (!g_getenv("DISABLE_GSTVIDEOOVERLAY") &&
+ streaming_mode &&
+ GDK_IS_X11_DISPLAY(gdk_display_get_default())) {
+ GdkWindow *window;
+
+ window = gtk_widget_get_window(GTK_WIDGET(display));
+ if (window) {
+#if GTK_CHECK_VERSION(2,18,0)
+ if (gdk_window_ensure_native (window))
+#endif
+ {
+ gtk_stack_set_visible_child_name(d->stack, "gst-area");
+ return (void*)GDK_WINDOW_XID(window);
+ }
+ }
+ }
+#endif
+ return NULL;
+}
+
static void invalidate(SpiceChannel *channel,
gint x, gint y, gint w, gint h, gpointer data)
{
@@ -2962,6 +2993,8 @@ static void channel_new(SpiceSession *s, SpiceChannel *channel, gpointer data)
spice_g_signal_connect_object(channel, "notify::monitors",
G_CALLBACK(spice_display_widget_update_monitor_area),
display, G_CONNECT_AFTER | G_CONNECT_SWAPPED);
+ spice_g_signal_connect_object(channel, "streaming-mode",
+ G_CALLBACK(prepare_streaming_mode), display, G_CONNECT_AFTER);
if (spice_display_channel_get_primary(channel, 0, &primary)) {
primary_create(channel, primary.format, primary.width, primary.height,
primary.stride, primary.shmid, primary.data, display);
--
2.14.3
More information about the Spice-devel
mailing list