[Spice-devel] [RFC spice-gtk v2 1/1] Gstreamer: Use GstVideoOverlay if possible
Snir Sheriber
ssheribe at redhat.com
Sun Apr 15 09:10:29 UTC 2018
Currently when gstreamer is used to decode a full-screen
stream sent from the server, the decoding frames are being
forced to RBGA format and pushed using appsink to be scaled
and rendered to screen.
Today most of the gstreamer sinks supports the GstVideoOverlay
interface which allows to render directly from the pipeline to
a window by a given windowing system id (i.e. xid). This patch
makes playbin to use this feature if possible.
Setting the DISABLE_GSTVIDEOOVERLAY environment variable will
make gstreamer to avoid of using the gstvideooverlay interface.
---
src/channel-display-gst.c | 99 ++++++++++++++++++++++++++++++++++++++---------
src/channel-display.c | 55 ++++++++++++++++++++++++++
src/channel-display.h | 3 ++
src/spice-widget-priv.h | 1 +
src/spice-widget.c | 40 ++++++++++++++++++-
5 files changed, 179 insertions(+), 19 deletions(-)
diff --git a/src/channel-display-gst.c b/src/channel-display-gst.c
index 8b23036..a29af3f 100644
--- a/src/channel-display-gst.c
+++ b/src/channel-display-gst.c
@@ -49,6 +49,8 @@ typedef struct SpiceGstDecoder {
GQueue *decoding_queue;
GQueue *display_queue;
guint timer_id;
+
+ gboolean gstvideooverlay;
} SpiceGstDecoder;
#define VALID_VIDEO_CODEC_TYPE(codec) \
@@ -89,6 +91,12 @@ static SpiceGstFrame *create_gst_frame(GstBuffer *buffer, SpiceFrame *frame)
return gstframe;
}
+static void free_spice_frame(SpiceFrame *frame)
+{
+ frame->unref_data(frame->data_opaque);
+ frame->free(frame);
+}
+
static void free_gst_frame(SpiceGstFrame *gstframe)
{
gstframe->frame->free(gstframe->frame);
@@ -265,7 +273,8 @@ static void free_pipeline(SpiceGstDecoder *decoder)
gst_element_set_state(decoder->pipeline, GST_STATE_NULL);
gst_object_unref(decoder->appsrc);
- gst_object_unref(decoder->appsink);
+ if (decoder->appsink)
+ gst_object_unref(decoder->appsink);
gst_object_unref(decoder->pipeline);
gst_object_unref(decoder->clock);
decoder->pipeline = NULL;
@@ -306,6 +315,20 @@ static gboolean handle_pipeline_message(GstBus *bus, GstMessage *msg, gpointer v
g_free(filename);
break;
}
+ case GST_MESSAGE_ELEMENT: {
+ if (gst_is_video_overlay_prepare_window_handle_message(msg)) {
+ GstVideoOverlay *overlay;
+ guintptr handle = 0;
+
+ g_object_get(decoder->base.stream->channel, "handle", &handle, NULL);
+ SPICE_DEBUG("prepare-window-handle msg received (handle: %lu)", handle);
+ if (handle != 0) {
+ overlay = GST_VIDEO_OVERLAY(GST_MESSAGE_SRC(msg));
+ gst_video_overlay_set_window_handle(overlay, handle);
+ }
+ }
+ break;
+ }
default:
/* not being handled */
break;
@@ -347,6 +370,7 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
#if GST_CHECK_VERSION(1,9,0)
GstElement *playbin, *sink;
SpiceGstPlayFlags flags;
+ guintptr handle;
GstCaps *caps;
playbin = gst_element_factory_make("playbin", "playbin");
@@ -355,26 +379,53 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
return FALSE;
}
- sink = gst_element_factory_make("appsink", "sink");
- if (sink == NULL) {
- spice_warning("error upon creation of 'appsink' element");
- gst_object_unref(playbin);
- return FALSE;
- }
-
- caps = gst_caps_from_string("video/x-raw,format=BGRx");
- g_object_set(sink,
+ g_object_get(decoder->base.stream->channel, "handle", &handle, NULL);
+ decoder->gstvideooverlay = (handle != 0);
+ SPICE_DEBUG("Creating Gstreamer pipline (handle for overlay %s)\n",
+ handle ? "received" : "not received");
+ if (handle == 0) {
+ sink = gst_element_factory_make("appsink", "sink");
+ if (sink == NULL) {
+ spice_warning("error upon creation of 'appsink' element");
+ gst_object_unref(playbin);
+ return FALSE;
+ }
+ caps = gst_caps_from_string("video/x-raw,format=BGRx");
+ g_object_set(sink,
"caps", caps,
"sync", FALSE,
"drop", FALSE,
NULL);
- gst_caps_unref(caps);
+ gst_caps_unref(caps);
+ g_object_set(playbin,
+ "video-sink", gst_object_ref(sink),
+ NULL);
+
+ decoder->appsink = GST_APP_SINK(sink);
+ } else {
+ /* handle has received, it means playbin will render directly into
+ * widget using the gstvideoooverlay interface instead of app-sink.
+ * Also avoid using vaapisink if exist since vaapisink could be
+ * buggy when it is combined with playbin. changing its rank to
+ * none will make playbin to avoid of using it.
+ */
+ GstRegistry *registry = NULL;
+ GstPluginFeature *vaapisink = NULL;
+
+ registry = gst_registry_get();
+ if (registry) {
+ vaapisink = gst_registry_lookup_feature(registry, "vaapisink");
+ }
+ if (vaapisink) {
+ gst_plugin_feature_set_rank(vaapisink, GST_RANK_NONE);
+ gst_object_unref(vaapisink);
+ }
+ }
g_signal_connect(playbin, "source-setup", G_CALLBACK(app_source_setup), decoder);
g_object_set(playbin,
"uri", "appsrc://",
- "video-sink", gst_object_ref(sink),
NULL);
/* Disable audio in playbin */
@@ -383,7 +434,6 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
g_object_set(playbin, "flags", flags, NULL);
g_warn_if_fail(decoder->appsrc == NULL);
- decoder->appsink = GST_APP_SINK(sink);
decoder->pipeline = playbin;
#else
gchar *desc;
@@ -416,7 +466,9 @@ static gboolean create_pipeline(SpiceGstDecoder *decoder)
#endif
appsink_cbs.new_sample = new_sample;
- gst_app_sink_set_callbacks(decoder->appsink, &appsink_cbs, decoder, NULL);
+ if (decoder->appsink) {
+ gst_app_sink_set_callbacks(decoder->appsink, &appsink_cbs, decoder, NULL);
+ }
bus = gst_pipeline_get_bus(GST_PIPELINE(decoder->pipeline));
gst_bus_add_watch(bus, handle_pipeline_message, decoder);
gst_object_unref(bus);
@@ -510,6 +562,8 @@ static gboolean spice_gst_decoder_queue_frame(VideoDecoder *video_decoder,
SpiceFrame *frame, int latency)
{
SpiceGstDecoder *decoder = (SpiceGstDecoder*)video_decoder;
+ gpointer data_opaque;
+ GDestroyNotify data_unref;
if (frame->size == 0) {
SPICE_DEBUG("got an empty frame buffer!");
@@ -551,17 +605,26 @@ static gboolean spice_gst_decoder_queue_frame(VideoDecoder *video_decoder,
/* ref() the frame data for the buffer */
frame->ref_data(frame->data_opaque);
+ if (decoder->gstvideooverlay) {
+ data_opaque = frame;
+ data_unref = (void*)free_spice_frame;
+ } else {
+ data_opaque = frame->data_opaque;
+ data_unref = frame->unref_data;
+ }
GstBuffer *buffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_PHYSICALLY_CONTIGUOUS,
frame->data, frame->size, 0, frame->size,
- frame->data_opaque, frame->unref_data);
+ data_opaque, data_unref);
GST_BUFFER_DURATION(buffer) = GST_CLOCK_TIME_NONE;
GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE;
GST_BUFFER_PTS(buffer) = gst_clock_get_time(decoder->clock) - gst_element_get_base_time(decoder->pipeline) + ((uint64_t)MAX(0, latency)) * 1000 * 1000;
- g_mutex_lock(&decoder->queues_mutex);
- g_queue_push_tail(decoder->decoding_queue, create_gst_frame(buffer, frame));
- g_mutex_unlock(&decoder->queues_mutex);
+ if (!decoder->gstvideooverlay) {
+ g_mutex_lock(&decoder->queues_mutex);
+ g_queue_push_tail(decoder->decoding_queue, create_gst_frame(buffer, frame));
+ g_mutex_unlock(&decoder->queues_mutex);
+ }
if (gst_app_src_push_buffer(decoder->appsrc, buffer) != GST_FLOW_OK) {
SPICE_DEBUG("GStreamer error: unable to push frame of size %u", frame->size);
diff --git a/src/channel-display.c b/src/channel-display.c
index d0d977f..ac040bc 100644
--- a/src/channel-display.c
+++ b/src/channel-display.c
@@ -70,6 +70,7 @@ struct _SpiceDisplayChannelPrivate {
GArray *monitors;
guint monitors_max;
gboolean enable_adaptive_streaming;
+ gpointer handle;
SpiceGlScanout scanout;
};
@@ -83,6 +84,7 @@ enum {
PROP_MONITORS,
PROP_MONITORS_MAX,
PROP_GL_SCANOUT,
+ PROP_HANDLE,
};
enum {
@@ -91,6 +93,7 @@ enum {
SPICE_DISPLAY_INVALIDATE,
SPICE_DISPLAY_MARK,
SPICE_DISPLAY_GL_DRAW,
+ SPICE_DISPLAY_STREAMING_MODE,
SPICE_DISPLAY_LAST_SIGNAL,
};
@@ -227,6 +230,10 @@ static void spice_display_get_property(GObject *object,
g_value_set_static_boxed(value, spice_display_channel_get_gl_scanout(channel));
break;
}
+ case PROP_HANDLE: {
+ g_value_set_pointer(value, c->handle);
+ break;
+ }
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
@@ -238,7 +245,14 @@ static void spice_display_set_property(GObject *object,
const GValue *value,
GParamSpec *pspec)
{
+ SpiceDisplayChannel *channel = SPICE_DISPLAY_CHANNEL(object);
+ SpiceDisplayChannelPrivate *c = channel->priv;
+
switch (prop_id) {
+ case PROP_HANDLE: {
+ c->handle = g_value_get_pointer(value);
+ break;
+ }
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
@@ -338,6 +352,22 @@ static void spice_display_channel_class_init(SpiceDisplayChannelClass *klass)
G_PARAM_READABLE |
G_PARAM_STATIC_STRINGS));
+ /**
+ * SpiceDisplayChannel:handle:
+ *
+ * The handle for current window.
+ *
+ * Since: 0.35
+ */
+ g_object_class_install_property
+ (gobject_class, PROP_HANDLE,
+ g_param_spec_pointer("handle",
+ "Display handle",
+ "Display handle",
+ G_PARAM_WRITABLE |
+ G_PARAM_READABLE |
+ G_PARAM_STATIC_STRINGS));
+
/**
* SpiceDisplayChannel::display-primary-create:
* @display: the #SpiceDisplayChannel that emitted the signal
@@ -454,6 +484,28 @@ static void spice_display_channel_class_init(SpiceDisplayChannelClass *klass)
4,
G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT);
+ /**
+ * SpiceDisplayChannel::streaming-mode:
+ * @display: the #SpiceDisplayChannel that emitted the signal
+ * @stream_enable: %TRUE when it's streaming mode
+ *
+ * The #SpiceDisplayChannel::streaming_mode signal is emitted when
+ * spice server is working in streaming mode.
+ *
+ * Since 0.35
+ **/
+ signals[SPICE_DISPLAY_STREAMING_MODE] =
+ g_signal_new("streaming-mode",
+ G_OBJECT_CLASS_TYPE(gobject_class),
+ G_SIGNAL_RUN_FIRST,
+ G_STRUCT_OFFSET(SpiceDisplayChannelClass,
+ streaming_mode),
+ NULL, NULL,
+ g_cclosure_marshal_VOID__INT,
+ G_TYPE_NONE,
+ 1,
+ G_TYPE_INT);
+
g_type_class_add_private(klass, sizeof(SpiceDisplayChannelPrivate));
channel_set_handlers(SPICE_CHANNEL_CLASS(klass));
@@ -1779,6 +1831,9 @@ static void display_handle_surface_create(SpiceChannel *channel, SpiceMsgIn *in)
if (create->flags & SPICE_SURFACE_FLAGS_PRIMARY) {
SPICE_DEBUG("primary flags: %x", create->flags);
surface->primary = true;
+ if (create->flags & SPICE_SURFACE_FLAGS_STREAMING_MODE) {
+ g_coroutine_signal_emit(channel, signals[SPICE_DISPLAY_STREAMING_MODE], 0, TRUE);
+ }
create_canvas(channel, surface);
if (c->mark_false_event_id != 0) {
g_source_remove(c->mark_false_event_id);
diff --git a/src/channel-display.h b/src/channel-display.h
index 5b48d2f..9c51aa2 100644
--- a/src/channel-display.h
+++ b/src/channel-display.h
@@ -125,6 +125,7 @@ struct _SpiceDisplayChannel {
* @display_primary_destroy: Signal class handler for the #SpiceDisplayChannel::display-primary-destroy signal.
* @display_invalidate: Signal class handler for the #SpiceDisplayChannel::display-invalidate signal.
* @display_mark: Signal class handler for the #SpiceDisplayChannel::display-mark signal.
+ * @streaming_mode: Signal class handler for the #SpiceDisplayChannel::streaming-mode signal.
*
* Class structure for #SpiceDisplayChannel.
*/
@@ -140,6 +141,8 @@ struct _SpiceDisplayChannelClass {
gint x, gint y, gint w, gint h);
void (*display_mark)(SpiceChannel *channel,
gboolean mark);
+ void (*streaming_mode)(SpiceChannel *channel,
+ gboolean streaming_mode);
/*< private >*/
};
diff --git a/src/spice-widget-priv.h b/src/spice-widget-priv.h
index 1189cbb..9930bab 100644
--- a/src/spice-widget-priv.h
+++ b/src/spice-widget-priv.h
@@ -70,6 +70,7 @@ struct _SpiceDisplayPrivate {
bool resize_guest_enable;
/* state */
+ gboolean streaming_mode;
gboolean ready;
gboolean monitor_ready;
struct {
diff --git a/src/spice-widget.c b/src/spice-widget.c
index 8a6b5ab..abfbb32 100644
--- a/src/spice-widget.c
+++ b/src/spice-widget.c
@@ -615,14 +615,42 @@ G_GNUC_END_IGNORE_DEPRECATIONS
static void
drawing_area_realize(GtkWidget *area, gpointer user_data)
{
-#if defined(GDK_WINDOWING_X11) && defined(HAVE_EGL)
+#ifdef GDK_WINDOWING_X11
SpiceDisplay *display = SPICE_DISPLAY(user_data);
+ SpiceDisplayPrivate *d = display->priv;
+
+ /* GstVideoOverlay will currently be used only under x */
+ if (d->streaming_mode &&
+ !g_getenv("DISABLE_GSTVIDEOOVERLAY") &&
+ GDK_IS_X11_DISPLAY(gdk_display_get_default())) {
+ GdkWindow *window;
+ GtkWidget *area;
+
+ window = gtk_widget_get_window(GTK_WIDGET(display));
+ if (window) {
+#if GTK_CHECK_VERSION(2,18,0)
+ if (gdk_window_ensure_native (window))
+#endif
+ {
+ g_object_set(G_OBJECT (d->display),
+ "handle", GDK_WINDOW_XID(window),
+ NULL);
+ area = gtk_stack_get_visible_child(d->stack);
+ g_object_disconnect(area,
+ "any_signal::draw", draw_event, display, // to avoid spice drawing
+ NULL);
+ return;
+ }
+ }
+ }
+#ifdef HAVE_EGL
if (GDK_IS_X11_DISPLAY(gdk_display_get_default()) &&
spice_display_channel_get_gl_scanout(display->priv->display) != NULL) {
spice_display_widget_gl_scanout(display);
}
#endif
+#endif
}
static void spice_display_init(SpiceDisplay *display)
@@ -2589,6 +2617,14 @@ static void queue_draw_area(SpiceDisplay *display, gint x, gint y,
x, y, width, height);
}
+static void set_streaming_mode(SpiceChannel *channel, guint streaming, gpointer data)
+{
+ SpiceDisplay *display = data;
+ SpiceDisplayPrivate *d = display->priv;
+
+ d->streaming_mode = streaming;
+}
+
static void invalidate(SpiceChannel *channel,
gint x, gint y, gint w, gint h, gpointer data)
{
@@ -2957,6 +2993,8 @@ static void channel_new(SpiceSession *s, SpiceChannel *channel, gpointer data)
spice_g_signal_connect_object(channel, "notify::monitors",
G_CALLBACK(spice_display_widget_update_monitor_area),
display, G_CONNECT_AFTER | G_CONNECT_SWAPPED);
+ spice_g_signal_connect_object(channel, "streaming-mode",
+ G_CALLBACK(set_streaming_mode), display, 0);
if (spice_display_channel_get_primary(channel, 0, &primary)) {
primary_create(channel, primary.format, primary.width, primary.height,
primary.stride, primary.shmid, primary.data, display);
--
2.14.3
More information about the Spice-devel
mailing list