[Spice-devel] [PATCH spice-gtk 9/11] Add the ability to use gstreamer for vp8 decoding.
Jeremy White
jwhite at codeweavers.com
Wed May 13 13:31:34 PDT 2015
Signed-off-by: Jeremy White <jwhite at codeweavers.com>
---
configure.ac | 21 ++--
gtk/Makefile.am | 12 ++-
gtk/channel-display-gst.c | 259 +++++++++++++++++++++++++++++++++++++++++++++
gtk/channel-display-priv.h | 13 +++
gtk/channel-display.c | 11 ++
5 files changed, 307 insertions(+), 9 deletions(-)
create mode 100644 gtk/channel-display-gst.c
diff --git a/configure.ac b/configure.ac
index cf5a039..3c89787 100644
--- a/configure.ac
+++ b/configure.ac
@@ -321,18 +321,25 @@ AC_SUBST(PULSE_CFLAGS)
AC_SUBST(PULSE_LIBS)
AS_IF([test "x$with_audio" = "xgstreamer"],
- [PKG_CHECK_MODULES(GST, gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-audio-1.0, [have_gst=yes], [have_gst=no])],
- [have_gst=no])
+ [PKG_CHECK_MODULES(GSTAUDIO, gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-audio-1.0, [have_gst_audio=yes], [have_gst_audio=no])],
+ [have_gst_audio=no])
-AS_IF([test "x$have_gst" = "xyes"],
+AS_IF([test "x$have_gst_audio" = "xyes"],
[AC_DEFINE([WITH_GSTAUDIO], 1, [Have GStreamer 1.0?])],
[AS_IF([test "x$with_audio" = "xgstreamer"],
[AC_MSG_ERROR([GStreamer 1.0 requested but not found])
])
])
-AM_CONDITIONAL([WITH_GSTAUDIO], [test "x$have_gst" = "xyes"])
-AC_SUBST(GST_CFLAGS)
-AC_SUBST(GST_LIBS)
+AM_CONDITIONAL([WITH_GSTAUDIO], [test "x$have_gst_audio" = "xyes"])
+AC_SUBST(GSTAUDIO_CFLAGS)
+AC_SUBST(GSTAUDIO_LIBS)
+
+PKG_CHECK_MODULES(GSTVIDEO, gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0, [have_gst_video=yes], [have_gst_video=no])
+AC_SUBST(GSTVIDEO_CFLAGS)
+AC_SUBST(GSTVIDEO_LIBS)
+AS_IF([test "x$have_gst_video" = "xyes"], [AC_DEFINE([HAVE_GSTVIDEO], 1, [Have Gstreamer Video?])])
+AM_CONDITIONAL([HAVE_GSTVIDEO],[test "$have_gst_video" != "no"])
+
AC_CHECK_LIB(jpeg, jpeg_destroy_decompress,
AC_MSG_CHECKING([for jpeglib.h])
@@ -724,7 +731,7 @@ SPICE_CFLAGS="$SPICE_CFLAGS $WARN_CFLAGS"
AC_SUBST(SPICE_CFLAGS)
-SPICE_GLIB_CFLAGS="$PROTOCOL_CFLAGS $PIXMAN_CFLAGS $PULSE_CFLAGS $GST_CFLAGS $GLIB2_CFLAGS $GIO_CFLAGS $GOBJECT2_CFLAGS $SSL_CFLAGS $SASL_CFLAGS"
+SPICE_GLIB_CFLAGS="$PROTOCOL_CFLAGS $PIXMAN_CFLAGS $PULSE_CFLAGS $GSTVIDEO_CFLAGS $GSTAUDIO_CFLAGS $GLIB2_CFLAGS $GIO_CFLAGS $GOBJECT2_CFLAGS $SSL_CFLAGS $SASL_CFLAGS"
SPICE_GTK_CFLAGS="$SPICE_GLIB_CFLAGS $GTK_CFLAGS "
AC_SUBST(SPICE_GLIB_CFLAGS)
diff --git a/gtk/Makefile.am b/gtk/Makefile.am
index e1d5d93..aa0519c 100644
--- a/gtk/Makefile.am
+++ b/gtk/Makefile.am
@@ -99,7 +99,8 @@ SPICE_COMMON_CPPFLAGS = \
$(DBUS_GLIB_CFLAGS) \
$(SSL_CFLAGS) \
$(SASL_CFLAGS) \
- $(GST_CFLAGS) \
+ $(GSTAUDIO_CFLAGS) \
+ $(GSTVIDEO_CFLAGS) \
$(SMARTCARD_CFLAGS) \
$(USBREDIR_CFLAGS) \
$(GUDEV_CFLAGS) \
@@ -213,7 +214,8 @@ libspice_client_glib_2_0_la_LIBADD = \
$(PIXMAN_LIBS) \
$(SSL_LIBS) \
$(PULSE_LIBS) \
- $(GST_LIBS) \
+ $(GSTAUDIO_LIBS) \
+ $(GSTVIDEO_LIBS) \
$(SASL_LIBS) \
$(SMARTCARD_LIBS) \
$(USBREDIR_LIBS) \
@@ -346,6 +348,12 @@ libspice_client_glib_2_0_la_SOURCES += \
$(NULL)
endif
+if HAVE_GSTVIDEO
+libspice_client_glib_2_0_la_SOURCES += \
+ channel-display-gst.c \
+ $(NULL)
+endif
+
if WITH_PHODAV
libspice_client_glib_2_0_la_SOURCES += \
giopipe.c \
diff --git a/gtk/channel-display-gst.c b/gtk/channel-display-gst.c
new file mode 100644
index 0000000..f368815
--- /dev/null
+++ b/gtk/channel-display-gst.c
@@ -0,0 +1,259 @@
+/* -*- Mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ Copyright (C) 2015 CodeWeavers, Inc
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "config.h"
+
+#include "spice-client.h"
+#include "spice-common.h"
+#include "spice-channel-priv.h"
+
+#include "channel-display-priv.h"
+
+#include <gst/gst.h>
+#include <gst/app/gstappsrc.h>
+#include <gst/app/gstappsink.h>
+
+struct GstreamerDecoder {
+ GstElement *pipeline;
+ GstElement *appsource;
+ GstElement *appsink;
+ GstElement *codec;
+ GstElement *convert;
+};
+
+static gboolean drain_pipeline(GstreamerDecoder *dec)
+{
+ GstSample *sample = NULL;
+ if (gst_app_sink_is_eos(GST_APP_SINK(dec->appsink)))
+ return TRUE;
+
+ do {
+ if (sample)
+ gst_sample_unref(sample);
+ sample = gst_app_sink_pull_sample(GST_APP_SINK(dec->appsink));
+ } while(sample);
+
+ return gst_app_sink_is_eos(GST_APP_SINK(dec->appsink));
+}
+
+static int construct_pipeline(display_stream *st, GstreamerDecoder *dec)
+{
+ int ret;
+ GstCaps *rgb;
+ char *dec_name;
+
+ if (st->codec == SPICE_VIDEO_CODEC_TYPE_VP8)
+ dec_name = "vp8dec";
+ else if (st->codec == SPICE_VIDEO_CODEC_TYPE_MJPEG)
+ dec_name = "jpegdec";
+ else {
+ SPICE_DEBUG("Uknown codec type %d", st->codec);
+ return -1;
+ }
+
+ if (dec->pipeline) {
+ GstPad *pad = gst_element_get_static_pad(dec->codec, "sink");
+ if (! pad) {
+ SPICE_DEBUG("Unable to get codec sink pad to flush the pipe");
+ return -1;
+ }
+ gst_pad_send_event(pad, gst_event_new_eos());
+
+ gst_object_unref(pad);
+ if (! drain_pipeline(dec))
+ return -1;
+
+ gst_bin_remove_many(GST_BIN(dec->pipeline), dec->appsource, dec->codec, dec->appsink, NULL);
+ gst_object_unref(dec->pipeline);
+ }
+
+ dec->appsource = gst_element_factory_make ("appsrc", NULL);
+ dec->appsink = gst_element_factory_make ("appsink", NULL);
+
+ dec->codec = gst_element_factory_make (dec_name, NULL);
+ dec->convert = gst_element_factory_make ("videoconvert", NULL);
+
+ rgb = gst_caps_from_string("video/x-raw,format=BGRx");
+ if (!rgb) {
+ SPICE_DEBUG("Gstreamer error: could not make BGRx caps.");
+ return -1;
+ }
+
+ if (st->codec == SPICE_VIDEO_CODEC_TYPE_VP8) {
+ GstCaps *vp8;
+ vp8 = gst_caps_from_string("video/x-vp8");
+ if (!vp8) {
+ SPICE_DEBUG("Gstreamer error: could not make vp8 caps.");
+ return -1;
+ }
+
+ gst_app_src_set_caps(GST_APP_SRC(dec->appsource), vp8);
+ gst_app_sink_set_caps(GST_APP_SINK(dec->appsink), rgb);
+ }
+
+ dec->pipeline = gst_pipeline_new ("pipeline");
+
+ if (!dec->pipeline || !dec->appsource || !dec->appsink || !dec->codec || !dec->convert) {
+ SPICE_DEBUG("Gstreamer error: not all elements could be created.");
+ return -1;
+ }
+
+ gst_bin_add_many (GST_BIN(dec->pipeline), dec->appsource, dec->codec, dec->convert, dec->appsink, NULL);
+ if (gst_element_link_many(dec->appsource, dec->codec, dec->convert, dec->appsink, NULL) != TRUE) {
+ SPICE_DEBUG("Gstreamer error: could not link all the pieces.");
+ gst_object_unref (dec->pipeline);
+ return -1;
+ }
+
+ ret = gst_element_set_state (dec->pipeline, GST_STATE_PLAYING);
+ if (ret == GST_STATE_CHANGE_FAILURE) {
+ SPICE_DEBUG("Gstreamer error: Unable to set the pipeline to the playing state.");
+ gst_object_unref (dec->pipeline);
+ return -1;
+ }
+
+ return 0;
+}
+
+static GstreamerDecoder *gst_decoder_new(display_stream *st)
+{
+ GstreamerDecoder *dec;
+
+ gst_init(NULL, NULL);
+
+ dec = g_malloc0(sizeof(*dec));
+ if (construct_pipeline(st, dec)) {
+ free(dec);
+ return NULL;
+ }
+
+ return dec;
+}
+
+
+void gst_decoder_destroy(GstreamerDecoder *dec)
+{
+ gst_object_unref(dec->pipeline);
+ // TODO - contemplate calling gst_deinit();
+ dec->pipeline = NULL;
+ free(dec);
+}
+
+
+gboolean push_frame(display_stream *st)
+{
+ uint8_t *data;
+ uint32_t size;
+ gpointer d;
+ GstBuffer *buffer;
+
+ size = stream_get_current_frame(st, &data);
+ if (size == 0)
+ return false;
+
+ // TODO. Grr. Seems like a wasted alloc
+ d = g_malloc(size);
+ memcpy(d, data, size);
+
+ buffer = gst_buffer_new_wrapped(d, size);
+
+ if (gst_app_src_push_buffer(GST_APP_SRC(st->gst_dec->appsource), buffer) != GST_FLOW_OK) {
+ SPICE_DEBUG("Error: unable to push frame of size %d", size);
+ return false;
+ }
+
+ // TODO. Unref buffer?
+
+ return true;
+}
+
+
+static void pull_frame(display_stream *st)
+{
+ int width;
+ int height;
+
+ GstSample *sample;
+ GstBuffer *buffer = NULL;
+ GstCaps *caps;
+ GstStructure *structure;
+ GstMemory *memory;
+ gint caps_width, caps_height;
+
+ sample = gst_app_sink_pull_sample(GST_APP_SINK(st->gst_dec->appsink));
+ if (! sample) {
+ SPICE_DEBUG("Unable to pull sample");
+ return;
+ }
+
+ buffer = gst_sample_get_buffer(sample);
+ memory = gst_buffer_get_all_memory(buffer);
+ caps = gst_sample_get_caps(sample);
+
+ structure = gst_caps_get_structure (caps, 0);
+ gst_structure_get_int(structure, "width", &caps_width);
+ gst_structure_get_int(structure, "height", &caps_height);
+
+ stream_get_dimensions(st, &width, &height);
+
+ if (width != caps_width || height != caps_height) {
+ SPICE_DEBUG("Stream size %dx%x does not match frame size %dx%d",
+ width, height, caps_width, caps_height);
+ }
+ else {
+ GstMapInfo mem_info;
+
+ // TODO seems like poor memory management
+ if (gst_memory_map(memory, &mem_info, GST_MAP_READ)) {
+ st->out_frame = g_malloc0(mem_info.size);
+ memcpy(st->out_frame, mem_info.data, mem_info.size);
+
+ gst_memory_unmap(memory, &mem_info);
+ }
+ }
+
+ gst_memory_unref(memory);
+ gst_sample_unref(sample);
+}
+
+
+G_GNUC_INTERNAL
+void stream_gst_init(display_stream *st)
+{
+ st->gst_dec = gst_decoder_new(st);
+}
+
+G_GNUC_INTERNAL
+void stream_gst_data(display_stream *st)
+{
+ if (! push_frame(st))
+ return;
+
+ pull_frame(st);
+}
+
+G_GNUC_INTERNAL
+void stream_gst_cleanup(display_stream *st)
+{
+ g_free(st->out_frame);
+ st->out_frame = NULL;
+ if (st->gst_dec) {
+ gst_decoder_destroy(st->gst_dec);
+ st->gst_dec = NULL;
+ }
+}
+
diff --git a/gtk/channel-display-priv.h b/gtk/channel-display-priv.h
index 71f5d17..853dd92 100644
--- a/gtk/channel-display-priv.h
+++ b/gtk/channel-display-priv.h
@@ -34,6 +34,9 @@
G_BEGIN_DECLS
+#if defined(HAVE_GSTVIDEO)
+typedef struct GstreamerDecoder GstreamerDecoder;
+#endif
typedef struct display_surface {
guint32 surface_id;
@@ -71,6 +74,11 @@ typedef struct display_stream {
struct jpeg_decompress_struct mjpeg_cinfo;
struct jpeg_error_mgr mjpeg_jerr;
+#if defined(HAVE_GSTVIDEO)
+ /* gstreamer decoder */
+ struct GstreamerDecoder *gst_dec;
+#endif
+
uint8_t *out_frame;
GQueue *msgq;
guint timeout;
@@ -108,6 +116,11 @@ void stream_mjpeg_init(display_stream *st);
void stream_mjpeg_data(display_stream *st);
void stream_mjpeg_cleanup(display_stream *st);
+/* channel-display-gst.c */
+void stream_gst_init(display_stream *st);
+void stream_gst_data(display_stream *st);
+void stream_gst_cleanup(display_stream *st);
+
G_END_DECLS
#endif // CHANNEL_DISPLAY_PRIV_H_
diff --git a/gtk/channel-display.c b/gtk/channel-display.c
index efe2259..260dde2 100644
--- a/gtk/channel-display.c
+++ b/gtk/channel-display.c
@@ -596,6 +596,9 @@ static void spice_display_channel_reset_capabilities(SpiceChannel *channel)
if (SPICE_DISPLAY_CHANNEL(channel)->priv->enable_adaptive_streaming) {
spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_STREAM_REPORT);
}
+ spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_MULTI_CODEC);
+ spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_CODEC_MJPEG);
+ spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_CODEC_VP8);
}
static void destroy_surface(gpointer data)
@@ -1000,6 +1003,8 @@ static void display_handle_stream_create(SpiceChannel *channel, SpiceMsgIn *in)
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
stream_mjpeg_init(st);
break;
+ default:
+ stream_gst_init(st);
}
}
@@ -1124,6 +1129,9 @@ static gboolean display_stream_render(display_stream *st)
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
stream_mjpeg_data(st);
break;
+ default:
+ stream_gst_data(st);
+ break;
}
if (st->out_frame) {
@@ -1472,6 +1480,9 @@ static void destroy_stream(SpiceChannel *channel, int id)
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
stream_mjpeg_cleanup(st);
break;
+ default:
+ stream_gst_cleanup(st);
+ break;
}
if (st->msg_clip)
--
2.1.4
More information about the Spice-devel
mailing list