[Spice-commits] 23 commits - configure.ac server/Makefile.am server/dcc-send.c server/dcc.c server/dcc.h server/display-channel.c server/display-channel.h server/gstreamer-encoder.c server/mjpeg-encoder.c server/red-qxl.c server/red-qxl.h server/red-worker.c server/reds.c server/reds.h server/spice-server.h server/spice-server.syms server/stream.c server/stream.h server/tests server/video-encoder.h
Christophe Fergau
teuf at kemper.freedesktop.org
Tue Jun 14 15:18:53 UTC 2016
configure.ac | 45 +
server/Makefile.am | 16
server/dcc-send.c | 76 -
server/dcc.c | 5
server/dcc.h | 3
server/display-channel.c | 12
server/display-channel.h | 5
server/gstreamer-encoder.c | 1723 +++++++++++++++++++++++++++++++++++++++++++++
server/mjpeg-encoder.c | 110 +-
server/red-qxl.c | 9
server/red-qxl.h | 6
server/red-worker.c | 18
server/reds.c | 160 +++-
server/reds.h | 1
server/spice-server.h | 8
server/spice-server.syms | 5
server/stream.c | 151 ++-
server/stream.h | 8
server/tests/replay.c | 11
server/video-encoder.h | 73 +
20 files changed, 2255 insertions(+), 190 deletions(-)
New commits:
commit 0503cd3d61f39008a81bae96a6f0a6081caf85ff
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Tue May 3 17:05:55 2016 +0200
streaming: Add support for GStreamer 0.10
configure will use GStreamer 1.0 if present and fall back to
GStreamer 0.10 otherwise.
ffenc_mjpeg takes its bitrate as a long so extend set_gstenc_bitrate().
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/configure.ac b/configure.ac
index 1c8b0d7..4b01a32 100644
--- a/configure.ac
+++ b/configure.ac
@@ -69,28 +69,48 @@ dnl Check optional features
SPICE_CHECK_SMARTCARD
AC_ARG_ENABLE(gstreamer,
- AS_HELP_STRING([--enable-gstreamer=@<:@auto/yes/no@:>@],
- [Enable GStreamer 1.0 support]),,
+ AS_HELP_STRING([--enable-gstreamer=@<:@auto/0.10/1.0/yes/no@:>@],
+ [Enable GStreamer support]),,
[enable_gstreamer="auto"])
-if test "x$enable_gstreamer" != "xno"; then
+if test "x$enable_gstreamer" != "xno" && test "x$enable_gstreamer" != "x0.10"; then
SPICE_CHECK_GSTREAMER(GSTREAMER_1_0, 1.0, [gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0],
- [enable_gstreamer="yes"
+ [enable_gstreamer="1.0"
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-base 1.0], [appsrc videoconvert appsink])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gstreamer-libav 1.0], [avenc_mjpeg])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-good 1.0], [vp8enc])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-ugly 1.0], [x264enc])
],
- [if test "x$enable_gstreamer" = "xyes"; then
+ [if test "x$enable_gstreamer" = "x1.0"; then
AC_MSG_ERROR([GStreamer 1.0 support requested but not found. You may set GSTREAMER_1_0_CFLAGS and GSTREAMER_1_0_LIBS to avoid the need to call pkg-config.])
fi
])
fi
AM_CONDITIONAL(HAVE_GSTREAMER_1_0, test "x$have_gstreamer_1_0" = "xyes")
-if test x"$gstreamer_missing" != x; then
- SPICE_WARNING([The following GStreamer $enable_gstreamer tools/elements are missing:$gstreamer_missing. The GStreamer video encoder can be built but may not work.])
+if test "x$enable_gstreamer" != "xno" && test "x$enable_gstreamer" != "x1.0"; then
+ SPICE_CHECK_GSTREAMER(GSTREAMER_0_10, 0.10, [gstreamer-0.10 gstreamer-base-0.10 gstreamer-app-0.10 gstreamer-video-0.10],
+ [enable_gstreamer="0.10"
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gst-plugins-base 0.10], [appsrc appsink])
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gstreamer-ffmpeg 0.10], [ffmpegcolorspace ffenc_mjpeg])
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gst-plugins-bad 0.10], [vp8enc])
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_0_10, [gst-plugins-ugly 0.10], [x264enc])
+ ],
+ [if test "x$enable_gstreamer" = "x0.10"; then
+ AC_MSG_ERROR([GStreamer 0.10 support requested but not found. You may set GSTREAMER_0_10_CFLAGS and GSTREAMER_0_10_LIBS to avoid the need to call pkg-config.])
+ fi
+ ])
fi
+AM_CONDITIONAL(HAVE_GSTREAMER_0_10, test "x$have_gstreamer_0_10" = "xyes")
+
+AS_IF([test "x$enable_gstreamer" = "xyes"],
+ [AC_MSG_ERROR("GStreamer support requested but not found")],
+ [test "x$enable_gstreamer" = "xauto"],
+ [enable_gstreamer="no"
+])
+AS_IF([test x"$missing_gstreamer_elements" = xyes],
+ [SPICE_WARNING([The GStreamer video encoder can be built but may not work.])
+])
AC_ARG_ENABLE([automated_tests],
AS_HELP_STRING([--enable-automated-tests], [Enable automated tests using spicy-screenshot (part of spice-gtk)]),,
@@ -264,7 +284,7 @@ AC_MSG_NOTICE([
LZ4 support: ${enable_lz4}
Smartcard: ${have_smartcard}
- GStreamer 1.0: ${have_gstreamer_1_0}
+ GStreamer: ${enable_gstreamer}
SASL support: ${have_sasl}
Automated tests: ${enable_automated_tests}
Manual: ${have_asciidoc}
diff --git a/server/Makefile.am b/server/Makefile.am
index ea3e2ff..09710eb 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -12,6 +12,7 @@ AM_CPPFLAGS = \
$(SASL_CFLAGS) \
$(SLIRP_CFLAGS) \
$(SMARTCARD_CFLAGS) \
+ $(GSTREAMER_0_10_CFLAGS) \
$(GSTREAMER_1_0_CFLAGS) \
$(SPICE_PROTOCOL_CFLAGS) \
$(SSL_CFLAGS) \
@@ -46,6 +47,7 @@ libserver_la_LIBADD = \
$(PIXMAN_LIBS) \
$(SASL_LIBS) \
$(SLIRP_LIBS) \
+ $(GSTREAMER_0_10_LIBS) \
$(GSTREAMER_1_0_LIBS) \
$(SSL_LIBS) \
$(Z_LIBS) \
@@ -159,6 +161,12 @@ libserver_la_SOURCES += \
$(NULL)
endif
+if HAVE_GSTREAMER_0_10
+libserver_la_SOURCES += \
+ gstreamer-encoder.c \
+ $(NULL)
+endif
+
if HAVE_GSTREAMER_1_0
libserver_la_SOURCES += \
gstreamer-encoder.c \
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 98bfe57..396687d 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -33,19 +33,28 @@
#define SPICE_GST_DEFAULT_FPS 30
-#define DO_ZERO_COPY
+#ifndef HAVE_GSTREAMER_0_10
+# define DO_ZERO_COPY
+#endif
typedef struct {
SpiceBitmapFmt spice_format;
const char *format;
uint32_t bpp;
+ uint32_t depth;
+ uint32_t endianness;
+ uint32_t blue_mask;
+ uint32_t green_mask;
+ uint32_t red_mask;
} SpiceFormatForGStreamer;
typedef struct SpiceGstVideoBuffer {
VideoBuffer base;
GstBuffer *gst_buffer;
+#ifndef HAVE_GSTREAMER_0_10
GstMapInfo map;
+#endif
} SpiceGstVideoBuffer;
typedef struct {
@@ -282,7 +291,9 @@ static void spice_gst_video_buffer_free(VideoBuffer *video_buffer)
{
SpiceGstVideoBuffer *buffer = (SpiceGstVideoBuffer*)video_buffer;
if (buffer->gst_buffer) {
+#ifndef HAVE_GSTREAMER_0_10
gst_buffer_unmap(buffer->gst_buffer, &buffer->map);
+#endif
gst_buffer_unref(buffer->gst_buffer);
}
free(buffer);
@@ -742,11 +753,11 @@ static const SpiceFormatForGStreamer *map_format(SpiceBitmapFmt format)
* section-types-definitions.html documents.
*/
static const SpiceFormatForGStreamer format_map[] = {
- {SPICE_BITMAP_FMT_RGBA, "BGRA", 32},
- {SPICE_BITMAP_FMT_16BIT, "RGB15", 16},
+ {SPICE_BITMAP_FMT_RGBA, "BGRA", 32, 24, 4321, 0xff000000, 0xff0000, 0xff00},
+ {SPICE_BITMAP_FMT_16BIT, "RGB15", 16, 15, 4321, 0x001f, 0x03E0, 0x7C00},
/* TODO: Test the other formats */
- {SPICE_BITMAP_FMT_32BIT, "BGRx", 32},
- {SPICE_BITMAP_FMT_24BIT, "BGR", 24},
+ {SPICE_BITMAP_FMT_32BIT, "BGRx", 32, 24, 4321, 0xff000000, 0xff0000, 0xff00},
+ {SPICE_BITMAP_FMT_24BIT, "BGR", 24, 24, 4321, 0xff0000, 0xff00, 0xff},
};
int i;
@@ -768,8 +779,18 @@ static void set_appsrc_caps(SpiceGstEncoder *encoder)
gst_caps_unref(encoder->src_caps);
}
encoder->src_caps = gst_caps_new_simple(
+#ifdef HAVE_GSTREAMER_0_10
+ "video/x-raw-rgb",
+ "bpp", G_TYPE_INT, encoder->format->bpp,
+ "depth", G_TYPE_INT, encoder->format->depth,
+ "endianness", G_TYPE_INT, encoder->format->endianness,
+ "red_mask", G_TYPE_INT, encoder->format->red_mask,
+ "green_mask", G_TYPE_INT, encoder->format->green_mask,
+ "blue_mask", G_TYPE_INT, encoder->format->blue_mask,
+#else
"video/x-raw",
"format", G_TYPE_STRING, encoder->format->format,
+#endif
"width", G_TYPE_INT, encoder->width,
"height", G_TYPE_INT, encoder->height,
"framerate", GST_TYPE_FRACTION, get_source_fps(encoder), 1,
@@ -807,6 +828,13 @@ static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_encoder)
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
SpiceGstVideoBuffer *outbuf = create_gst_video_buffer();
+#ifdef HAVE_GSTREAMER_0_10
+ outbuf->gst_buffer = gst_app_sink_pull_buffer(encoder->appsink);
+ if (outbuf->gst_buffer) {
+ outbuf->base.data = GST_BUFFER_DATA(outbuf->gst_buffer);
+ outbuf->base.size = GST_BUFFER_SIZE(outbuf->gst_buffer);
+ }
+#else
GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
if (sample) {
outbuf->gst_buffer = gst_sample_get_buffer(sample);
@@ -817,6 +845,7 @@ static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_encoder)
outbuf->base.size = gst_buffer_get_size(outbuf->gst_buffer);
}
}
+#endif
/* Notify the main thread that the output buffer is ready */
g_mutex_lock(&encoder->outbuf_mutex);
@@ -832,7 +861,11 @@ static const gchar* get_gst_codec_name(SpiceGstEncoder *encoder)
switch (encoder->base.codec_type)
{
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+#ifdef HAVE_GSTREAMER_0_10
+ return "ffenc_mjpeg";
+#else
return "avenc_mjpeg";
+#endif
case SPICE_VIDEO_CODEC_TYPE_VP8:
return "vp8enc";
case SPICE_VIDEO_CODEC_TYPE_H264:
@@ -846,6 +879,11 @@ static const gchar* get_gst_codec_name(SpiceGstEncoder *encoder)
static gboolean create_pipeline(SpiceGstEncoder *encoder)
{
+#ifdef HAVE_GSTREAMER_0_10
+ const gchar *converter = "ffmpegcolorspace";
+#else
+ const gchar *converter = "videoconvert";
+#endif
const gchar* gstenc_name = get_gst_codec_name(encoder);
if (!gstenc_name) {
return FALSE;
@@ -854,12 +892,17 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
switch (encoder->base.codec_type)
{
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+#ifdef HAVE_GSTREAMER_0_10
+ gstenc_opts = g_strdup("");
+#else
/* Set max-threads to ensure zero-frame latency */
gstenc_opts = g_strdup("max-threads=1");
+#endif
break;
case SPICE_VIDEO_CODEC_TYPE_VP8: {
/* See http://www.webmproject.org/docs/encoder-parameters/
- * - Set end-usage to get a constant bitrate to help with streaming.
+ * - Set mode/end-usage to get a constant bitrate to help with
+ * streaming.
* - min-quantizer ensures the bitrate does not get needlessly high.
* - resize-allowed would be useful for low bitrate situations but
* the decoder does not return a frame of the expected size so
@@ -867,13 +910,17 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
* - error-resilient minimises artifacts in case the client drops a
* frame.
* - Set lag-in-frames, deadline and cpu-used to match
- * "Profile Realtime". lag-in-frames ensures zero-frame latency,
- * deadline turns on realtime behavior, and cpu-used targets a 75%
- * CPU usage.
+ * "Profile Realtime". max-latency/lag-in-frames ensures zero-frame
+ * latency, deadline turns on realtime behavior, cpu-used targets a
+ * 75% CPU usage while speed simply prioritizes encoding speed.
* - deadline is supposed to be set in microseconds but in practice
* it behaves like a boolean.
*/
+#ifdef HAVE_GSTREAMER_0_10
+ gstenc_opts = g_strdup_printf("mode=cbr min-quantizer=10 error-resilient=true max-latency=0 speed=7");
+#else
gstenc_opts = g_strdup_printf("end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4");
+#endif
break;
}
case SPICE_VIDEO_CODEC_TYPE_H264:
@@ -892,7 +939,7 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
}
GError *err = NULL;
- gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! %s %s name=encoder ! appsink name=sink", gstenc_name, gstenc_opts);
+ gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! %s ! %s %s name=encoder ! appsink name=sink", converter, gstenc_name, gstenc_opts);
spice_debug("GStreamer pipeline: %s", desc);
encoder->pipeline = gst_parse_launch_full(desc, NULL, GST_PARSE_FLAG_FATAL_ERRORS, &err);
g_free(gstenc_opts);
@@ -910,12 +957,20 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
encoder->gstenc = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "encoder");
encoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "sink"));
+#ifdef HAVE_GSTREAMER_0_10
+ GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, NULL, {NULL}};
+#else
GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, {NULL}};
+#endif
gst_app_sink_set_callbacks(encoder->appsink, &appsink_cbs, encoder, NULL);
/* Hook into the bus so we can handle errors */
GstBus *bus = gst_element_get_bus(encoder->pipeline);
+#ifdef HAVE_GSTREAMER_0_10
+ gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder);
+#else
gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder, NULL);
+#endif
gst_object_unref(bus);
if (encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG) {
@@ -971,6 +1026,18 @@ static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
g_object_set(gobject, prop, (guint)gst_bit_rate, NULL);
break;
}
+ case G_TYPE_LONG: {
+ GParamSpecLong *range = G_PARAM_SPEC_LONG(param);
+ gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+ g_object_set(gobject, prop, (glong)gst_bit_rate, NULL);
+ break;
+ }
+ case G_TYPE_ULONG: {
+ GParamSpecULong *range = G_PARAM_SPEC_ULONG(param);
+ gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+ g_object_set(gobject, prop, (gulong)gst_bit_rate, NULL);
+ break;
+ }
case G_TYPE_INT64: {
GParamSpecInt64 *range = G_PARAM_SPEC_INT64(param);
gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
@@ -1205,9 +1272,25 @@ static inline int chunk_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap
return TRUE;
}
-/* A helper for push_raw_frame() */
+#ifdef HAVE_GSTREAMER_0_10
+/* Dummy structure to avoid too many #ifdef in the main codepaths */
+typedef struct {
+ gpointer memory;
+} GstMapInfo;
+#endif
+
+/* A helper for push_raw_frame()
+ * Note: In case of error the buffer is unref-ed.
+ */
static uint8_t *allocate_and_map_memory(gsize size, GstMapInfo *map, GstBuffer *buffer)
{
+#ifdef HAVE_GSTREAMER_0_10
+ buffer->malloc_data = g_malloc(size);
+ GST_BUFFER_DATA(buffer) = buffer->malloc_data;
+ GST_BUFFER_SIZE(buffer) = size;
+
+ return GST_BUFFER_DATA(buffer);
+#else
GstMemory *mem = gst_allocator_alloc(NULL, size, NULL);
if (!mem) {
gst_buffer_unref(buffer);
@@ -1219,6 +1302,16 @@ static uint8_t *allocate_and_map_memory(gsize size, GstMapInfo *map, GstBuffer *
return NULL;
}
return map->data;
+#endif
+}
+
+static void unmap_and_release_memory(GstMapInfo *map, GstBuffer *buffer)
+{
+#ifndef HAVE_GSTREAMER_0_10
+ gst_memory_unmap(map->memory, map);
+ gst_memory_unref(map->memory);
+#endif
+ gst_buffer_unref(buffer);
}
/* A helper for spice_gst_encoder_encode_frame() */
@@ -1251,9 +1344,7 @@ static int push_raw_frame(SpiceGstEncoder *encoder,
chunk_offset += src->left * encoder->format->bpp / 8;
if (!line_copy(encoder, bitmap, chunk_offset, stream_stride, height, dst)) {
- gst_memory_unmap(map.memory, &map);
- gst_memory_unref(map.memory);
- gst_buffer_unref(buffer);
+ unmap_and_release_memory(&map, buffer);
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
} else {
@@ -1277,18 +1368,21 @@ static int push_raw_frame(SpiceGstEncoder *encoder,
if (!dst) {
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
- if (!chunk_copy(encoder, bitmap, chunk_index, chunk_offset, len, dst)) {
- gst_memory_unmap(map.memory, &map);
- gst_memory_unref(map.memory);
- gst_buffer_unref(buffer);
+ if (!chunk_copy(encoder, bitmap, chunk_index, chunk_offset,
+ len, dst)) {
+ unmap_and_release_memory(&map, buffer);
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
}
}
+#ifdef HAVE_GSTREAMER_0_10
+ gst_buffer_set_caps(buffer, encoder->src_caps);
+#else
if (map.memory) {
gst_memory_unmap(map.memory, &map);
gst_buffer_append_memory(buffer, map.memory);
}
+#endif
GstFlowReturn ret = gst_app_src_push_buffer(encoder->appsrc, buffer);
if (ret != GST_FLOW_OK) {
diff --git a/server/reds.c b/server/reds.c
index f3b5874..10b943a 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -3570,7 +3570,7 @@ static const EnumNames video_encoder_names[] = {
static new_video_encoder_t video_encoder_procs[] = {
&mjpeg_encoder_new,
-#ifdef HAVE_GSTREAMER_1_0
+#if defined(HAVE_GSTREAMER_1_0) || defined(HAVE_GSTREAMER_0_10)
&gstreamer_encoder_new,
#else
NULL,
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 3423118..5522818 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -197,7 +197,7 @@ VideoEncoder* mjpeg_encoder_new(SpiceVideoCodecType codec_type,
VideoEncoderRateControlCbs *cbs,
bitmap_ref_t bitmap_ref,
bitmap_unref_t bitmap_unref);
-#ifdef HAVE_GSTREAMER_1_0
+#if defined(HAVE_GSTREAMER_1_0) || defined(HAVE_GSTREAMER_0_10)
VideoEncoder* gstreamer_encoder_new(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs,
commit b64b9e5657debb4eeb3cc17250d884af85ae38af
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Sat Apr 16 15:06:11 2016 +0200
streaming: Dynamically adjust the GStreamer encoder bitrate if possible
This is faster and lets the encoder leverage past bitrate shaping
history to attain the target faster.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index d5ead37..98bfe57 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -99,6 +99,9 @@ typedef struct SpiceGstEncoder {
GstElement *gstenc;
GParamSpec *gstenc_bitrate_param;
+ /* True if the encoder's bitrate can be modified while playing. */
+ gboolean gstenc_bitrate_is_dynamic;
+
/* Pipeline parameters to modify before the next frame. */
# define SPICE_GST_VIDEO_PIPELINE_STATE 0x1
# define SPICE_GST_VIDEO_PIPELINE_BITRATE 0x2
@@ -473,9 +476,16 @@ static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
/* ---------- Encoder bit rate control ---------- */
+static void set_gstenc_bitrate(SpiceGstEncoder *encoder);
+
static void set_video_bit_rate(SpiceGstEncoder *encoder, uint64_t bit_rate)
{
- if (abs(bit_rate - encoder->video_bit_rate) > encoder->video_bit_rate * SPICE_GST_VIDEO_BITRATE_MARGIN) {
+ if (encoder->video_bit_rate != bit_rate &&
+ encoder->gstenc_bitrate_is_dynamic) {
+ encoder->video_bit_rate = bit_rate;
+ set_gstenc_bitrate(encoder);
+
+ } else if (abs(bit_rate - encoder->video_bit_rate) > encoder->video_bit_rate * SPICE_GST_VIDEO_BITRATE_MARGIN) {
encoder->video_bit_rate = bit_rate;
set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_BITRATE);
}
@@ -920,7 +930,9 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
if (encoder->gstenc_bitrate_param == NULL) {
encoder->gstenc_bitrate_param = g_object_class_find_property(class, "target-bitrate");
}
- if (!encoder->gstenc_bitrate_param) {
+ if (encoder->gstenc_bitrate_param) {
+ encoder->gstenc_bitrate_is_dynamic = (encoder->gstenc_bitrate_param->flags & GST_PARAM_MUTABLE_PLAYING);
+ } else {
spice_warning("GStreamer error: could not find the %s bitrate parameter", gstenc_name);
}
commit fc68e2e5bab5d4b03bcb4002a111594f13443b3a
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Mon Jul 27 18:02:04 2015 +0200
streaming: Respect the GStreamer encoder's valid bit rate range
GObject returns an error instead of clamping if given an out of range
property value.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 0b63c9d..d5ead37 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -20,6 +20,8 @@
#include <config.h>
#endif
+#include <inttypes.h>
+
#include <gst/gst.h>
#include <gst/app/gstappsrc.h>
#include <gst/app/gstappsink.h>
@@ -95,6 +97,7 @@ typedef struct SpiceGstEncoder {
GstAppSrc *appsrc;
GstCaps *src_caps;
GstElement *gstenc;
+ GParamSpec *gstenc_bitrate_param;
/* Pipeline parameters to modify before the next frame. */
# define SPICE_GST_VIDEO_PIPELINE_STATE 0x1
@@ -911,6 +914,16 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
gst_pipeline_use_clock(GST_PIPELINE(encoder->pipeline), NULL);
}
+ /* Figure out which parameter controls the GStreamer encoder's bitrate */
+ GObjectClass *class = G_OBJECT_GET_CLASS(encoder->gstenc);
+ encoder->gstenc_bitrate_param = g_object_class_find_property(class, "bitrate");
+ if (encoder->gstenc_bitrate_param == NULL) {
+ encoder->gstenc_bitrate_param = g_object_class_find_property(class, "target-bitrate");
+ }
+ if (!encoder->gstenc_bitrate_param) {
+ spice_warning("GStreamer error: could not find the %s bitrate parameter", gstenc_name);
+ }
+
set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_STATE |
SPICE_GST_VIDEO_PIPELINE_BITRATE |
SPICE_GST_VIDEO_PIPELINE_CAPS);
@@ -921,28 +934,48 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
/* A helper for configure_pipeline() */
static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
{
- /* Configure the encoder bitrate */
- switch (encoder->base.codec_type) {
- case SPICE_VIDEO_CODEC_TYPE_MJPEG:
- g_object_set(G_OBJECT(encoder->gstenc),
- "bitrate", (gint)encoder->video_bit_rate,
- NULL);
+ GParamSpec *param = encoder->gstenc_bitrate_param;
+ if (!param) {
+ return;
+ }
+
+ uint64_t gst_bit_rate = encoder->video_bit_rate;
+ if (strstr(g_param_spec_get_blurb(param), "kbit")) {
+ gst_bit_rate = gst_bit_rate / 1024;
+ }
+
+ GObject * gobject = G_OBJECT(encoder->gstenc);
+ const gchar *prop = g_param_spec_get_name(param);
+ switch (param->value_type) {
+ case G_TYPE_INT: {
+ GParamSpecInt *range = G_PARAM_SPEC_INT(param);
+ gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+ g_object_set(gobject, prop, (gint)gst_bit_rate, NULL);
break;
- case SPICE_VIDEO_CODEC_TYPE_VP8:
- g_object_set(G_OBJECT(encoder->gstenc),
- "target-bitrate", (gint)encoder->video_bit_rate,
- NULL);
+ }
+ case G_TYPE_UINT: {
+ GParamSpecUInt *range = G_PARAM_SPEC_UINT(param);
+ gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+ g_object_set(gobject, prop, (guint)gst_bit_rate, NULL);
break;
- case SPICE_VIDEO_CODEC_TYPE_H264:
- g_object_set(G_OBJECT(encoder->gstenc),
- "bitrate", (guint)(encoder->bit_rate / 1024),
- NULL);
+ }
+ case G_TYPE_INT64: {
+ GParamSpecInt64 *range = G_PARAM_SPEC_INT64(param);
+ gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+ g_object_set(gobject, prop, (gint64)gst_bit_rate, NULL);
+ break;
+ }
+ case G_TYPE_UINT64: {
+ GParamSpecUInt64 *range = G_PARAM_SPEC_UINT64(param);
+ gst_bit_rate = MAX(range->minimum, MIN(range->maximum, gst_bit_rate));
+ g_object_set(gobject, prop, (guint64)gst_bit_rate, NULL);
break;
+ }
default:
- /* gstreamer_encoder_new() should have rejected this codec type */
- spice_warning("unsupported codec type %d", encoder->base.codec_type);
- free_pipeline(encoder);
+ spice_warning("the %s property has an unsupported type %zu",
+ prop, param->value_type);
}
+ spice_debug("setting the GStreamer %s to %"PRIu64, prop, gst_bit_rate);
}
/* A helper for spice_gst_encoder_encode_frame() */
commit c6f6471eb8e382be90fbe994c0d71c7e8ddb821e
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Wed Jul 22 18:42:23 2015 +0200
streaming: Give up after a while if GStreamer cannot handle the video
This typically happens when sending very small frames (less than
16 pixels in one dimension) to the x264enc encoder.
This avoids repeatedly wasting time rebuilding the pipeline.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index ed4acd7..0b63c9d 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -82,6 +82,9 @@ typedef struct SpiceGstEncoder {
const SpiceFormatForGStreamer *format;
SpiceBitmapFmt spice_format;
+ /* Number of consecutive frame encoding errors. */
+ uint32_t errors;
+
/* ---------- GStreamer pipeline ---------- */
/* Pointers to the GStreamer pipeline elements. If pipeline is NULL the
@@ -811,14 +814,35 @@ static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_encoder)
return GST_FLOW_OK;
}
+static const gchar* get_gst_codec_name(SpiceGstEncoder *encoder)
+{
+ switch (encoder->base.codec_type)
+ {
+ case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+ return "avenc_mjpeg";
+ case SPICE_VIDEO_CODEC_TYPE_VP8:
+ return "vp8enc";
+ case SPICE_VIDEO_CODEC_TYPE_H264:
+ return "x264enc";
+ default:
+ /* gstreamer_encoder_new() should have rejected this codec type */
+ spice_warning("unsupported codec type %d", encoder->base.codec_type);
+ return NULL;
+ }
+}
+
static gboolean create_pipeline(SpiceGstEncoder *encoder)
{
- gchar *gstenc;
+ const gchar* gstenc_name = get_gst_codec_name(encoder);
+ if (!gstenc_name) {
+ return FALSE;
+ }
+ gchar* gstenc_opts;
switch (encoder->base.codec_type)
{
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
/* Set max-threads to ensure zero-frame latency */
- gstenc = g_strdup("avenc_mjpeg max-threads=1");
+ gstenc_opts = g_strdup("max-threads=1");
break;
case SPICE_VIDEO_CODEC_TYPE_VP8: {
/* See http://www.webmproject.org/docs/encoder-parameters/
@@ -836,7 +860,7 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
* - deadline is supposed to be set in microseconds but in practice
* it behaves like a boolean.
*/
- gstenc = g_strdup_printf("vp8enc end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4");
+ gstenc_opts = g_strdup_printf("end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4");
break;
}
case SPICE_VIDEO_CODEC_TYPE_H264:
@@ -846,7 +870,7 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
* - Set intra-refresh to get more uniform compressed frame sizes,
* thus helping with streaming.
*/
- gstenc = g_strdup("x264enc byte-stream=true aud=true qp-min=15 tune=4 sliced-threads=true speed-preset=ultrafast intra-refresh=true");
+ gstenc_opts = g_strdup("byte-stream=true aud=true qp-min=15 tune=4 sliced-threads=true speed-preset=ultrafast intra-refresh=true");
break;
default:
/* gstreamer_encoder_new() should have rejected this codec type */
@@ -855,10 +879,10 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
}
GError *err = NULL;
- gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! %s name=encoder ! appsink name=sink", gstenc);
+ gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! %s %s name=encoder ! appsink name=sink", gstenc_name, gstenc_opts);
spice_debug("GStreamer pipeline: %s", desc);
encoder->pipeline = gst_parse_launch_full(desc, NULL, GST_PARSE_FLAG_FATAL_ERRORS, &err);
- g_free(gstenc);
+ g_free(gstenc_opts);
g_free(desc);
if (!encoder->pipeline || err) {
spice_warning("GStreamer error: %s", err->message);
@@ -873,20 +897,12 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
encoder->gstenc = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "encoder");
encoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "sink"));
-#ifdef HAVE_GSTREAMER_0_10
- GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, NULL, {NULL}};
-#else
GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, {NULL}};
-#endif
gst_app_sink_set_callbacks(encoder->appsink, &appsink_cbs, encoder, NULL);
/* Hook into the bus so we can handle errors */
GstBus *bus = gst_element_get_bus(encoder->pipeline);
-#ifdef HAVE_GSTREAMER_0_10
- gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder);
-#else
gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder, NULL);
-#endif
gst_object_unref(bus);
if (encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG) {
@@ -1301,6 +1317,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
encoder->format = map_format(bitmap->format);
if (!encoder->format) {
spice_warning("unable to map format type %d", bitmap->format);
+ encoder->errors = 4;
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
encoder->spice_format = bitmap->format;
@@ -1316,6 +1333,19 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
} else if (encoder->pipeline) {
set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_CAPS);
}
+ encoder->errors = 0;
+ } else if (encoder->errors >= 3) {
+ /* The pipeline keeps failing to handle the frames we send it, which
+ * is usually because they are too small (mouse pointer-sized).
+ * So give up until something changes.
+ */
+ if (encoder->errors == 3) {
+ spice_debug("%s cannot compress %dx%d:%dbpp frames",
+ get_gst_codec_name(encoder), encoder->width,
+ encoder->height, encoder->format->bpp);
+ encoder->errors++;
+ }
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
if (rate_control_is_active(encoder) &&
@@ -1326,6 +1356,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
}
if (!configure_pipeline(encoder)) {
+ encoder->errors++;
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
@@ -1340,6 +1371,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
* from scratch.
*/
free_pipeline(encoder);
+ encoder->errors++;
}
}
commit ef6d6b063d0baea38a1e3cb34d87db345fb2f0ad
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Mar 4 15:38:28 2016 +0100
streaming: Adjust the frame rate based on the GStreamer encoding time
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 8eb7686..ed4acd7 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -48,6 +48,7 @@ typedef struct SpiceGstVideoBuffer {
typedef struct {
uint32_t mm_time;
+ uint64_t duration;
uint32_t size;
} SpiceGstFrameInformation;
@@ -134,6 +135,9 @@ typedef struct SpiceGstEncoder {
/* The index of the oldest frame taken into account for the statistics. */
uint32_t stat_first;
+ /* Used to compute the average frame encoding time. */
+ uint64_t stat_duration_sum;
+
/* Used to compute the average frame size. */
uint64_t stat_size_sum;
@@ -353,6 +357,14 @@ static uint64_t get_effective_bit_rate(SpiceGstEncoder *encoder)
return elapsed ? encoder->stat_size_sum * 8 * MSEC_PER_SEC / elapsed : 0;
}
+static uint64_t get_average_encoding_time(SpiceGstEncoder *encoder)
+{
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? SPICE_GST_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ return encoder->stat_duration_sum / count;
+}
+
static uint64_t get_average_frame_size(SpiceGstEncoder *encoder)
{
uint32_t count = encoder->history_last +
@@ -422,19 +434,21 @@ static uint64_t get_period_bit_rate(SpiceGstEncoder *encoder, uint32_t from,
}
static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
- uint32_t size)
+ uint64_t duration, uint32_t size)
{
/* Update the statistics */
uint32_t count = encoder->history_last +
(encoder->history_last < encoder->stat_first ? SPICE_GST_HISTORY_SIZE : 0) -
encoder->stat_first + 1;
if (count == SPICE_GST_FRAME_STATISTICS_COUNT) {
+ encoder->stat_duration_sum -= encoder->history[encoder->stat_first].duration;
encoder->stat_size_sum -= encoder->history[encoder->stat_first].size;
if (encoder->stat_size_max == encoder->history[encoder->stat_first].size) {
encoder->stat_size_max = 0;
}
encoder->stat_first = (encoder->stat_first + 1) % SPICE_GST_HISTORY_SIZE;
}
+ encoder->stat_duration_sum += duration;
encoder->stat_size_sum += size;
if (encoder->stat_size_max > 0 && size > encoder->stat_size_max) {
encoder->stat_size_max = size;
@@ -446,6 +460,7 @@ static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
encoder->history_first = (encoder->history_first + 1) % SPICE_GST_HISTORY_SIZE;
}
encoder->history[encoder->history_last].mm_time = frame_mm_time;
+ encoder->history[encoder->history_last].duration = duration;
encoder->history[encoder->history_last].size = size;
}
@@ -478,15 +493,23 @@ static uint32_t get_min_playback_delay(SpiceGstEncoder *encoder)
static void update_client_playback_delay(SpiceGstEncoder *encoder)
{
if (encoder->cbs.update_client_playback_delay) {
- uint32_t min_delay = get_min_playback_delay(encoder);
+ uint32_t min_delay = get_min_playback_delay(encoder) + get_average_encoding_time(encoder) / NSEC_PER_MILLISEC;
encoder->cbs.update_client_playback_delay(encoder->cbs.opaque, min_delay);
}
}
static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
{
+ uint64_t period_ns = NSEC_PER_SEC / get_source_fps(encoder);
+ uint64_t min_delay_ns = get_average_encoding_time(encoder);
+ if (min_delay_ns > period_ns) {
+ spice_warning("your system seems to be too slow to encode this %dx%d video in real time", encoder->width, encoder->height);
+ }
+
+ min_delay_ns = MIN(min_delay_ns, SPICE_GST_MAX_PERIOD);
if (encoder->vbuffer_free >= 0) {
- encoder->next_frame_mm_time = 0;
+ encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) +
+ min_delay_ns / NSEC_PER_MILLISEC;
return;
}
@@ -494,7 +517,6 @@ static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
* Use nanoseconds to avoid precision loss.
*/
uint64_t delay_ns = -encoder->vbuffer_free * 8 * NSEC_PER_SEC / encoder->bit_rate;
- uint64_t period_ns = NSEC_PER_SEC / get_source_fps(encoder);
uint32_t drops = (delay_ns + period_ns - 1) / period_ns; /* round up */
spice_debug("drops=%u vbuffer %d/%d", drops, encoder->vbuffer_free,
encoder->vbuffer_size);
@@ -509,7 +531,8 @@ static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
}
delay_ns = SPICE_GST_MAX_PERIOD;
}
- encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) + delay_ns / NSEC_PER_MILLISEC;
+ encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) +
+ MAX(delay_ns, min_delay_ns) / NSEC_PER_MILLISEC;
/* Drops mean a higher delay between encoded frames so update the
* playback delay.
@@ -600,6 +623,7 @@ static void set_bit_rate(SpiceGstEncoder *encoder, uint64_t bit_rate)
* situation anymore.
*/
encoder->stat_first = encoder->history_last;
+ encoder->stat_duration_sum = encoder->history[encoder->history_last].duration;
encoder->stat_size_sum = encoder->stat_size_max = encoder->history[encoder->history_last].size;
if (bit_rate > encoder->video_bit_rate) {
@@ -659,7 +683,7 @@ static inline gboolean handle_server_drops(SpiceGstEncoder *encoder,
* time during which the buffer was refilling. This implies dropping this
* frame.
*/
- add_frame(encoder, frame_mm_time, 0);
+ add_frame(encoder, frame_mm_time, 0, 0);
if (encoder->server_drops >= get_source_fps(encoder)) {
spice_debug("cut the bit rate");
@@ -1305,6 +1329,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
+ uint64_t start = spice_get_monotonic_time_ns();
int rc = push_raw_frame(encoder, bitmap, src, top_down, bitmap_opaque);
if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
rc = pull_compressed_buffer(encoder, outbuf);
@@ -1325,7 +1350,8 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
return rc;
}
uint32_t last_mm_time = get_last_frame_mm_time(encoder);
- add_frame(encoder, frame_mm_time, (*outbuf)->size);
+ add_frame(encoder, frame_mm_time, spice_get_monotonic_time_ns() - start,
+ (*outbuf)->size);
int32_t refill = encoder->bit_rate * (frame_mm_time - last_mm_time) / MSEC_PER_SEC / 8;
encoder->vbuffer_free = MIN(encoder->vbuffer_free + refill,
commit 669d7090ca4522c3270b0985ec0ddafc03fe5c96
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Mon Jul 20 11:45:20 2015 +0200
streaming: Adjust the GStreamer encoder bit rate to the network
The video encoder uses the client reports and/or notifications of
server frame drops as its feedback mechanisms. In particular it keeps
track of the maximum video margin and reduces the bit rate whenever the
margin goes below certain thresholds or decreases too sharply.
It uses these to figure out the lowest bit rate that causes negative
feedback, and the highest bit rate that allows a return to positive
feedbacks. It then works to narrow this range and settles on the lower
end once the spread has gone below a given threshold.
All the while it monitors the effective bit rate to ensure the target
bit rate does not grow significantly beyond what the GStreamer encoder
will produce: this avoids target bit rate 'bubbles' which would
invariably be followed by a bit rate crash with accompanying frame loss.
As soon as the network feedback indicates a significant degradation the
bit rate is lowered to minimize the risk of frame loss and/or long
freezes.
It also relies on the existing shaping of the GStreamer output bit rate
to minimize the pipeline reconfigurations.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index c2af141..8eb7686 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -51,6 +51,12 @@ typedef struct {
uint32_t size;
} SpiceGstFrameInformation;
+typedef enum SpiceGstBitRateStatus {
+ SPICE_GST_BITRATE_DECREASING,
+ SPICE_GST_BITRATE_INCREASING,
+ SPICE_GST_BITRATE_STABLE,
+} SpiceGstBitRateStatus;
+
typedef struct SpiceGstEncoder {
VideoEncoder base;
@@ -97,6 +103,13 @@ typedef struct SpiceGstEncoder {
GCond outbuf_cond;
VideoBuffer *outbuf;
+ /* The video bit rate. */
+ uint64_t video_bit_rate;
+
+ /* Don't bother changing the GStreamer bit rate if close enough. */
+# define SPICE_GST_VIDEO_BITRATE_MARGIN 0.05
+
+
/* ---------- Encoded frame statistics ---------- */
/* Should be >= than FRAME_STATISTICS_COUNT. This is also used to
@@ -130,7 +143,7 @@ typedef struct SpiceGstEncoder {
/* ---------- Encoder bit rate control ----------
*
- * GStreamer encoders don't follow the specified bit rate very
+ * GStreamer encoders don't follow the specified video_bit_rate very
* closely. These fields are used to ensure we don't exceed the desired
* stream bit rate, regardless of the GStreamer encoder's output.
*/
@@ -138,7 +151,7 @@ typedef struct SpiceGstEncoder {
/* The bit rate target for the outgoing network stream. (bits per second) */
uint64_t bit_rate;
- /* The minimum bit rate. */
+ /* The minimum bit rate / bit rate increment. */
# define SPICE_GST_MIN_BITRATE (128 * 1024)
/* The default bit rate. */
@@ -164,6 +177,89 @@ typedef struct SpiceGstEncoder {
/* How big of a margin to take to cover for latency jitter. */
# define SPICE_GST_LATENCY_MARGIN 0.1
+
+
+ /* ---------- Network bit rate control ----------
+ *
+ * State information for figuring out the optimal bit rate for the
+ * current network conditions.
+ */
+
+ /* The mm_time of the last bit rate change. */
+ uint32_t last_change;
+
+ /* How much to reduce the bit rate in case of network congestion. */
+# define SPICE_GST_BITRATE_CUT 2
+# define SPICE_GST_BITRATE_REDUCE (4.0 / 3.0)
+
+ /* Never increase the bit rate by more than this amount (bits per second). */
+# define SPICE_GST_BITRATE_MAX_STEP (1024 * 1024)
+
+ /* The maximum bit rate that one can maybe use without causing network
+ * congestion.
+ */
+ uint64_t max_bit_rate;
+
+ /* The last bit rate that let us recover from network congestion. */
+ uint64_t min_bit_rate;
+
+ /* Defines when the spread between max_bit_rate and min_bit_rate has been
+ * narrowed down enough. Note that this value should be large enough for
+ * min_bit_rate to allow recovery from network congestion in a reasonable
+ * time frame, and to absorb transient traffic spikes (potentially from
+ * other sources).
+ * This is also used as a multiplier for the video_bit_rate so it does
+ * not have to be changed too often.
+ */
+# define SPICE_GST_BITRATE_MARGIN SPICE_GST_BITRATE_REDUCE
+
+ /* Whether the bit rate was last decreased, increased or kept stable. */
+ SpiceGstBitRateStatus status;
+
+ /* The network bit rate control uses an AIMD scheme (Additive Increase,
+ * Multiplicative Decrease). The increment step depends on the spread
+ * between the minimum and maximum bit rates.
+ */
+ uint64_t bit_rate_step;
+
+ /* How often to increase the bit rate. */
+ uint32_t increase_interval;
+
+# define SPICE_GST_BITRATE_UP_INTERVAL (MSEC_PER_SEC * 2)
+# define SPICE_GST_BITRATE_UP_CLIENT_STABLE (MSEC_PER_SEC * 60 * 2)
+# define SPICE_GST_BITRATE_UP_SERVER_STABLE (MSEC_PER_SEC * 3600 * 4)
+# define SPICE_GST_BITRATE_UP_RESET_MAX (MSEC_PER_SEC * 30)
+
+
+ /* ---------- Client feedback ---------- */
+
+ /* TRUE if gst_encoder_client_stream_report() is being called. */
+ gboolean has_client_reports;
+
+ /* The margin is the amount of time between the reception of a piece of
+ * media data by the client and the time when it should be displayed.
+ * Increasing the bit rate increases the transmission time and thus
+ * reduces the margin.
+ */
+ int32_t last_video_margin;
+ int32_t max_video_margin;
+ uint32_t max_audio_margin;
+
+# define SPICE_GST_VIDEO_MARGIN_GOOD 0.75
+# define SPICE_GST_VIDEO_MARGIN_AVERAGE 0.5
+# define SPICE_GST_VIDEO_MARGIN_BAD 0.3
+
+# define SPICE_GST_VIDEO_DELTA_BAD 0.2
+# define SPICE_GST_VIDEO_DELTA_AVERAGE 0.15
+
+# define SPICE_GST_AUDIO_MARGIN_BAD 0.5
+# define SPICE_GST_AUDIO_VIDEO_RATIO 1.25
+
+
+ /* ---------- Server feedback ---------- */
+
+ /* How many frames were dropped by the server since the last encoded frame. */
+ uint32_t server_drops;
} SpiceGstEncoder;
@@ -356,6 +452,14 @@ static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
/* ---------- Encoder bit rate control ---------- */
+static void set_video_bit_rate(SpiceGstEncoder *encoder, uint64_t bit_rate)
+{
+ if (abs(bit_rate - encoder->video_bit_rate) > encoder->video_bit_rate * SPICE_GST_VIDEO_BITRATE_MARGIN) {
+ encoder->video_bit_rate = bit_rate;
+ set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_BITRATE);
+ }
+}
+
static uint32_t get_min_playback_delay(SpiceGstEncoder *encoder)
{
/* Make sure the delay is large enough to send a large frame (typically
@@ -397,6 +501,12 @@ static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
delay_ns = drops * period_ns + period_ns / 2;
if (delay_ns > SPICE_GST_MAX_PERIOD) {
+ /* Reduce the video bit rate so we don't have to drop so many frames. */
+ if (encoder->video_bit_rate > encoder->bit_rate * SPICE_GST_BITRATE_MARGIN) {
+ set_video_bit_rate(encoder, encoder->bit_rate * SPICE_GST_BITRATE_MARGIN);
+ } else {
+ set_video_bit_rate(encoder, encoder->bit_rate);
+ }
delay_ns = SPICE_GST_MAX_PERIOD;
}
encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) + delay_ns / NSEC_PER_MILLISEC;
@@ -421,19 +531,165 @@ static uint64_t get_bit_rate_cap(SpiceGstEncoder *encoder)
return raw_frame_bits * get_source_fps(encoder) / 10;
}
-static void adjust_bit_rate(SpiceGstEncoder *encoder)
+static void set_bit_rate(SpiceGstEncoder *encoder, uint64_t bit_rate)
{
- if (encoder->bit_rate == 0) {
- /* Use the default value, */
- encoder->bit_rate = SPICE_GST_DEFAULT_BITRATE;
- } else if (encoder->bit_rate < SPICE_GST_MIN_BITRATE) {
- /* don't let the bit rate go too low */
+ if (bit_rate == 0) {
+ /* Use the default value */
+ bit_rate = SPICE_GST_DEFAULT_BITRATE;
+ }
+ if (bit_rate == encoder->bit_rate) {
+ return;
+ }
+ if (bit_rate < SPICE_GST_MIN_BITRATE) {
+ /* Don't let the bit rate go too low... */
encoder->bit_rate = SPICE_GST_MIN_BITRATE;
- } else {
+ } else if (bit_rate > encoder->bit_rate) {
/* or too high */
- encoder->bit_rate = MIN(encoder->bit_rate, get_bit_rate_cap(encoder));
+ bit_rate = MIN(bit_rate, get_bit_rate_cap(encoder));
+ }
+
+ if (bit_rate < encoder->min_bit_rate) {
+ encoder->min_bit_rate = bit_rate;
+ encoder->bit_rate_step = 0;
+ } else if (encoder->status == SPICE_GST_BITRATE_DECREASING &&
+ bit_rate > encoder->bit_rate) {
+ encoder->min_bit_rate = encoder->bit_rate;
+ encoder->bit_rate_step = 0;
+ } else if (encoder->status != SPICE_GST_BITRATE_DECREASING &&
+ bit_rate < encoder->bit_rate) {
+ encoder->max_bit_rate = encoder->bit_rate - SPICE_GST_MIN_BITRATE;
+ encoder->bit_rate_step = 0;
+ }
+ encoder->increase_interval = SPICE_GST_BITRATE_UP_INTERVAL;
+
+ if (encoder->bit_rate_step == 0) {
+ encoder->bit_rate_step = MAX(SPICE_GST_MIN_BITRATE,
+ MIN(SPICE_GST_BITRATE_MAX_STEP,
+ (encoder->max_bit_rate - encoder->min_bit_rate) / 10));
+ encoder->status = (bit_rate < encoder->bit_rate) ? SPICE_GST_BITRATE_DECREASING : SPICE_GST_BITRATE_INCREASING;
+ if (encoder->max_bit_rate / SPICE_GST_BITRATE_MARGIN < encoder->min_bit_rate) {
+ /* We have sufficiently narrowed down the optimal bit rate range.
+ * Settle on the lower end to keep a safety margin and stop
+ * rocking the boat.
+ */
+ bit_rate = encoder->min_bit_rate;
+ encoder->status = SPICE_GST_BITRATE_STABLE;
+ encoder->increase_interval = encoder->has_client_reports ? SPICE_GST_BITRATE_UP_CLIENT_STABLE : SPICE_GST_BITRATE_UP_SERVER_STABLE;
+ set_video_bit_rate(encoder, bit_rate);
+ }
+ }
+ spice_debug("%u set_bit_rate(%.3fMbps) eff %.3f %.3f-%.3f %d",
+ get_last_frame_mm_time(encoder) - encoder->last_change,
+ get_mbps(bit_rate), get_mbps(get_effective_bit_rate(encoder)),
+ get_mbps(encoder->min_bit_rate),
+ get_mbps(encoder->max_bit_rate), encoder->status);
+
+ encoder->last_change = get_last_frame_mm_time(encoder);
+ encoder->bit_rate = bit_rate;
+ /* Adjust the vbuffer size without ever increasing vbuffer_free to avoid
+ * sudden bit rate increases.
+ */
+ int32_t new_size = bit_rate * SPICE_GST_VBUFFER_SIZE / MSEC_PER_SEC / 8;
+ if (new_size < encoder->vbuffer_size && encoder->vbuffer_free > 0) {
+ encoder->vbuffer_free = MAX(0, encoder->vbuffer_free + new_size - encoder->vbuffer_size);
+ }
+ encoder->vbuffer_size = new_size;
+ update_next_frame_mm_time(encoder);
+
+ /* Frames preceeding the bit rate change are not relevant to the current
+ * situation anymore.
+ */
+ encoder->stat_first = encoder->history_last;
+ encoder->stat_size_sum = encoder->stat_size_max = encoder->history[encoder->history_last].size;
+
+ if (bit_rate > encoder->video_bit_rate) {
+ set_video_bit_rate(encoder, bit_rate * SPICE_GST_BITRATE_MARGIN);
+ }
+}
+
+static void increase_bit_rate(SpiceGstEncoder *encoder)
+{
+ if (get_effective_bit_rate(encoder) < encoder->bit_rate) {
+ /* The GStreamer encoder currently uses less bandwidth than allowed.
+ * So increasing the limit again makes no sense.
+ */
+ return;
+ }
+
+ if (encoder->bit_rate == encoder->max_bit_rate &&
+ get_last_frame_mm_time(encoder) - encoder->last_change > SPICE_GST_BITRATE_UP_RESET_MAX) {
+ /* The maximum bit rate seems to be sustainable so it was probably
+ * set too low. Probe for the maximum bit rate again.
+ */
+ encoder->max_bit_rate = get_bit_rate_cap(encoder);
+ encoder->status = SPICE_GST_BITRATE_INCREASING;
+ }
+
+ uint64_t new_bit_rate = MIN(encoder->bit_rate + encoder->bit_rate_step,
+ encoder->max_bit_rate);
+ spice_debug("increase bit rate to %.3fMbps %.3f-%.3fMbps %d",
+ get_mbps(new_bit_rate), get_mbps(encoder->min_bit_rate),
+ get_mbps(encoder->max_bit_rate), encoder->status);
+ set_bit_rate(encoder, new_bit_rate);
+}
+
+
+/* ---------- Server feedback ---------- */
+
+/* A helper for gst_encoder_encode_frame()
+ *
+ * Checks how many frames got dropped since the last encoded frame and
+ * adjusts the bit rate accordingly.
+ */
+static inline gboolean handle_server_drops(SpiceGstEncoder *encoder,
+ uint32_t frame_mm_time)
+{
+ if (encoder->server_drops == 0) {
+ return FALSE;
+ }
+
+ spice_debug("server report: got %u drops in %ums after %ums",
+ encoder->server_drops,
+ frame_mm_time - get_last_frame_mm_time(encoder),
+ frame_mm_time - encoder->last_change);
+
+ /* The server dropped a frame so clearly the buffer is full. */
+ encoder->vbuffer_free = MIN(encoder->vbuffer_free, 0);
+ /* Add a 0 byte frame so the time spent dropping frames is not counted as
+ * time during which the buffer was refilling. This implies dropping this
+ * frame.
+ */
+ add_frame(encoder, frame_mm_time, 0);
+
+ if (encoder->server_drops >= get_source_fps(encoder)) {
+ spice_debug("cut the bit rate");
+ uint64_t bit_rate = (encoder->bit_rate == encoder->min_bit_rate) ?
+ encoder->bit_rate / SPICE_GST_BITRATE_CUT :
+ MAX(encoder->min_bit_rate, encoder->bit_rate / SPICE_GST_BITRATE_CUT);
+ set_bit_rate(encoder, bit_rate);
+
+ } else {
+ spice_debug("reduce the bit rate");
+ uint64_t bit_rate = (encoder->bit_rate == encoder->min_bit_rate) ?
+ encoder->bit_rate / SPICE_GST_BITRATE_REDUCE :
+ MAX(encoder->min_bit_rate, encoder->bit_rate / SPICE_GST_BITRATE_REDUCE);
+ set_bit_rate(encoder, bit_rate);
+ }
+ encoder->server_drops = 0;
+ return TRUE;
+}
+
+/* A helper for gst_encoder_encode_frame() */
+static inline void server_increase_bit_rate(SpiceGstEncoder *encoder,
+ uint32_t frame_mm_time)
+{
+ /* Let gst_encoder_client_stream_report() deal with bit rate increases if
+ * we receive client reports.
+ */
+ if (!encoder->has_client_reports && encoder->server_drops == 0 &&
+ frame_mm_time - encoder->last_change >= encoder->increase_interval) {
+ increase_bit_rate(encoder);
}
- spice_debug("adjust_bit_rate(%.3fMbps)", get_mbps(encoder->bit_rate));
}
@@ -625,22 +881,21 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
/* A helper for configure_pipeline() */
static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
{
- adjust_bit_rate(encoder);
- switch (encoder->base.codec_type)
- {
+ /* Configure the encoder bitrate */
+ switch (encoder->base.codec_type) {
case SPICE_VIDEO_CODEC_TYPE_MJPEG:
g_object_set(G_OBJECT(encoder->gstenc),
- "bitrate", (gint)encoder->bit_rate,
+ "bitrate", (gint)encoder->video_bit_rate,
NULL);
break;
case SPICE_VIDEO_CODEC_TYPE_VP8:
g_object_set(G_OBJECT(encoder->gstenc),
- "target-bitrate", (gint)encoder->bit_rate,
+ "target-bitrate", (gint)encoder->video_bit_rate,
NULL);
break;
case SPICE_VIDEO_CODEC_TYPE_H264:
g_object_set(G_OBJECT(encoder->gstenc),
- "bitrate", encoder->bit_rate / 1024,
+ "bitrate", (guint)(encoder->bit_rate / 1024),
NULL);
break;
default:
@@ -1029,8 +1284,10 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
encoder->height = height;
if (encoder->bit_rate == 0) {
encoder->history[0].mm_time = frame_mm_time;
- encoder->bit_rate = encoder->starting_bit_rate;
- adjust_bit_rate(encoder);
+ encoder->max_bit_rate = get_bit_rate_cap(encoder);
+ encoder->min_bit_rate = SPICE_GST_MIN_BITRATE;
+ encoder->status = SPICE_GST_BITRATE_DECREASING;
+ set_bit_rate(encoder, encoder->starting_bit_rate);
encoder->vbuffer_free = 0; /* Slow start */
} else if (encoder->pipeline) {
set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_CAPS);
@@ -1038,7 +1295,8 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
}
if (rate_control_is_active(encoder) &&
- frame_mm_time < encoder->next_frame_mm_time) {
+ (handle_server_drops(encoder, frame_mm_time) ||
+ frame_mm_time < encoder->next_frame_mm_time)) {
/* Drop the frame to limit the outgoing bit rate. */
return VIDEO_ENCODER_FRAME_DROP;
}
@@ -1066,8 +1324,14 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
return rc;
}
+ uint32_t last_mm_time = get_last_frame_mm_time(encoder);
add_frame(encoder, frame_mm_time, (*outbuf)->size);
+ int32_t refill = encoder->bit_rate * (frame_mm_time - last_mm_time) / MSEC_PER_SEC / 8;
+ encoder->vbuffer_free = MIN(encoder->vbuffer_free + refill,
+ encoder->vbuffer_size) - (*outbuf)->size;
+
+ server_increase_bit_rate(encoder, frame_mm_time);
update_next_frame_mm_time(encoder);
return rc;
@@ -1078,21 +1342,115 @@ static void spice_gst_encoder_client_stream_report(VideoEncoder *video_encoder,
uint32_t num_drops,
uint32_t start_frame_mm_time,
uint32_t end_frame_mm_time,
- int32_t end_frame_delay,
- uint32_t audio_delay)
+ int32_t video_margin,
+ uint32_t audio_margin)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ encoder->has_client_reports = TRUE;
+
+ encoder->max_video_margin = MAX(encoder->max_video_margin, video_margin);
+ encoder->max_audio_margin = MAX(encoder->max_audio_margin, audio_margin);
+ int32_t margin_delta = video_margin - encoder->last_video_margin;
+ encoder->last_video_margin = video_margin;
+
uint64_t period_bit_rate = get_period_bit_rate(encoder, start_frame_mm_time, end_frame_mm_time);
- spice_debug("client report: %u/%u drops in %ums margins video %3d audio %3u bw %.3f/%.3fMbps",
+ spice_debug("client report: %u/%u drops in %ums margins video %3d/%3d audio %3u/%3u bw %.3f/%.3fMbps%s",
num_drops, num_frames, end_frame_mm_time - start_frame_mm_time,
- end_frame_delay, audio_delay,
+ video_margin, encoder->max_video_margin,
+ audio_margin, encoder->max_audio_margin,
get_mbps(period_bit_rate),
- get_mbps(get_effective_bit_rate(encoder)));
+ get_mbps(get_effective_bit_rate(encoder)),
+ start_frame_mm_time < encoder->last_change ? " obsolete" : "");
+ if (encoder->status == SPICE_GST_BITRATE_DECREASING &&
+ start_frame_mm_time < encoder->last_change) {
+ /* Some of this data predates the last bit rate reduction
+ * so it is obsolete.
+ */
+ return;
+ }
+
+ /* We normally arrange for even the largest frames to arrive a bit over
+ * one period before they should be displayed.
+ */
+ uint32_t min_margin = MSEC_PER_SEC / get_source_fps(encoder) +
+ get_network_latency(encoder) * SPICE_GST_LATENCY_MARGIN;
+
+ /* A low video margin indicates that the bit rate is too high. */
+ uint32_t score;
+ if (num_drops) {
+ score = 4;
+ } else if (margin_delta >= 0) {
+ /* The situation was bad but seems to be improving */
+ score = 0;
+ } else if (video_margin < min_margin * SPICE_GST_VIDEO_MARGIN_BAD ||
+ video_margin < encoder->max_video_margin * SPICE_GST_VIDEO_MARGIN_BAD) {
+ score = 3;
+ } else if (video_margin < min_margin ||
+ video_margin < encoder->max_video_margin * SPICE_GST_VIDEO_MARGIN_AVERAGE) {
+ score = 2;
+ } else if (video_margin < encoder->max_video_margin * SPICE_GST_VIDEO_MARGIN_GOOD) {
+ score = 1;
+ } else {
+ score = 0;
+ }
+ /* A fast dropping video margin is a compounding factor. */
+ if (margin_delta < -abs(encoder->max_video_margin) * SPICE_GST_VIDEO_DELTA_BAD) {
+ score += 2;
+ } else if (margin_delta < -abs(encoder->max_video_margin) * SPICE_GST_VIDEO_DELTA_AVERAGE) {
+ score += 1;
+ }
+
+ if (score > 3) {
+ spice_debug("score %u, cut the bit rate", score);
+ uint64_t bit_rate = (encoder->bit_rate == encoder->min_bit_rate) ?
+ encoder->bit_rate / SPICE_GST_BITRATE_CUT :
+ MAX(encoder->min_bit_rate, encoder->bit_rate / SPICE_GST_BITRATE_CUT);
+ set_bit_rate(encoder, bit_rate);
+
+ } else if (score == 3) {
+ spice_debug("score %u, reduce the bit rate", score);
+ uint64_t bit_rate = (encoder->bit_rate == encoder->min_bit_rate) ?
+ encoder->bit_rate / SPICE_GST_BITRATE_REDUCE :
+ MAX(encoder->min_bit_rate, encoder->bit_rate / SPICE_GST_BITRATE_REDUCE);
+ set_bit_rate(encoder, bit_rate);
+
+ } else if (score == 2) {
+ spice_debug("score %u, decrement the bit rate", score);
+ set_bit_rate(encoder, encoder->bit_rate - encoder->bit_rate_step);
+
+ } else if (audio_margin < encoder->max_audio_margin * SPICE_GST_AUDIO_MARGIN_BAD &&
+ audio_margin * SPICE_GST_AUDIO_VIDEO_RATIO < video_margin) {
+ /* The audio margin has decreased a lot while the video_margin
+ * remained higher. It may be that the video stream is starving the
+ * audio one of bandwidth. So reduce the bit rate.
+ */
+ spice_debug("free some bandwidth for the audio stream");
+ set_bit_rate(encoder, encoder->bit_rate - encoder->bit_rate_step);
+
+ } else if (score == 1 && period_bit_rate <= encoder->bit_rate &&
+ encoder->status == SPICE_GST_BITRATE_INCREASING) {
+ /* We only increase the bit rate when score == 0 so things got worse
+ * since the last increase, and not because of a transient bit rate
+ * peak.
+ */
+ spice_debug("degraded margin, decrement bit rate %.3f <= %.3fMbps",
+ get_mbps(period_bit_rate), get_mbps(encoder->bit_rate));
+ set_bit_rate(encoder, encoder->bit_rate - encoder->bit_rate_step);
+
+ } else if (score == 0 &&
+ get_last_frame_mm_time(encoder) - encoder->last_change >= encoder->increase_interval) {
+ /* The video margin is consistently high so increase the bit rate. */
+ increase_bit_rate(encoder);
+ }
}
static void spice_gst_encoder_notify_server_frame_drop(VideoEncoder *video_encoder)
{
- spice_debug("server report: getting frame drops...");
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ if (encoder->server_drops == 0) {
+ spice_debug("server report: getting frame drops...");
+ }
+ encoder->server_drops++;
}
static uint64_t spice_gst_encoder_get_bit_rate(VideoEncoder *video_encoder)
commit 4f04c9ac79e679146eb90acbd05f1ca19ac9270c
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Wed May 20 08:23:46 2015 +0200
streaming: Shape the bit rate of the GStreamer codecs output
The GStreamer codecs don't follow the specified bit rate very closely:
they can decide to exceed it for ten seconds or more if they consider
the scene deserves it. Such long bursts are enough to cause network
congestion, resulting in many lost frames which cause significant
display corruption.
So the GStreamer video encoder now uses a short 300ms virtual buffer
to shape the compressed video output and ensure we don't exceed the
target bit rate for any significant length of time.
It could instead rely on the network feedback (when available) to lower
the bit rate. However frequent GStreamer bit rate changes lower the
overall compression level and also result in a lower average bit rate,
both of which result in lower video quality.
The GStreamer video encoder also keeps track of the encoded frame size
so it can gather statistics and call update_client_playback_delay()
with accurate information and also annotate the client report debug
traces with the corresponding bit rate information.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index e79d9c5..c2af141 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -26,6 +26,7 @@
#include "red-common.h"
#include "video-encoder.h"
+#include "utils.h"
#define SPICE_GST_DEFAULT_FPS 30
@@ -45,6 +46,11 @@ typedef struct SpiceGstVideoBuffer {
GstMapInfo map;
} SpiceGstVideoBuffer;
+typedef struct {
+ uint32_t mm_time;
+ uint32_t size;
+} SpiceGstFrameInformation;
+
typedef struct SpiceGstEncoder {
VideoEncoder base;
@@ -91,6 +97,44 @@ typedef struct SpiceGstEncoder {
GCond outbuf_cond;
VideoBuffer *outbuf;
+ /* ---------- Encoded frame statistics ---------- */
+
+ /* Should be >= than FRAME_STATISTICS_COUNT. This is also used to
+ * annotate the client report debug traces with bit rate information.
+ */
+# define SPICE_GST_HISTORY_SIZE 60
+
+ /* A circular buffer containing the past encoded frames information. */
+ SpiceGstFrameInformation history[SPICE_GST_HISTORY_SIZE];
+
+ /* The indices of the oldest and newest frames in the history buffer. */
+ uint32_t history_first;
+ uint32_t history_last;
+
+ /* How many frames to take into account when computing the effective
+ * bit rate, average frame size, etc. This should be large enough so the
+ * I and P frames average out, and short enough for it to reflect the
+ * current situation.
+ */
+# define SPICE_GST_FRAME_STATISTICS_COUNT 21
+
+ /* The index of the oldest frame taken into account for the statistics. */
+ uint32_t stat_first;
+
+ /* Used to compute the average frame size. */
+ uint64_t stat_size_sum;
+
+ /* Tracks the maximum frame size. */
+ uint32_t stat_size_max;
+
+
+ /* ---------- Encoder bit rate control ----------
+ *
+ * GStreamer encoders don't follow the specified bit rate very
+ * closely. These fields are used to ensure we don't exceed the desired
+ * stream bit rate, regardless of the GStreamer encoder's output.
+ */
+
/* The bit rate target for the outgoing network stream. (bits per second) */
uint64_t bit_rate;
@@ -99,6 +143,27 @@ typedef struct SpiceGstEncoder {
/* The default bit rate. */
# define SPICE_GST_DEFAULT_BITRATE (8 * 1024 * 1024)
+
+ /* The bit rate control is performed using a virtual buffer to allow
+ * short term variations: bursts are allowed until the virtual buffer is
+ * full. Then frames are dropped to limit the bit rate. VBUFFER_SIZE
+ * defines the size of the virtual buffer in milliseconds worth of data.
+ */
+# define SPICE_GST_VBUFFER_SIZE 300
+
+ int32_t vbuffer_size;
+ int32_t vbuffer_free;
+
+ /* When dropping frames, this is set to the minimum mm_time of the next
+ * frame to encode. Otherwise set to zero.
+ */
+ uint32_t next_frame_mm_time;
+
+ /* Defines the minimum allowed fps. */
+# define SPICE_GST_MAX_PERIOD (NSEC_PER_SEC / 3)
+
+ /* How big of a margin to take to cover for latency jitter. */
+# define SPICE_GST_LATENCY_MARGIN 0.1
} SpiceGstEncoder;
@@ -138,6 +203,18 @@ static uint32_t get_source_fps(SpiceGstEncoder *encoder)
encoder->cbs.get_source_fps(encoder->cbs.opaque) : SPICE_GST_DEFAULT_FPS;
}
+static uint32_t get_network_latency(SpiceGstEncoder *encoder)
+{
+ /* Assume that the network latency is symmetric */
+ return encoder->cbs.get_roundtrip_ms ?
+ encoder->cbs.get_roundtrip_ms(encoder->cbs.opaque) / 2 : 0;
+}
+
+static inline int rate_control_is_active(SpiceGstEncoder* encoder)
+{
+ return encoder->cbs.get_roundtrip_ms != NULL;
+}
+
static void set_pipeline_changes(SpiceGstEncoder *encoder, uint32_t flags)
{
encoder->set_pipeline |= flags;
@@ -159,6 +236,180 @@ static void free_pipeline(SpiceGstEncoder *encoder)
}
}
+
+/* ---------- Encoded frame statistics ---------- */
+
+static inline uint32_t get_last_frame_mm_time(SpiceGstEncoder *encoder)
+{
+ return encoder->history[encoder->history_last].mm_time;
+}
+
+/* Returns the current bit rate based on the last
+ * SPICE_GST_FRAME_STATISTICS_COUNT frames.
+ */
+static uint64_t get_effective_bit_rate(SpiceGstEncoder *encoder)
+{
+ uint32_t next_mm_time = encoder->next_frame_mm_time ?
+ encoder->next_frame_mm_time :
+ get_last_frame_mm_time(encoder) +
+ MSEC_PER_SEC / get_source_fps(encoder);
+ uint32_t elapsed = next_mm_time - encoder->history[encoder->stat_first].mm_time;
+ return elapsed ? encoder->stat_size_sum * 8 * MSEC_PER_SEC / elapsed : 0;
+}
+
+static uint64_t get_average_frame_size(SpiceGstEncoder *encoder)
+{
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? SPICE_GST_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ return encoder->stat_size_sum / count;
+}
+
+static uint32_t get_maximum_frame_size(SpiceGstEncoder *encoder)
+{
+ if (encoder->stat_size_max == 0) {
+ uint32_t index = encoder->history_last;
+ while (1) {
+ encoder->stat_size_max = MAX(encoder->stat_size_max,
+ encoder->history[index].size);
+ if (index == encoder->stat_first) {
+ break;
+ }
+ index = (index ? index : SPICE_GST_HISTORY_SIZE) - 1;
+ }
+ }
+ return encoder->stat_size_max;
+}
+
+/* Returns the bit rate of the specified period. from and to must be the
+ * mm time of the first and last frame to consider.
+ */
+static uint64_t get_period_bit_rate(SpiceGstEncoder *encoder, uint32_t from,
+ uint32_t to)
+{
+ uint32_t sum = 0;
+ uint32_t last_mm_time = 0;
+ uint32_t index = encoder->history_last;
+ while (1) {
+ if (encoder->history[index].mm_time == to) {
+ if (last_mm_time == 0) {
+ /* We don't know how much time elapsed between the period's
+ * last frame and the next so we cannot include it.
+ */
+ sum = 1;
+ last_mm_time = to;
+ } else {
+ sum = encoder->history[index].size + 1;
+ }
+
+ } else if (encoder->history[index].mm_time == from) {
+ sum += encoder->history[index].size;
+ return (sum - 1) * 8 * MSEC_PER_SEC / (last_mm_time - from);
+
+ } else if (index == encoder->history_first) {
+ /* This period is outside the recorded history */
+ spice_debug("period (%u-%u) outside known history (%u-%u)",
+ from, to,
+ encoder->history[encoder->history_first].mm_time,
+ encoder->history[encoder->history_last].mm_time);
+ return 0;
+
+ } else if (sum > 0) {
+ sum += encoder->history[index].size;
+
+ } else {
+ last_mm_time = encoder->history[index].mm_time;
+ }
+ index = (index ? index : SPICE_GST_HISTORY_SIZE) - 1;
+ }
+
+}
+
+static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
+ uint32_t size)
+{
+ /* Update the statistics */
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? SPICE_GST_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ if (count == SPICE_GST_FRAME_STATISTICS_COUNT) {
+ encoder->stat_size_sum -= encoder->history[encoder->stat_first].size;
+ if (encoder->stat_size_max == encoder->history[encoder->stat_first].size) {
+ encoder->stat_size_max = 0;
+ }
+ encoder->stat_first = (encoder->stat_first + 1) % SPICE_GST_HISTORY_SIZE;
+ }
+ encoder->stat_size_sum += size;
+ if (encoder->stat_size_max > 0 && size > encoder->stat_size_max) {
+ encoder->stat_size_max = size;
+ }
+
+ /* Update the frame history */
+ encoder->history_last = (encoder->history_last + 1) % SPICE_GST_HISTORY_SIZE;
+ if (encoder->history_last == encoder->history_first) {
+ encoder->history_first = (encoder->history_first + 1) % SPICE_GST_HISTORY_SIZE;
+ }
+ encoder->history[encoder->history_last].mm_time = frame_mm_time;
+ encoder->history[encoder->history_last].size = size;
+}
+
+
+/* ---------- Encoder bit rate control ---------- */
+
+static uint32_t get_min_playback_delay(SpiceGstEncoder *encoder)
+{
+ /* Make sure the delay is large enough to send a large frame (typically
+ * an I frame) and an average frame. This also takes into account the
+ * frames dropped by the encoder bit rate control.
+ */
+ uint32_t size = get_maximum_frame_size(encoder) + get_average_frame_size(encoder);
+ uint32_t send_time = MSEC_PER_SEC * size * 8 / encoder->bit_rate;
+
+ /* Also factor in the network latency with a margin for jitter. */
+ uint32_t net_latency = get_network_latency(encoder) * (1.0 + SPICE_GST_LATENCY_MARGIN);
+
+ return send_time + net_latency;
+}
+
+static void update_client_playback_delay(SpiceGstEncoder *encoder)
+{
+ if (encoder->cbs.update_client_playback_delay) {
+ uint32_t min_delay = get_min_playback_delay(encoder);
+ encoder->cbs.update_client_playback_delay(encoder->cbs.opaque, min_delay);
+ }
+}
+
+static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
+{
+ if (encoder->vbuffer_free >= 0) {
+ encoder->next_frame_mm_time = 0;
+ return;
+ }
+
+ /* Figure out how many frames to drop to not exceed the current bit rate.
+ * Use nanoseconds to avoid precision loss.
+ */
+ uint64_t delay_ns = -encoder->vbuffer_free * 8 * NSEC_PER_SEC / encoder->bit_rate;
+ uint64_t period_ns = NSEC_PER_SEC / get_source_fps(encoder);
+ uint32_t drops = (delay_ns + period_ns - 1) / period_ns; /* round up */
+ spice_debug("drops=%u vbuffer %d/%d", drops, encoder->vbuffer_free,
+ encoder->vbuffer_size);
+
+ delay_ns = drops * period_ns + period_ns / 2;
+ if (delay_ns > SPICE_GST_MAX_PERIOD) {
+ delay_ns = SPICE_GST_MAX_PERIOD;
+ }
+ encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) + delay_ns / NSEC_PER_MILLISEC;
+
+ /* Drops mean a higher delay between encoded frames so update the
+ * playback delay.
+ */
+ update_client_playback_delay(encoder);
+}
+
+
+/* ---------- Network bit rate control ---------- */
+
/* The maximum bit rate we will use for the current video.
*
* This is based on a 10x compression ratio which should be more than enough
@@ -776,11 +1027,22 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
encoder->spice_format = bitmap->format;
encoder->width = width;
encoder->height = height;
- if (encoder->pipeline) {
+ if (encoder->bit_rate == 0) {
+ encoder->history[0].mm_time = frame_mm_time;
+ encoder->bit_rate = encoder->starting_bit_rate;
+ adjust_bit_rate(encoder);
+ encoder->vbuffer_free = 0; /* Slow start */
+ } else if (encoder->pipeline) {
set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_CAPS);
}
}
+ if (rate_control_is_active(encoder) &&
+ frame_mm_time < encoder->next_frame_mm_time) {
+ /* Drop the frame to limit the outgoing bit rate. */
+ return VIDEO_ENCODER_FRAME_DROP;
+ }
+
if (!configure_pipeline(encoder)) {
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
@@ -800,6 +1062,14 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
/* Unref the last frame's bitmap_opaque structures if any */
clear_zero_copy_queue(encoder, FALSE);
+
+ if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ return rc;
+ }
+ add_frame(encoder, frame_mm_time, (*outbuf)->size);
+
+ update_next_frame_mm_time(encoder);
+
return rc;
}
@@ -811,10 +1081,13 @@ static void spice_gst_encoder_client_stream_report(VideoEncoder *video_encoder,
int32_t end_frame_delay,
uint32_t audio_delay)
{
- spice_debug("client report: #frames %u, #drops %d, duration %u video-delay %d audio-delay %u",
- num_frames, num_drops,
- end_frame_mm_time - start_frame_mm_time,
- end_frame_delay, audio_delay);
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ uint64_t period_bit_rate = get_period_bit_rate(encoder, start_frame_mm_time, end_frame_mm_time);
+ spice_debug("client report: %u/%u drops in %ums margins video %3d audio %3u bw %.3f/%.3fMbps",
+ num_drops, num_frames, end_frame_mm_time - start_frame_mm_time,
+ end_frame_delay, audio_delay,
+ get_mbps(period_bit_rate),
+ get_mbps(get_effective_bit_rate(encoder)));
}
static void spice_gst_encoder_notify_server_frame_drop(VideoEncoder *video_encoder)
@@ -825,7 +1098,7 @@ static void spice_gst_encoder_notify_server_frame_drop(VideoEncoder *video_encod
static uint64_t spice_gst_encoder_get_bit_rate(VideoEncoder *video_encoder)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
- return encoder->bit_rate;
+ return get_effective_bit_rate(encoder);
}
static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
@@ -836,7 +1109,7 @@ static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
spice_return_if_fail(stats != NULL);
stats->starting_bit_rate = encoder->starting_bit_rate;
- stats->cur_bit_rate = encoder->bit_rate;
+ stats->cur_bit_rate = get_effective_bit_rate(encoder);
/* Use the compression level as a proxy for the quality */
stats->avg_quality = stats->cur_bit_rate ? 100.0 - raw_bit_rate / stats->cur_bit_rate : 0;
@@ -851,6 +1124,7 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
bitmap_ref_t bitmap_ref,
bitmap_unref_t bitmap_unref)
{
+ spice_return_val_if_fail(SPICE_GST_FRAME_STATISTICS_COUNT <= SPICE_GST_HISTORY_SIZE, NULL);
spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP8 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H264, NULL);
commit a21ec4e2516c9731d7fda4f00b6a0f8779d58e01
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Wed May 20 07:58:34 2015 +0200
streaming: Add h264 support to the GStreamer video encoder
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/configure.ac b/configure.ac
index e6be07a..1c8b0d7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -79,6 +79,7 @@ if test "x$enable_gstreamer" != "xno"; then
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-base 1.0], [appsrc videoconvert appsink])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gstreamer-libav 1.0], [avenc_mjpeg])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-good 1.0], [vp8enc])
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-ugly 1.0], [x264enc])
],
[if test "x$enable_gstreamer" = "xyes"; then
AC_MSG_ERROR([GStreamer 1.0 support requested but not found. You may set GSTREAMER_1_0_CFLAGS and GSTREAMER_1_0_LIBS to avoid the need to call pkg-config.])
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 9242814..e79d9c5 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -308,6 +308,15 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
gstenc = g_strdup_printf("vp8enc end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4");
break;
}
+ case SPICE_VIDEO_CODEC_TYPE_H264:
+ /* - Set tune and sliced-threads to ensure a zero-frame latency
+ * - qp-min ensures the bitrate does not get needlessly high.
+ * - Set speed-preset to get realtime speed.
+ * - Set intra-refresh to get more uniform compressed frame sizes,
+ * thus helping with streaming.
+ */
+ gstenc = g_strdup("x264enc byte-stream=true aud=true qp-min=15 tune=4 sliced-threads=true speed-preset=ultrafast intra-refresh=true");
+ break;
default:
/* gstreamer_encoder_new() should have rejected this codec type */
spice_warning("unsupported codec type %d", encoder->base.codec_type);
@@ -378,6 +387,11 @@ static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
"target-bitrate", (gint)encoder->bit_rate,
NULL);
break;
+ case SPICE_VIDEO_CODEC_TYPE_H264:
+ g_object_set(G_OBJECT(encoder->gstenc),
+ "bitrate", encoder->bit_rate / 1024,
+ NULL);
+ break;
default:
/* gstreamer_encoder_new() should have rejected this codec type */
spice_warning("unsupported codec type %d", encoder->base.codec_type);
@@ -838,7 +852,8 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
bitmap_unref_t bitmap_unref)
{
spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG ||
- codec_type == SPICE_VIDEO_CODEC_TYPE_VP8, NULL);
+ codec_type == SPICE_VIDEO_CODEC_TYPE_VP8 ||
+ codec_type == SPICE_VIDEO_CODEC_TYPE_H264, NULL);
GError *err = NULL;
if (!gst_init_check(NULL, NULL, &err)) {
diff --git a/server/reds.c b/server/reds.c
index b4ad4bb..f3b5874 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -3492,7 +3492,7 @@ err:
}
static const char default_renderer[] = "sw";
-static const char default_video_codecs[] = "spice:mjpeg;gstreamer:mjpeg;gstreamer:vp8";
+static const char default_video_codecs[] = "spice:mjpeg;gstreamer:mjpeg;gstreamer:h264;gstreamer:vp8";
/* new interface */
SPICE_GNUC_VISIBLE SpiceServer *spice_server_new(void)
@@ -3580,12 +3580,14 @@ static new_video_encoder_t video_encoder_procs[] = {
static const EnumNames video_codec_names[] = {
{SPICE_VIDEO_CODEC_TYPE_MJPEG, "mjpeg"},
{SPICE_VIDEO_CODEC_TYPE_VP8, "vp8"},
+ {SPICE_VIDEO_CODEC_TYPE_H264, "h264"},
{0, NULL},
};
static int video_codec_caps[] = {
SPICE_DISPLAY_CAP_CODEC_MJPEG,
SPICE_DISPLAY_CAP_CODEC_VP8,
+ SPICE_DISPLAY_CAP_CODEC_H264,
};
commit 49036e6aae86c1b32045e81dee1915cd47e064bf
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Tue May 24 18:37:46 2016 +0200
streaming: Avoid copying the input frame in the GStreamer encoder
Note that we can only avoid copies for the first 1 Mpixels or so.
That's because Spice splits larger frames into more chunks than we can
fit GstMemory fragments in a GStreamer buffer. So if there are more
pixels we will avoid copies for the first 3840 KB and copy the rest.
Furthermore, while in practice the GStreamer encoder will only modify
the RedDrawable refcount during the encode_frame(), in theory the
refcount could be decremented from the GStreamer thread after
encode_frame() returns.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/dcc-send.c b/server/dcc-send.c
index 472575d..9ed3a20 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1678,7 +1678,8 @@ static void red_release_video_encoder_buffer(uint8_t *data, void *opaque)
}
static int red_marshall_stream_data(RedChannelClient *rcc,
- SpiceMarshaller *base_marshaller, Drawable *drawable)
+ SpiceMarshaller *base_marshaller,
+ Drawable *drawable)
{
DisplayChannelClient *dcc = RCC_TO_DCC(rcc);
DisplayChannel *display = DCC_TO_DC(dcc);
@@ -1727,6 +1728,7 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
frame_mm_time,
©->src_bitmap->u.bitmap,
©->src_area, stream->top_down,
+ drawable->red_drawable,
&outbuf);
switch (ret) {
case VIDEO_ENCODER_FRAME_DROP:
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 08a1afa..9242814 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -30,6 +30,8 @@
#define SPICE_GST_DEFAULT_FPS 30
+#define DO_ZERO_COPY
+
typedef struct {
SpiceBitmapFmt spice_format;
@@ -46,6 +48,14 @@ typedef struct SpiceGstVideoBuffer {
typedef struct SpiceGstEncoder {
VideoEncoder base;
+ /* Callbacks to adjust the refcount of the bitmap being encoded. */
+ bitmap_ref_t bitmap_ref;
+ bitmap_unref_t bitmap_unref;
+
+#ifdef DO_ZERO_COPY
+ GAsyncQueue *unused_bitmap_opaques;
+#endif
+
/* Rate control callbacks */
VideoEncoderRateControlCbs cbs;
@@ -460,12 +470,109 @@ static inline int line_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
return TRUE;
}
+#ifdef DO_ZERO_COPY
+typedef struct {
+ gint refs;
+ SpiceGstEncoder *encoder;
+ gpointer opaque;
+} BitmapWrapper;
+
+static void clear_zero_copy_queue(SpiceGstEncoder *encoder, gboolean unref_queue)
+{
+ gpointer bitmap_opaque;
+ while ((bitmap_opaque = g_async_queue_try_pop(encoder->unused_bitmap_opaques))) {
+ encoder->bitmap_unref(bitmap_opaque);
+ }
+ if (unref_queue) {
+ g_async_queue_unref(encoder->unused_bitmap_opaques);
+ }
+}
+
+static BitmapWrapper *bitmap_wrapper_new(SpiceGstEncoder *encoder, gpointer bitmap_opaque)
+{
+ BitmapWrapper *wrapper = spice_new(BitmapWrapper, 1);
+ wrapper->refs = 1;
+ wrapper->encoder = encoder;
+ wrapper->opaque = bitmap_opaque;
+ encoder->bitmap_ref(bitmap_opaque);
+ return wrapper;
+}
+
+static void bitmap_wrapper_unref(gpointer data)
+{
+ BitmapWrapper *wrapper = data;
+ if (g_atomic_int_dec_and_test(&wrapper->refs)) {
+ g_async_queue_push(wrapper->encoder->unused_bitmap_opaques, wrapper->opaque);
+ free(wrapper);
+ }
+}
+
+
+/* A helper for push_raw_frame() */
+static inline int zero_copy(SpiceGstEncoder *encoder,
+ const SpiceBitmap *bitmap, gpointer bitmap_opaque,
+ GstBuffer *buffer, uint32_t *chunk_index,
+ uint32_t *chunk_offset, uint32_t *len)
+{
+ const SpiceChunks *chunks = bitmap->data;
+ while (*chunk_index < chunks->num_chunks &&
+ *chunk_offset >= chunks->chunk[*chunk_index].len) {
+ if (!is_chunk_stride_aligned(bitmap, *chunk_index)) {
+ return FALSE;
+ }
+ *chunk_offset -= chunks->chunk[*chunk_index].len;
+ (*chunk_index)++;
+ }
+
+ int max_block_count = gst_buffer_get_max_memory();
+ if (chunks->num_chunks - *chunk_index > max_block_count) {
+ /* There are more chunks than we can fit memory objects in a
+ * buffer. This will cause the buffer to merge memory objects to
+ * fit the extra chunks, which means doing wasteful memory copies.
+ * So use the zero-copy approach for the first max_mem-1 chunks, and
+ * let push_raw_frame() deal with the rest.
+ */
+ max_block_count = *chunk_index + max_block_count - 1;
+ } else {
+ max_block_count = chunks->num_chunks;
+ }
+
+ BitmapWrapper *wrapper = NULL;
+ while (*len && *chunk_index < max_block_count) {
+ if (!is_chunk_stride_aligned(bitmap, *chunk_index)) {
+ return FALSE;
+ }
+ if (wrapper) {
+ g_atomic_int_inc(&wrapper->refs);
+ } else {
+ wrapper = bitmap_wrapper_new(encoder, bitmap_opaque);
+ }
+ uint32_t thislen = MIN(chunks->chunk[*chunk_index].len - *chunk_offset, *len);
+ GstMemory *mem = gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY,
+ chunks->chunk[*chunk_index].data,
+ chunks->chunk[*chunk_index].len,
+ *chunk_offset, thislen,
+ wrapper, bitmap_wrapper_unref);
+ gst_buffer_append_memory(buffer, mem);
+ *len -= thislen;
+ *chunk_offset = 0;
+ (*chunk_index)++;
+ }
+ return TRUE;
+}
+#else
+static void clear_zero_copy_queue(SpiceGstEncoder *encoder, gboolean unref_queue)
+{
+ /* Nothing to do */
+}
+#endif
+
/* A helper for push_raw_frame() */
static inline int chunk_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
- uint32_t chunk_offset, uint32_t len, uint8_t *dst)
+ uint32_t chunk_index, uint32_t chunk_offset,
+ uint32_t len, uint8_t *dst)
{
SpiceChunks *chunks = bitmap->data;
- uint32_t chunk_index = 0;
/* Skip chunks until we find the start of the frame */
while (chunk_index < chunks->num_chunks &&
chunk_offset >= chunks->chunk[chunk_index].len) {
@@ -493,17 +600,34 @@ static inline int chunk_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap
return TRUE;
}
+/* A helper for push_raw_frame() */
+static uint8_t *allocate_and_map_memory(gsize size, GstMapInfo *map, GstBuffer *buffer)
+{
+ GstMemory *mem = gst_allocator_alloc(NULL, size, NULL);
+ if (!mem) {
+ gst_buffer_unref(buffer);
+ return NULL;
+ }
+ if (!gst_memory_map(mem, map, GST_MAP_WRITE)) {
+ gst_memory_unref(mem);
+ gst_buffer_unref(buffer);
+ return NULL;
+ }
+ return map->data;
+}
+
/* A helper for spice_gst_encoder_encode_frame() */
-static int push_raw_frame(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
- const SpiceRect *src, int top_down)
+static int push_raw_frame(SpiceGstEncoder *encoder,
+ const SpiceBitmap *bitmap,
+ const SpiceRect *src, int top_down,
+ gpointer bitmap_opaque)
{
uint32_t height = src->bottom - src->top;
uint32_t stream_stride = (src->right - src->left) * encoder->format->bpp / 8;
uint32_t len = stream_stride * height;
- GstBuffer *buffer = gst_buffer_new_and_alloc(len);
- GstMapInfo map;
- gst_buffer_map(buffer, &map, GST_MAP_WRITE);
- uint8_t *dst = map.data;
+ GstBuffer *buffer = gst_buffer_new();
+ /* TODO Use GST_MAP_INFO_INIT once GStreamer 1.4.5 is no longer relevant */
+ GstMapInfo map = { .memory = NULL };
/* Note that we should not reorder the lines, even if top_down is false.
* It just changes the number of lines to skip at the start of the bitmap.
@@ -515,21 +639,51 @@ static int push_raw_frame(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
/* We have to do a line-by-line copy because for each we have to
* leave out pixels on the left or right.
*/
+ uint8_t *dst = allocate_and_map_memory(len, &map, buffer);
+ if (!dst) {
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+
chunk_offset += src->left * encoder->format->bpp / 8;
if (!line_copy(encoder, bitmap, chunk_offset, stream_stride, height, dst)) {
- gst_buffer_unmap(buffer, &map);
+ gst_memory_unmap(map.memory, &map);
+ gst_memory_unref(map.memory);
gst_buffer_unref(buffer);
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
} else {
/* We can copy the bitmap chunk by chunk */
- if (!chunk_copy(encoder, bitmap, chunk_offset, len, dst)) {
- gst_buffer_unmap(buffer, &map);
+ uint32_t chunk_index = 0;
+#ifdef DO_ZERO_COPY
+ if (!zero_copy(encoder, bitmap, bitmap_opaque, buffer, &chunk_index,
+ &chunk_offset, &len)) {
gst_buffer_unref(buffer);
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
+ /* len now contains the remaining number of bytes to copy.
+ * However we must avoid any write to the GstBuffer object as it
+ * would cause a copy of the read-only memory objects we just added.
+ * Fortunately we can append extra writable memory objects instead.
+ */
+#endif
+
+ if (len) {
+ uint8_t *dst = allocate_and_map_memory(len, &map, buffer);
+ if (!dst) {
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+ if (!chunk_copy(encoder, bitmap, chunk_index, chunk_offset, len, dst)) {
+ gst_memory_unmap(map.memory, &map);
+ gst_memory_unref(map.memory);
+ gst_buffer_unref(buffer);
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+ }
+ }
+ if (map.memory) {
+ gst_memory_unmap(map.memory, &map);
+ gst_buffer_append_memory(buffer, map.memory);
}
- gst_buffer_unmap(buffer, &map);
GstFlowReturn ret = gst_app_src_push_buffer(encoder->appsrc, buffer);
if (ret != GST_FLOW_OK) {
@@ -573,6 +727,9 @@ static void spice_gst_encoder_destroy(VideoEncoder *video_encoder)
g_mutex_clear(&encoder->outbuf_mutex);
g_cond_clear(&encoder->outbuf_cond);
+ /* Unref any lingering bitmap opaque structures from past frames */
+ clear_zero_copy_queue(encoder, TRUE);
+
free(encoder);
}
@@ -580,12 +737,16 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
+ gpointer bitmap_opaque,
VideoBuffer **outbuf)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
g_return_val_if_fail(outbuf != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
*outbuf = NULL;
+ /* Unref the last frame's bitmap_opaque structures if any */
+ clear_zero_copy_queue(encoder, FALSE);
+
uint32_t width = src->right - src->left;
uint32_t height = src->bottom - src->top;
if (width != encoder->width || height != encoder->height ||
@@ -610,7 +771,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
- int rc = push_raw_frame(encoder, bitmap, src, top_down);
+ int rc = push_raw_frame(encoder, bitmap, src, top_down, bitmap_opaque);
if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
rc = pull_compressed_buffer(encoder, outbuf);
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
@@ -622,6 +783,9 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
free_pipeline(encoder);
}
}
+
+ /* Unref the last frame's bitmap_opaque structures if any */
+ clear_zero_copy_queue(encoder, FALSE);
return rc;
}
@@ -669,7 +833,9 @@ static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
- VideoEncoderRateControlCbs *cbs)
+ VideoEncoderRateControlCbs *cbs,
+ bitmap_ref_t bitmap_ref,
+ bitmap_unref_t bitmap_unref)
{
spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP8, NULL);
@@ -689,11 +855,16 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
encoder->base.get_bit_rate = spice_gst_encoder_get_bit_rate;
encoder->base.get_stats = spice_gst_encoder_get_stats;
encoder->base.codec_type = codec_type;
+#ifdef DO_ZERO_COPY
+ encoder->unused_bitmap_opaques = g_async_queue_new();
+#endif
if (cbs) {
encoder->cbs = *cbs;
}
encoder->starting_bit_rate = starting_bit_rate;
+ encoder->bitmap_ref = bitmap_ref;
+ encoder->bitmap_unref = bitmap_unref;
g_mutex_init(&encoder->outbuf_mutex);
g_cond_init(&encoder->outbuf_cond);
diff --git a/server/mjpeg-encoder.c b/server/mjpeg-encoder.c
index 2db6ca9..1649516 100644
--- a/server/mjpeg-encoder.c
+++ b/server/mjpeg-encoder.c
@@ -948,6 +948,7 @@ static int mjpeg_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
+ gpointer bitmap_opaque,
VideoBuffer **outbuf)
{
MJpegEncoder *encoder = (MJpegEncoder*)video_encoder;
@@ -1367,7 +1368,9 @@ static void mjpeg_encoder_get_stats(VideoEncoder *video_encoder,
VideoEncoder *mjpeg_encoder_new(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
- VideoEncoderRateControlCbs *cbs)
+ VideoEncoderRateControlCbs *cbs,
+ bitmap_ref_t bitmap_ref,
+ bitmap_unref_t bitmap_unref)
{
MJpegEncoder *encoder = spice_new0(MJpegEncoder, 1);
diff --git a/server/stream.c b/server/stream.c
index 3d3e627..8a137a1 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -706,6 +706,18 @@ static void update_client_playback_delay(void *opaque, uint32_t delay_ms)
agent->dcc->streams_max_latency);
}
+void bitmap_ref(gpointer data)
+{
+ RedDrawable *red_drawable = (RedDrawable*)data;
+ red_drawable_ref(red_drawable);
+}
+
+void bitmap_unref(gpointer data)
+{
+ RedDrawable *red_drawable = (RedDrawable*)data;
+ red_drawable_unref(red_drawable);
+}
+
/* A helper for dcc_create_stream(). */
static VideoEncoder* dcc_create_video_encoder(DisplayChannelClient *dcc,
uint64_t starting_bit_rate,
@@ -730,7 +742,7 @@ static VideoEncoder* dcc_create_video_encoder(DisplayChannelClient *dcc,
continue;
}
- VideoEncoder* video_encoder = video_codec->create(video_codec->type, starting_bit_rate, cbs);
+ VideoEncoder* video_encoder = video_codec->create(video_codec->type, starting_bit_rate, cbs, bitmap_ref, bitmap_unref);
if (video_encoder) {
return video_encoder;
}
@@ -738,7 +750,7 @@ static VideoEncoder* dcc_create_video_encoder(DisplayChannelClient *dcc,
/* Try to use the builtin MJPEG video encoder as a fallback */
if (!client_has_multi_codec || red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_CODEC_MJPEG)) {
- return mjpeg_encoder_new(SPICE_VIDEO_CODEC_TYPE_MJPEG, starting_bit_rate, cbs);
+ return mjpeg_encoder_new(SPICE_VIDEO_CODEC_TYPE_MJPEG, starting_bit_rate, cbs, bitmap_ref, bitmap_unref);
}
return NULL;
diff --git a/server/video-encoder.h b/server/video-encoder.h
index f4be396..3423118 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -65,6 +65,8 @@ struct VideoEncoder {
* @bitmap: A bitmap containing the source video frame.
* @src: A rectangle specifying the area occupied by the video.
* @top_down: If true the first video line is specified by src.top.
+ * @bitmap_opaque: The parameter for the bitmap_ref() and bitmap_unref()
+ * callbacks.
* @outbuf: A pointer to a VideoBuffer structure containing the
* compressed frame if successful. Call the buffer's
* free() method as soon as it is no longer needed.
@@ -77,7 +79,7 @@ struct VideoEncoder {
int (*encode_frame)(VideoEncoder *encoder, uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
- VideoBuffer** outbuf);
+ gpointer bitmap_opaque, VideoBuffer** outbuf);
/*
* Bit rate control methods.
@@ -166,6 +168,8 @@ typedef struct VideoEncoderRateControlCbs {
void (*update_client_playback_delay)(void *opaque, uint32_t delay_ms);
} VideoEncoderRateControlCbs;
+typedef void (*bitmap_ref_t)(gpointer data);
+typedef void (*bitmap_unref_t)(gpointer data);
/* Instantiates the video encoder.
*
@@ -173,22 +177,35 @@ typedef struct VideoEncoderRateControlCbs {
* @starting_bit_rate: An initial estimate of the available stream bit rate
* or zero if the client does not support rate control.
* @cbs: A set of callback methods to be used for rate control.
+ * @bitmap_ref: A callback that the encoder can use to increase the
+ * bitmap refcount.
+ * This must be called from the main context.
+ * @bitmap_unref: A callback that the encoder can use to decrease the
+ * bitmap refcount.
+ * This must be called from the main context.
* @return: A pointer to a structure implementing the VideoEncoder
* methods.
*/
typedef VideoEncoder* (*new_video_encoder_t)(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
- VideoEncoderRateControlCbs *cbs);
+ VideoEncoderRateControlCbs *cbs,
+ bitmap_ref_t bitmap_ref,
+ bitmap_unref_t bitmap_unref);
VideoEncoder* mjpeg_encoder_new(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
- VideoEncoderRateControlCbs *cbs);
+ VideoEncoderRateControlCbs *cbs,
+ bitmap_ref_t bitmap_ref,
+ bitmap_unref_t bitmap_unref);
#ifdef HAVE_GSTREAMER_1_0
VideoEncoder* gstreamer_encoder_new(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
- VideoEncoderRateControlCbs *cbs);
+ VideoEncoderRateControlCbs *cbs,
+ bitmap_ref_t bitmap_ref,
+ bitmap_unref_t bitmap_unref);
#endif
+
typedef struct RedVideoCodec {
new_video_encoder_t create;
SpiceVideoCodecType type;
commit a6795d1971ff658c1591ec635802f8fe7a78965d
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Tue Apr 12 20:35:19 2016 +0200
streaming: Handle and recover from GStreamer encoding errors
If an error occurs for whatever reason (e.g. codec not supporting odd
frame sizes), the GStreamer pipeline will drop the current buffer,
causing the encoder to be stuck waiting for the sample. So this patch
tracks error notifications and ensures we don't wait for a sample if
none will come.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 8a44c41..08a1afa 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -76,6 +76,11 @@ typedef struct SpiceGstEncoder {
# define SPICE_GST_VIDEO_PIPELINE_CAPS 0x4
uint32_t set_pipeline;
+ /* Output buffer */
+ GMutex outbuf_mutex;
+ GCond outbuf_cond;
+ VideoBuffer *outbuf;
+
/* The bit rate target for the outgoing network stream. (bits per second) */
uint64_t bit_rate;
@@ -215,6 +220,56 @@ static void set_appsrc_caps(SpiceGstEncoder *encoder)
gst_app_src_set_caps(encoder->appsrc, encoder->src_caps);
}
+static GstBusSyncReply handle_pipeline_message(GstBus *bus, GstMessage *msg, gpointer video_encoder)
+{
+ SpiceGstEncoder *encoder = video_encoder;
+
+ if (GST_MESSAGE_TYPE(msg) == GST_MESSAGE_ERROR) {
+ GError *err = NULL;
+ gchar *debug_info = NULL;
+ gst_message_parse_error(msg, &err, &debug_info);
+ spice_warning("GStreamer error from element %s: %s",
+ GST_OBJECT_NAME(msg->src), err->message);
+ if (debug_info) {
+ spice_debug("debug details: %s", debug_info);
+ g_free(debug_info);
+ }
+ g_clear_error(&err);
+
+ /* Unblock the main thread */
+ g_mutex_lock(&encoder->outbuf_mutex);
+ encoder->outbuf = (VideoBuffer*)create_gst_video_buffer();
+ g_cond_signal(&encoder->outbuf_cond);
+ g_mutex_unlock(&encoder->outbuf_mutex);
+ }
+ return GST_BUS_PASS;
+}
+
+static GstFlowReturn new_sample(GstAppSink *gstappsink, gpointer video_encoder)
+{
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ SpiceGstVideoBuffer *outbuf = create_gst_video_buffer();
+
+ GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
+ if (sample) {
+ outbuf->gst_buffer = gst_sample_get_buffer(sample);
+ gst_buffer_ref(outbuf->gst_buffer);
+ gst_sample_unref(sample);
+ if (gst_buffer_map(outbuf->gst_buffer, &outbuf->map, GST_MAP_READ)) {
+ outbuf->base.data = outbuf->map.data;
+ outbuf->base.size = gst_buffer_get_size(outbuf->gst_buffer);
+ }
+ }
+
+ /* Notify the main thread that the output buffer is ready */
+ g_mutex_lock(&encoder->outbuf_mutex);
+ encoder->outbuf = (VideoBuffer*)outbuf;
+ g_cond_signal(&encoder->outbuf_cond);
+ g_mutex_unlock(&encoder->outbuf_mutex);
+
+ return GST_FLOW_OK;
+}
+
static gboolean create_pipeline(SpiceGstEncoder *encoder)
{
gchar *gstenc;
@@ -268,6 +323,22 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
encoder->gstenc = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "encoder");
encoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "sink"));
+#ifdef HAVE_GSTREAMER_0_10
+ GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, NULL, {NULL}};
+#else
+ GstAppSinkCallbacks appsink_cbs = {NULL, NULL, &new_sample, {NULL}};
+#endif
+ gst_app_sink_set_callbacks(encoder->appsink, &appsink_cbs, encoder, NULL);
+
+ /* Hook into the bus so we can handle errors */
+ GstBus *bus = gst_element_get_bus(encoder->pipeline);
+#ifdef HAVE_GSTREAMER_0_10
+ gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder);
+#else
+ gst_bus_set_sync_handler(bus, handle_pipeline_message, encoder, NULL);
+#endif
+ gst_object_unref(bus);
+
if (encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG) {
/* See https://bugzilla.gnome.org/show_bug.cgi?id=753257 */
spice_debug("removing the pipeline clock");
@@ -473,23 +544,21 @@ static int push_raw_frame(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
static int pull_compressed_buffer(SpiceGstEncoder *encoder,
VideoBuffer **outbuf)
{
- GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
- if (sample) {
- SpiceGstVideoBuffer *buffer = create_gst_video_buffer();
- buffer->gst_buffer = gst_sample_get_buffer(sample);
- if (buffer->gst_buffer &&
- gst_buffer_map(buffer->gst_buffer, &buffer->map, GST_MAP_READ)) {
- buffer->base.data = buffer->map.data;
- buffer->base.size = gst_buffer_get_size(buffer->gst_buffer);
- *outbuf = (VideoBuffer*)buffer;
- gst_buffer_ref(buffer->gst_buffer);
- gst_sample_unref(sample);
- return VIDEO_ENCODER_FRAME_ENCODE_DONE;
- }
- buffer->base.free((VideoBuffer*)buffer);
- gst_sample_unref(sample);
+ g_mutex_lock(&encoder->outbuf_mutex);
+ while (!encoder->outbuf) {
+ g_cond_wait(&encoder->outbuf_cond, &encoder->outbuf_mutex);
+ }
+ *outbuf = encoder->outbuf;
+ encoder->outbuf = NULL;
+ g_mutex_unlock(&encoder->outbuf_mutex);
+
+ if ((*outbuf)->data) {
+ return VIDEO_ENCODER_FRAME_ENCODE_DONE;
}
+
spice_debug("failed to pull the compressed buffer");
+ (*outbuf)->free(*outbuf);
+ *outbuf = NULL;
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
@@ -499,7 +568,11 @@ static int pull_compressed_buffer(SpiceGstEncoder *encoder,
static void spice_gst_encoder_destroy(VideoEncoder *video_encoder)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+
free_pipeline(encoder);
+ g_mutex_clear(&encoder->outbuf_mutex);
+ g_cond_clear(&encoder->outbuf_cond);
+
free(encoder);
}
@@ -621,6 +694,8 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
encoder->cbs = *cbs;
}
encoder->starting_bit_rate = starting_bit_rate;
+ g_mutex_init(&encoder->outbuf_mutex);
+ g_cond_init(&encoder->outbuf_cond);
/* All the other fields are initialized to zero by spice_new0(). */
commit 2db59fef3a8d34cf563e9fa9906bf3ffaaf14f66
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Thu Jul 16 16:36:09 2015 +0200
streaming: Let the video encoder manage the compressed buffer
This way the video encoder is not forced to use malloc()/free().
This also allows more flexibility in how the video encoder manages the
buffer which allows for a zero-copy implementation in both video
encoders.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/dcc-send.c b/server/dcc-send.c
index f8087ea..472575d 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1671,6 +1671,12 @@ static void red_lossy_marshall_qxl_draw_text(RedChannelClient *rcc,
}
}
+static void red_release_video_encoder_buffer(uint8_t *data, void *opaque)
+{
+ VideoBuffer *buffer = (VideoBuffer*)opaque;
+ buffer->free(buffer);
+}
+
static int red_marshall_stream_data(RedChannelClient *rcc,
SpiceMarshaller *base_marshaller, Drawable *drawable)
{
@@ -1679,7 +1685,6 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
Stream *stream = drawable->stream;
SpiceCopy *copy;
uint32_t frame_mm_time;
- uint32_t n;
int is_sized;
int ret;
@@ -1701,7 +1706,6 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
StreamAgent *agent = &dcc->stream_agents[get_stream_id(display, stream)];
uint64_t time_now = spice_get_monotonic_time_ns();
- size_t outbuf_size;
if (!dcc->use_video_encoder_rate_control) {
if (time_now - agent->last_send_time < (1000 * 1000 * 1000) / agent->fps) {
@@ -1713,18 +1717,17 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
}
}
+ VideoBuffer *outbuf;
/* workaround for vga streams */
frame_mm_time = drawable->red_drawable->mm_time ?
drawable->red_drawable->mm_time :
reds_get_mm_time();
- outbuf_size = dcc->send_data.stream_outbuf_size;
ret = !agent->video_encoder ? VIDEO_ENCODER_FRAME_UNSUPPORTED :
agent->video_encoder->encode_frame(agent->video_encoder,
frame_mm_time,
©->src_bitmap->u.bitmap,
©->src_area, stream->top_down,
- &dcc->send_data.stream_outbuf,
- &outbuf_size, &n);
+ &outbuf);
switch (ret) {
case VIDEO_ENCODER_FRAME_DROP:
spice_assert(dcc->use_video_encoder_rate_control);
@@ -1740,7 +1743,6 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
spice_error("bad return value (%d) from VideoEncoder::encode_frame", ret);
return FALSE;
}
- dcc->send_data.stream_outbuf_size = outbuf_size;
if (!is_sized) {
SpiceMsgDisplayStreamData stream_data;
@@ -1749,7 +1751,7 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
stream_data.base.id = get_stream_id(display, stream);
stream_data.base.multi_media_time = frame_mm_time;
- stream_data.data_size = n;
+ stream_data.data_size = outbuf->size;
spice_marshall_msg_display_stream_data(base_marshaller, &stream_data);
} else {
@@ -1759,7 +1761,7 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
stream_data.base.id = get_stream_id(display, stream);
stream_data.base.multi_media_time = frame_mm_time;
- stream_data.data_size = n;
+ stream_data.data_size = outbuf->size;
stream_data.width = copy->src_area.right - copy->src_area.left;
stream_data.height = copy->src_area.bottom - copy->src_area.top;
stream_data.dest = drawable->red_drawable->bbox;
@@ -1768,12 +1770,12 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
rect_debug(&stream_data.dest);
spice_marshall_msg_display_stream_data_sized(base_marshaller, &stream_data);
}
- spice_marshaller_add_ref(base_marshaller,
- dcc->send_data.stream_outbuf, n);
+ spice_marshaller_add_ref_full(base_marshaller, outbuf->data, outbuf->size,
+ &red_release_video_encoder_buffer, outbuf);
agent->last_send_time = time_now;
#ifdef STREAM_STATS
agent->stats.num_frames_sent++;
- agent->stats.size_sent += n;
+ agent->stats.size_sent += outbuf->size;
agent->stats.end = frame_mm_time;
#endif
diff --git a/server/dcc.c b/server/dcc.c
index 6511ce5..41aaf1b 100644
--- a/server/dcc.c
+++ b/server/dcc.c
@@ -381,10 +381,6 @@ DisplayChannelClient *dcc_new(DisplayChannel *display,
// TODO: tune quality according to bandwidth
dcc->encoders.jpeg_quality = 85;
- size_t stream_buf_size;
- stream_buf_size = 32*1024;
- dcc->send_data.stream_outbuf = spice_malloc(stream_buf_size);
- dcc->send_data.stream_outbuf_size = stream_buf_size;
dcc->send_data.free_list.res =
spice_malloc(sizeof(SpiceResourceList) +
DISPLAY_FREE_LIST_DEFAULT_SIZE * sizeof(SpiceResourceID));
@@ -491,7 +487,6 @@ void dcc_stop(DisplayChannelClient *dcc)
dcc->pixmap_cache = NULL;
image_encoders_release_glz(&dcc->encoders);
dcc_palette_cache_reset(dcc);
- free(dcc->send_data.stream_outbuf);
free(dcc->send_data.free_list.res);
dcc_destroy_stream_agents(dcc);
image_encoders_free(&dcc->encoders);
diff --git a/server/dcc.h b/server/dcc.h
index e87f110..0c19c02 100644
--- a/server/dcc.h
+++ b/server/dcc.h
@@ -75,9 +75,6 @@ struct DisplayChannelClient {
uint32_t palette_cache_items;
struct {
- uint32_t stream_outbuf_size;
- uint8_t *stream_outbuf; // caution stream buffer is also used as compress bufs!!!
-
FreeList free_list;
uint64_t pixmap_cache_items[MAX_DRAWABLE_PIXMAP_CACHE_ITEMS];
int num_pixmap_cache_items;
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 91d654f..8a44c41 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -37,6 +37,12 @@ typedef struct {
uint32_t bpp;
} SpiceFormatForGStreamer;
+typedef struct SpiceGstVideoBuffer {
+ VideoBuffer base;
+ GstBuffer *gst_buffer;
+ GstMapInfo map;
+} SpiceGstVideoBuffer;
+
typedef struct SpiceGstEncoder {
VideoEncoder base;
@@ -81,6 +87,26 @@ typedef struct SpiceGstEncoder {
} SpiceGstEncoder;
+/* ---------- The SpiceGstVideoBuffer implementation ---------- */
+
+static void spice_gst_video_buffer_free(VideoBuffer *video_buffer)
+{
+ SpiceGstVideoBuffer *buffer = (SpiceGstVideoBuffer*)video_buffer;
+ if (buffer->gst_buffer) {
+ gst_buffer_unmap(buffer->gst_buffer, &buffer->map);
+ gst_buffer_unref(buffer->gst_buffer);
+ }
+ free(buffer);
+}
+
+static SpiceGstVideoBuffer* create_gst_video_buffer(void)
+{
+ SpiceGstVideoBuffer *buffer = spice_new0(SpiceGstVideoBuffer, 1);
+ buffer->base.free = spice_gst_video_buffer_free;
+ return buffer;
+}
+
+
/* ---------- Miscellaneous SpiceGstEncoder helpers ---------- */
static inline double get_mbps(uint64_t bit_rate)
@@ -445,29 +471,22 @@ static int push_raw_frame(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
/* A helper for spice_gst_encoder_encode_frame() */
static int pull_compressed_buffer(SpiceGstEncoder *encoder,
- uint8_t **outbuf, size_t *outbuf_size,
- uint32_t *data_size)
+ VideoBuffer **outbuf)
{
- spice_return_val_if_fail(outbuf && outbuf_size, VIDEO_ENCODER_FRAME_UNSUPPORTED);
-
GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
if (sample) {
- GstMapInfo map;
- GstBuffer *buffer = gst_sample_get_buffer(sample);
- if (buffer && gst_buffer_map(buffer, &map, GST_MAP_READ)) {
- gint size = gst_buffer_get_size(buffer);
- if (!*outbuf || *outbuf_size < size) {
- free(*outbuf);
- *outbuf = spice_malloc(size);
- *outbuf_size = size;
- }
- /* TODO Try to avoid this copy by changing the GstBuffer handling */
- memcpy(*outbuf, map.data, size);
- *data_size = size;
- gst_buffer_unmap(buffer, &map);
+ SpiceGstVideoBuffer *buffer = create_gst_video_buffer();
+ buffer->gst_buffer = gst_sample_get_buffer(sample);
+ if (buffer->gst_buffer &&
+ gst_buffer_map(buffer->gst_buffer, &buffer->map, GST_MAP_READ)) {
+ buffer->base.data = buffer->map.data;
+ buffer->base.size = gst_buffer_get_size(buffer->gst_buffer);
+ *outbuf = (VideoBuffer*)buffer;
+ gst_buffer_ref(buffer->gst_buffer);
gst_sample_unref(sample);
return VIDEO_ENCODER_FRAME_ENCODE_DONE;
}
+ buffer->base.free((VideoBuffer*)buffer);
gst_sample_unref(sample);
}
spice_debug("failed to pull the compressed buffer");
@@ -488,10 +507,11 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
- uint8_t **outbuf, size_t *outbuf_size,
- uint32_t *data_size)
+ VideoBuffer **outbuf)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ g_return_val_if_fail(outbuf != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
+ *outbuf = NULL;
uint32_t width = src->right - src->left;
uint32_t height = src->bottom - src->top;
@@ -519,7 +539,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
int rc = push_raw_frame(encoder, bitmap, src, top_down);
if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
- rc = pull_compressed_buffer(encoder, outbuf, outbuf_size, data_size);
+ rc = pull_compressed_buffer(encoder, outbuf);
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
/* The input buffer will be stuck in the pipeline, preventing
* later ones from being processed. Furthermore something went
diff --git a/server/mjpeg-encoder.c b/server/mjpeg-encoder.c
index 5c143a6..2db6ca9 100644
--- a/server/mjpeg-encoder.c
+++ b/server/mjpeg-encoder.c
@@ -70,6 +70,9 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
*/
#define MJPEG_WARMUP_TIME (NSEC_PER_SEC * 3)
+/* The compressed buffer initial size. */
+#define MJPEG_INITIAL_BUFFER_SIZE (32 * 1024)
+
enum {
MJPEG_QUALITY_EVAL_TYPE_SET,
MJPEG_QUALITY_EVAL_TYPE_UPGRADE,
@@ -154,6 +157,11 @@ typedef struct MJpegEncoderRateControl {
uint64_t warmup_start_time;
} MJpegEncoderRateControl;
+typedef struct MJpegVideoBuffer {
+ VideoBuffer base;
+ size_t maxsize;
+} MJpegVideoBuffer;
+
typedef struct MJpegEncoder {
VideoEncoder base;
uint8_t *row;
@@ -180,6 +188,26 @@ static uint32_t get_min_required_playback_delay(uint64_t frame_enc_size,
uint64_t byte_rate,
uint32_t latency);
+static void mjpeg_video_buffer_free(VideoBuffer *video_buffer)
+{
+ MJpegVideoBuffer *buffer = (MJpegVideoBuffer*)video_buffer;
+ free(buffer->base.data);
+ free(buffer);
+}
+
+static MJpegVideoBuffer* create_mjpeg_video_buffer(void)
+{
+ MJpegVideoBuffer *buffer = spice_new0(MJpegVideoBuffer, 1);
+ buffer->base.free = mjpeg_video_buffer_free;
+ buffer->maxsize = MJPEG_INITIAL_BUFFER_SIZE;
+ buffer->base.data = malloc(buffer->maxsize);
+ if (!buffer->base.data) {
+ free(buffer);
+ buffer = NULL;
+ }
+ return buffer;
+}
+
static inline int rate_control_is_active(MJpegEncoder* encoder)
{
return encoder->cbs.get_roundtrip_ms != NULL;
@@ -283,24 +311,22 @@ static void term_mem_destination(j_compress_ptr cinfo)
/*
* Prepare for output to a memory buffer.
- * The caller may supply an own initial buffer with appropriate size.
- * Otherwise, or when the actual data output exceeds the given size,
- * the library adapts the buffer size as necessary.
- * The standard library functions malloc/free are used for allocating
- * larger memory, so the buffer is available to the application after
- * finishing compression, and then the application is responsible for
- * freeing the requested memory.
+ * The caller must supply its own initial buffer and size.
+ * When the actual data output exceeds the given size, the library
+ * will adapt the buffer size as necessary using the malloc()/free()
+ * functions. The buffer is available to the application after the
+ * compression and the application is then responsible for freeing it.
*/
-
static void
spice_jpeg_mem_dest(j_compress_ptr cinfo,
unsigned char ** outbuffer, size_t * outsize)
{
mem_destination_mgr *dest;
-#define OUTPUT_BUF_SIZE 4096 /* choose an efficiently fwrite'able size */
- if (outbuffer == NULL || outsize == NULL) /* sanity check */
+ if (outbuffer == NULL || *outbuffer == NULL ||
+ outsize == NULL || *outsize == 0) { /* sanity check */
ERREXIT(cinfo, JERR_BUFFER_SIZE);
+ }
/* The destination object is made permanent so that multiple JPEG images
* can be written to the same buffer without re-executing jpeg_mem_dest.
@@ -315,13 +341,6 @@ spice_jpeg_mem_dest(j_compress_ptr cinfo,
dest->pub.term_destination = term_mem_destination;
dest->outbuffer = outbuffer;
dest->outsize = outsize;
- if (*outbuffer == NULL || *outsize == 0) {
- /* Allocate initial buffer */
- *outbuffer = malloc(OUTPUT_BUF_SIZE);
- if (*outbuffer == NULL)
- ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 10);
- *outsize = OUTPUT_BUF_SIZE;
- }
dest->pub.next_output_byte = dest->buffer = *outbuffer;
dest->pub.free_in_buffer = dest->bufsize = *outsize;
@@ -707,7 +726,7 @@ static void mjpeg_encoder_adjust_fps(MJpegEncoder *encoder, uint64_t now)
static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
SpiceBitmapFmt format,
const SpiceRect *src,
- uint8_t **dest, size_t *dest_len,
+ MJpegVideoBuffer *buffer,
uint32_t frame_mm_time)
{
uint32_t quality;
@@ -791,7 +810,8 @@ static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
}
}
- spice_jpeg_mem_dest(&encoder->cinfo, dest, dest_len);
+ spice_jpeg_mem_dest(&encoder->cinfo, &buffer->base.data, &buffer->maxsize);
+
jpeg_set_defaults(&encoder->cinfo);
encoder->cinfo.dct_method = JDCT_IFAST;
quality = mjpeg_quality_samples[encoder->rate_control.quality_id];
@@ -928,25 +948,29 @@ static int mjpeg_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
- uint8_t **outbuf, size_t *outbuf_size,
- uint32_t *data_size)
+ VideoBuffer **outbuf)
{
MJpegEncoder *encoder = (MJpegEncoder*)video_encoder;
+ MJpegVideoBuffer *buffer = create_mjpeg_video_buffer();
+ if (!buffer) {
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
int ret = mjpeg_encoder_start_frame(encoder, bitmap->format, src,
- outbuf, outbuf_size,
- frame_mm_time);
- if (ret != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
- return ret;
+ buffer, frame_mm_time);
+ if (ret == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ if (encode_frame(encoder, src, bitmap, top_down)) {
+ buffer->base.size = mjpeg_encoder_end_frame(encoder);
+ *outbuf = (VideoBuffer*)buffer;
+ } else {
+ ret = VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
}
- if (!encode_frame(encoder, src, bitmap, top_down)) {
- return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ if (ret != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ buffer->base.free((VideoBuffer*)buffer);
}
-
- *data_size = mjpeg_encoder_end_frame(encoder);
-
- return VIDEO_ENCODER_FRAME_ENCODE_DONE;
+ return ret;
}
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 7a04bc6..f4be396 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -24,6 +24,23 @@
#include <inttypes.h>
#include <common/draw.h>
+
+/* A structure containing the data for a compressed frame. See encode_frame(). */
+typedef struct VideoBuffer VideoBuffer;
+struct VideoBuffer {
+ /* A pointer to the compressed frame data. */
+ uint8_t *data;
+
+ /* The size of the compressed frame in bytes. */
+ uint32_t size;
+
+ /* Releases the video buffer resources and deallocates it.
+ *
+ * @buffer: The video buffer.
+ */
+ void (*free)(VideoBuffer *buffer);
+};
+
enum {
VIDEO_ENCODER_FRAME_UNSUPPORTED = -1,
VIDEO_ENCODER_FRAME_DROP,
@@ -48,11 +65,9 @@ struct VideoEncoder {
* @bitmap: A bitmap containing the source video frame.
* @src: A rectangle specifying the area occupied by the video.
* @top_down: If true the first video line is specified by src.top.
- * @outbuf: The buffer for the compressed frame. This must either
- * be NULL or point to a buffer allocated by malloc
- * since it may be reallocated, if its size is too small.
- * @outbuf_size: The size of the outbuf buffer.
- * @data_size: The size of the compressed frame.
+ * @outbuf: A pointer to a VideoBuffer structure containing the
+ * compressed frame if successful. Call the buffer's
+ * free() method as soon as it is no longer needed.
* @return:
* VIDEO_ENCODER_FRAME_ENCODE_DONE if successful.
* VIDEO_ENCODER_FRAME_UNSUPPORTED if the frame cannot be encoded.
@@ -62,8 +77,7 @@ struct VideoEncoder {
int (*encode_frame)(VideoEncoder *encoder, uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
- uint8_t **outbuf, size_t *outbuf_size,
- uint32_t *data_size);
+ VideoBuffer** outbuf);
/*
* Bit rate control methods.
commit 6dc0dadf8d0c0b8652d01cc2e46292662a91f59c
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Nov 13 17:27:20 2015 +0100
streaming: Add VP8 support to the GStreamer video encoder
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/configure.ac b/configure.ac
index 35da955..e6be07a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -78,6 +78,7 @@ if test "x$enable_gstreamer" != "xno"; then
[enable_gstreamer="yes"
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-base 1.0], [appsrc videoconvert appsink])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gstreamer-libav 1.0], [avenc_mjpeg])
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-good 1.0], [vp8enc])
],
[if test "x$enable_gstreamer" = "xyes"; then
AC_MSG_ERROR([GStreamer 1.0 support requested but not found. You may set GSTREAMER_1_0_CFLAGS and GSTREAMER_1_0_LIBS to avoid the need to call pkg-config.])
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 0e4dc8d..91d654f 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -191,11 +191,44 @@ static void set_appsrc_caps(SpiceGstEncoder *encoder)
static gboolean create_pipeline(SpiceGstEncoder *encoder)
{
+ gchar *gstenc;
+ switch (encoder->base.codec_type)
+ {
+ case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+ /* Set max-threads to ensure zero-frame latency */
+ gstenc = g_strdup("avenc_mjpeg max-threads=1");
+ break;
+ case SPICE_VIDEO_CODEC_TYPE_VP8: {
+ /* See http://www.webmproject.org/docs/encoder-parameters/
+ * - Set end-usage to get a constant bitrate to help with streaming.
+ * - min-quantizer ensures the bitrate does not get needlessly high.
+ * - resize-allowed would be useful for low bitrate situations but
+ * the decoder does not return a frame of the expected size so
+ * avoid it.
+ * - error-resilient minimises artifacts in case the client drops a
+ * frame.
+ * - Set lag-in-frames, deadline and cpu-used to match
+ * "Profile Realtime". lag-in-frames ensures zero-frame latency,
+ * deadline turns on realtime behavior, and cpu-used targets a 75%
+ * CPU usage.
+ * - deadline is supposed to be set in microseconds but in practice
+ * it behaves like a boolean.
+ */
+ gstenc = g_strdup_printf("vp8enc end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4");
+ break;
+ }
+ default:
+ /* gstreamer_encoder_new() should have rejected this codec type */
+ spice_warning("unsupported codec type %d", encoder->base.codec_type);
+ return FALSE;
+ }
+
GError *err = NULL;
- /* Set max-threads to ensure zero-frame latency */
- const gchar *desc = "appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! avenc_mjpeg max-threads=1 name=encoder ! appsink name=sink";
+ gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! %s name=encoder ! appsink name=sink", gstenc);
spice_debug("GStreamer pipeline: %s", desc);
encoder->pipeline = gst_parse_launch_full(desc, NULL, GST_PARSE_FLAG_FATAL_ERRORS, &err);
+ g_free(gstenc);
+ g_free(desc);
if (!encoder->pipeline || err) {
spice_warning("GStreamer error: %s", err->message);
g_clear_error(&err);
@@ -209,9 +242,11 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
encoder->gstenc = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "encoder");
encoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "sink"));
- /* See https://bugzilla.gnome.org/show_bug.cgi?id=753257 */
- spice_debug("removing the pipeline clock");
- gst_pipeline_use_clock(GST_PIPELINE(encoder->pipeline), NULL);
+ if (encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG) {
+ /* See https://bugzilla.gnome.org/show_bug.cgi?id=753257 */
+ spice_debug("removing the pipeline clock");
+ gst_pipeline_use_clock(GST_PIPELINE(encoder->pipeline), NULL);
+ }
set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_STATE |
SPICE_GST_VIDEO_PIPELINE_BITRATE |
@@ -224,8 +259,23 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
{
adjust_bit_rate(encoder);
- g_object_set(G_OBJECT(encoder->gstenc),
- "bitrate", (gint)encoder->bit_rate, NULL);
+ switch (encoder->base.codec_type)
+ {
+ case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+ g_object_set(G_OBJECT(encoder->gstenc),
+ "bitrate", (gint)encoder->bit_rate,
+ NULL);
+ break;
+ case SPICE_VIDEO_CODEC_TYPE_VP8:
+ g_object_set(G_OBJECT(encoder->gstenc),
+ "target-bitrate", (gint)encoder->bit_rate,
+ NULL);
+ break;
+ default:
+ /* gstreamer_encoder_new() should have rejected this codec type */
+ spice_warning("unsupported codec type %d", encoder->base.codec_type);
+ free_pipeline(encoder);
+ }
}
/* A helper for spice_gst_encoder_encode_frame() */
@@ -528,7 +578,8 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs)
{
- spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG, NULL);
+ spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG ||
+ codec_type == SPICE_VIDEO_CODEC_TYPE_VP8, NULL);
GError *err = NULL;
if (!gst_init_check(NULL, NULL, &err)) {
diff --git a/server/reds.c b/server/reds.c
index 4c8826e..b4ad4bb 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -3492,7 +3492,7 @@ err:
}
static const char default_renderer[] = "sw";
-static const char default_video_codecs[] = "spice:mjpeg;gstreamer:mjpeg";
+static const char default_video_codecs[] = "spice:mjpeg;gstreamer:mjpeg;gstreamer:vp8";
/* new interface */
SPICE_GNUC_VISIBLE SpiceServer *spice_server_new(void)
@@ -3579,11 +3579,13 @@ static new_video_encoder_t video_encoder_procs[] = {
static const EnumNames video_codec_names[] = {
{SPICE_VIDEO_CODEC_TYPE_MJPEG, "mjpeg"},
+ {SPICE_VIDEO_CODEC_TYPE_VP8, "vp8"},
{0, NULL},
};
static int video_codec_caps[] = {
SPICE_DISPLAY_CAP_CODEC_MJPEG,
+ SPICE_DISPLAY_CAP_CODEC_VP8,
};
commit ce6113d838023d035fc20554a9a60aa3710338a0
Author: Pavel Grunt <pgrunt at redhat.com>
Date: Mon Jan 25 17:28:13 2016 +0100
replay: Add an option to change video codec
diff --git a/server/tests/replay.c b/server/tests/replay.c
index 3e4af15..86a7bab 100644
--- a/server/tests/replay.c
+++ b/server/tests/replay.c
@@ -308,7 +308,7 @@ int main(int argc, char **argv)
{
GError *error = NULL;
GOptionContext *context = NULL;
- gchar *client = NULL, **file = NULL;
+ gchar *client = NULL, *codecs = NULL, **file = NULL;
gint port = 5000, compression = SPICE_IMAGE_COMPRESSION_AUTO_GLZ;
gint streaming = SPICE_STREAM_VIDEO_FILTER;
gboolean wait = FALSE;
@@ -318,6 +318,7 @@ int main(int argc, char **argv)
{ "client", 'c', 0, G_OPTION_ARG_STRING, &client, "Client", "CMD" },
{ "compression", 'C', 0, G_OPTION_ARG_INT, &compression, "Compression (default 2)", "INT" },
{ "streaming", 'S', 0, G_OPTION_ARG_INT, &streaming, "Streaming (default 3)", "INT" },
+ { "video-codecs", 'v', 0, G_OPTION_ARG_STRING, &codecs, "Video codecs", "STRING" },
{ "port", 'p', 0, G_OPTION_ARG_INT, &port, "Server port (default 5000)", "PORT" },
{ "wait", 'w', 0, G_OPTION_ARG_NONE, &wait, "Wait for client", NULL },
{ "slow", 's', 0, G_OPTION_ARG_INT, &slow, "Slow down replay. Delays USEC microseconds before each command", "USEC" },
@@ -405,6 +406,14 @@ int main(int argc, char **argv)
server = spice_server_new();
spice_server_set_image_compression(server, compression);
spice_server_set_streaming_video(server, streaming);
+
+ if (codecs != NULL) {
+ if (spice_server_set_video_codecs(server, codecs) != 0) {
+ g_warning("could not set codecs: %s", codecs);
+ }
+ g_free(codecs);
+ }
+
spice_server_set_port(server, port);
spice_server_set_noauth(server);
commit 497fcbb0a315b034bae4a565ec550a7cba058e3c
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Mon May 11 16:51:35 2015 +0200
streaming: Let the administrator pick the video encoder and codec
The Spice server administrator can specify the encoder and codec
preferences to optimize for CPU or bandwidth usage. Preferences are
described in a semi-colon separated list of encoder:codec pairs.
The server has a default preference list which can explicitly be
selected by specifying 'auto'.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/dcc-send.c b/server/dcc-send.c
index 9c5d970..f8087ea 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -2166,7 +2166,7 @@ static void marshall_stream_start(RedChannelClient *rcc,
stream_create.surface_id = 0;
stream_create.id = get_stream_id(DCC_TO_DC(dcc), stream);
stream_create.flags = stream->top_down ? SPICE_STREAM_FLAGS_TOP_DOWN : 0;
- stream_create.codec_type = SPICE_VIDEO_CODEC_TYPE_MJPEG;
+ stream_create.codec_type = agent->video_encoder->codec_type;
stream_create.src_width = stream->width;
stream_create.src_height = stream->height;
diff --git a/server/display-channel.c b/server/display-channel.c
index 76d22f9..6dce831 100644
--- a/server/display-channel.c
+++ b/server/display-channel.c
@@ -140,6 +140,14 @@ void display_channel_set_stream_video(DisplayChannel *display, int stream_video)
display->stream_video = stream_video;
}
+void display_channel_set_video_codecs(DisplayChannel *display, GArray *video_codecs)
+{
+ spice_return_if_fail(display);
+
+ g_array_unref(display->video_codecs);
+ display->video_codecs = g_array_ref(video_codecs);
+}
+
static void stop_streams(DisplayChannel *display)
{
Ring *ring = &display->streams;
@@ -1890,6 +1898,7 @@ static SpiceCanvas *image_surfaces_get(SpiceImageSurfaces *surfaces, uint32_t su
DisplayChannel* display_channel_new(SpiceServer *reds, RedWorker *worker,
int migrate, int stream_video,
+ GArray *video_codecs,
uint32_t n_surfaces)
{
DisplayChannel *display;
@@ -1935,6 +1944,7 @@ DisplayChannel* display_channel_new(SpiceServer *reds, RedWorker *worker,
drawables_init(display);
image_cache_init(&display->image_cache);
display->stream_video = stream_video;
+ display->video_codecs = g_array_ref(video_codecs);
display_channel_init_streams(display);
return display;
diff --git a/server/display-channel.h b/server/display-channel.h
index 0baad62..7984c85 100644
--- a/server/display-channel.h
+++ b/server/display-channel.h
@@ -183,6 +183,7 @@ struct DisplayChannel {
_Drawable *free_drawables;
int stream_video;
+ GArray *video_codecs;
uint32_t stream_count;
Stream streams_buf[NUM_STREAMS];
Stream *free_streams;
@@ -237,6 +238,7 @@ DisplayChannel* display_channel_new (SpiceServe
RedWorker *worker,
int migrate,
int stream_video,
+ GArray *video_codecs,
uint32_t n_surfaces);
void display_channel_create_surface (DisplayChannel *display, uint32_t surface_id,
uint32_t width, uint32_t height,
@@ -258,6 +260,8 @@ void display_channel_update (DisplayCha
void display_channel_free_some (DisplayChannel *display);
void display_channel_set_stream_video (DisplayChannel *display,
int stream_video);
+void display_channel_set_video_codecs (DisplayChannel *display,
+ GArray *video_codecs);
int display_channel_get_streams_timeout (DisplayChannel *display);
void display_channel_compress_stats_print (const DisplayChannel *display);
void display_channel_compress_stats_reset (DisplayChannel *display);
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 6f748fb..0e4dc8d 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -524,9 +524,12 @@ static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
}
}
-VideoEncoder *gstreamer_encoder_new(uint64_t starting_bit_rate,
+VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
+ uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs)
{
+ spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG, NULL);
+
GError *err = NULL;
if (!gst_init_check(NULL, NULL, &err)) {
spice_warning("GStreamer error: %s", err->message);
@@ -541,6 +544,7 @@ VideoEncoder *gstreamer_encoder_new(uint64_t starting_bit_rate,
encoder->base.notify_server_frame_drop = spice_gst_encoder_notify_server_frame_drop;
encoder->base.get_bit_rate = spice_gst_encoder_get_bit_rate;
encoder->base.get_stats = spice_gst_encoder_get_stats;
+ encoder->base.codec_type = codec_type;
if (cbs) {
encoder->cbs = *cbs;
diff --git a/server/mjpeg-encoder.c b/server/mjpeg-encoder.c
index 57708cd..5c143a6 100644
--- a/server/mjpeg-encoder.c
+++ b/server/mjpeg-encoder.c
@@ -1341,17 +1341,21 @@ static void mjpeg_encoder_get_stats(VideoEncoder *video_encoder,
stats->avg_quality = (double)encoder->avg_quality / encoder->num_frames;
}
-VideoEncoder *mjpeg_encoder_new(uint64_t starting_bit_rate,
+VideoEncoder *mjpeg_encoder_new(SpiceVideoCodecType codec_type,
+ uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs)
{
MJpegEncoder *encoder = spice_new0(MJpegEncoder, 1);
+ spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG, NULL);
+
encoder->base.destroy = mjpeg_encoder_destroy;
encoder->base.encode_frame = mjpeg_encoder_encode_frame;
encoder->base.client_stream_report = mjpeg_encoder_client_stream_report;
encoder->base.notify_server_frame_drop = mjpeg_encoder_notify_server_frame_drop;
encoder->base.get_bit_rate = mjpeg_encoder_get_bit_rate;
encoder->base.get_stats = mjpeg_encoder_get_stats;
+ encoder->base.codec_type = codec_type;
encoder->first_frame = TRUE;
encoder->rate_control.byte_rate = starting_bit_rate / 8;
encoder->starting_bit_rate = starting_bit_rate;
diff --git a/server/red-qxl.c b/server/red-qxl.c
index 47fafab..b84ae69 100644
--- a/server/red-qxl.c
+++ b/server/red-qxl.c
@@ -1045,6 +1045,15 @@ void red_qxl_on_sv_change(QXLInstance *qxl, int sv)
&payload);
}
+void red_qxl_on_vc_change(QXLInstance *qxl, GArray *video_codecs)
+{
+ RedWorkerMessageSetVideoCodecs payload;
+ payload.video_codecs = g_array_ref(video_codecs);
+ dispatcher_send_message(qxl->st->dispatcher,
+ RED_WORKER_MESSAGE_SET_VIDEO_CODECS,
+ &payload);
+}
+
void red_qxl_set_mouse_mode(QXLInstance *qxl, uint32_t mode)
{
RedWorkerMessageSetMouseMode payload;
diff --git a/server/red-qxl.h b/server/red-qxl.h
index c9b6b36..00c5486 100644
--- a/server/red-qxl.h
+++ b/server/red-qxl.h
@@ -27,6 +27,7 @@ void red_qxl_init(SpiceServer *reds, QXLInstance *qxl);
void red_qxl_on_ic_change(QXLInstance *qxl, SpiceImageCompression ic);
void red_qxl_on_sv_change(QXLInstance *qxl, int sv);
+void red_qxl_on_vc_change(QXLInstance *qxl, GArray* video_codecs);
void red_qxl_set_mouse_mode(QXLInstance *qxl, uint32_t mode);
void red_qxl_attach_worker(QXLInstance *qxl);
void red_qxl_set_compression_level(QXLInstance *qxl, int level);
@@ -113,6 +114,7 @@ enum {
RED_WORKER_MESSAGE_DRIVER_UNLOAD,
RED_WORKER_MESSAGE_GL_SCANOUT,
RED_WORKER_MESSAGE_GL_DRAW_ASYNC,
+ RED_WORKER_MESSAGE_SET_VIDEO_CODECS,
RED_WORKER_MESSAGE_COUNT // LAST
};
@@ -250,6 +252,10 @@ typedef struct RedWorkerMessageSetStreamingVideo {
uint32_t streaming_video;
} RedWorkerMessageSetStreamingVideo;
+typedef struct RedWorkerMessageSetVideoCodecs {
+ GArray* video_codecs;
+} RedWorkerMessageSetVideoCodecs;
+
typedef struct RedWorkerMessageSetMouseMode {
uint32_t mode;
} RedWorkerMessageSetMouseMode;
diff --git a/server/red-worker.c b/server/red-worker.c
index aba3e6f..b38d7c7 100644
--- a/server/red-worker.c
+++ b/server/red-worker.c
@@ -1059,6 +1059,15 @@ static void handle_dev_set_streaming_video(void *opaque, void *payload)
display_channel_set_stream_video(worker->display_channel, msg->streaming_video);
}
+void handle_dev_set_video_codecs(void *opaque, void *payload)
+{
+ RedWorkerMessageSetVideoCodecs *msg = payload;
+ RedWorker *worker = opaque;
+
+ display_channel_set_video_codecs(worker->display_channel, msg->video_codecs);
+ g_array_unref(msg->video_codecs);
+}
+
static void handle_dev_set_mouse_mode(void *opaque, void *payload)
{
RedWorkerMessageSetMouseMode *msg = payload;
@@ -1332,6 +1341,11 @@ static void register_callbacks(Dispatcher *dispatcher)
sizeof(RedWorkerMessageSetStreamingVideo),
DISPATCHER_NONE);
dispatcher_register_handler(dispatcher,
+ RED_WORKER_MESSAGE_SET_VIDEO_CODECS,
+ handle_dev_set_video_codecs,
+ sizeof(RedWorkerMessageSetVideoCodecs),
+ DISPATCHER_NONE);
+ dispatcher_register_handler(dispatcher,
RED_WORKER_MESSAGE_SET_MOUSE_MODE,
handle_dev_set_mouse_mode,
sizeof(RedWorkerMessageSetMouseMode),
@@ -1516,7 +1530,9 @@ RedWorker* red_worker_new(QXLInstance *qxl,
reds_register_channel(reds, channel);
// TODO: handle seemless migration. Temp, setting migrate to FALSE
- worker->display_channel = display_channel_new(reds, worker, FALSE, reds_get_streaming_video(reds),
+ worker->display_channel = display_channel_new(reds, worker, FALSE,
+ reds_get_streaming_video(reds),
+ reds_get_video_codecs(reds),
init_info.n_surfaces);
channel = RED_CHANNEL(worker->display_channel);
diff --git a/server/reds.c b/server/reds.c
index 27a5957..4c8826e 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -71,6 +71,7 @@
#include "utils.h"
#include "reds-private.h"
+#include "video-encoder.h"
static void reds_client_monitors_config(RedsState *reds, VDAgentMonitorsConfig *monitors_config);
static gboolean reds_use_client_monitors_config(RedsState *reds);
@@ -178,6 +179,7 @@ struct RedServerConfig {
gboolean ticketing_enabled;
uint32_t streaming_video;
+ GArray* video_codecs;
SpiceImageCompression image_compression;
uint32_t playback_compression;
spice_wan_compression_t jpeg_state;
@@ -298,6 +300,7 @@ static void reds_add_char_device(RedsState *reds, RedCharDevice *dev);
static void reds_send_mm_time(RedsState *reds);
static void reds_on_ic_change(RedsState *reds);
static void reds_on_sv_change(RedsState *reds);
+static void reds_on_vc_change(RedsState *reds);
static void reds_on_vm_stop(RedsState *reds);
static void reds_on_vm_start(RedsState *reds);
static void reds_set_mouse_mode(RedsState *reds, uint32_t mode);
@@ -3489,6 +3492,7 @@ err:
}
static const char default_renderer[] = "sw";
+static const char default_video_codecs[] = "spice:mjpeg;gstreamer:mjpeg";
/* new interface */
SPICE_GNUC_VISIBLE SpiceServer *spice_server_new(void)
@@ -3510,6 +3514,7 @@ SPICE_GNUC_VISIBLE SpiceServer *spice_server_new(void)
memset(reds->config->spice_uuid, 0, sizeof(reds->config->spice_uuid));
reds->config->ticketing_enabled = TRUE; /* ticketing enabled by default */
reds->config->streaming_video = SPICE_STREAM_VIDEO_FILTER;
+ reds->config->video_codecs = g_array_new(FALSE, FALSE, sizeof(RedVideoCodec));
reds->config->image_compression = SPICE_IMAGE_COMPRESSION_AUTO_GLZ;
reds->config->playback_compression = TRUE;
reds->config->jpeg_state = SPICE_WAN_COMPRESSION_AUTO;
@@ -3521,37 +3526,129 @@ SPICE_GNUC_VISIBLE SpiceServer *spice_server_new(void)
return reds;
}
-typedef struct RendererInfo {
- int id;
+typedef struct {
+ uint32_t id;
const char *name;
-} RendererInfo;
+} EnumNames;
-static const RendererInfo renderers_info[] = {
+static gboolean get_name_index(const EnumNames names[], const char *name, uint32_t *index)
+{
+ if (name) {
+ int i;
+ for (i = 0; names[i].name; i++) {
+ if (strcmp(name, names[i].name) == 0) {
+ *index = i;
+ return TRUE;
+ }
+ }
+ }
+ return FALSE;
+}
+
+static const EnumNames renderer_names[] = {
{RED_RENDERER_SW, "sw"},
{RED_RENDERER_INVALID, NULL},
};
-static const RendererInfo *find_renderer(const char *name)
+static gboolean reds_add_renderer(RedsState *reds, const char *name)
+{
+ uint32_t index;
+
+ if (reds->config->renderers->len == RED_RENDERER_LAST ||
+ !get_name_index(renderer_names, name, &index)) {
+ return FALSE;
+ }
+ g_array_append_val(reds->config->renderers, renderer_names[index].id);
+ return TRUE;
+}
+
+static const EnumNames video_encoder_names[] = {
+ {0, "spice"},
+ {1, "gstreamer"},
+ {0, NULL},
+};
+
+static new_video_encoder_t video_encoder_procs[] = {
+ &mjpeg_encoder_new,
+#ifdef HAVE_GSTREAMER_1_0
+ &gstreamer_encoder_new,
+#else
+ NULL,
+#endif
+};
+
+static const EnumNames video_codec_names[] = {
+ {SPICE_VIDEO_CODEC_TYPE_MJPEG, "mjpeg"},
+ {0, NULL},
+};
+
+static int video_codec_caps[] = {
+ SPICE_DISPLAY_CAP_CODEC_MJPEG,
+};
+
+
+/* Expected string: encoder:codec;encoder:codec */
+static const char* parse_video_codecs(const char *codecs, char **encoder,
+ char **codec)
{
- const RendererInfo *inf = renderers_info;
- while (inf->name) {
- if (strcmp(name, inf->name) == 0) {
- return inf;
+ if (!codecs) {
+ return NULL;
+ }
+ while (*codecs == ';') {
+ codecs++;
+ }
+ if (!*codecs) {
+ return NULL;
+ }
+ int n;
+ *encoder = *codec = NULL;
+ if (sscanf(codecs, "%m[0-9a-zA-Z_]:%m[0-9a-zA-Z_]%n", encoder, codec, &n) != 2) {
+ while (*codecs != '\0' && *codecs != ';') {
+ codecs++;
}
- inf++;
+ return codecs;
}
- return NULL;
+ return codecs + n;
}
-static int reds_add_renderer(RedsState *reds, const char *name)
+static void reds_set_video_codecs(RedsState *reds, const char *codecs)
{
- const RendererInfo *inf;
+ char *encoder_name, *codec_name;
- if (reds->config->renderers->len == RED_RENDERER_LAST || !(inf = find_renderer(name))) {
- return FALSE;
+ if (strcmp(codecs, "auto") == 0) {
+ codecs = default_video_codecs;
+ }
+
+ /* The video_codecs array is immutable */
+ g_array_unref(reds->config->video_codecs);
+ reds->config->video_codecs = g_array_new(FALSE, FALSE, sizeof(RedVideoCodec));
+ const char *c = codecs;
+ while ( (c = parse_video_codecs(c, &encoder_name, &codec_name)) ) {
+ uint32_t encoder_index, codec_index;
+ if (!encoder_name || !codec_name) {
+ spice_warning("spice: invalid encoder:codec value at %s", codecs);
+
+ } else if (!get_name_index(video_encoder_names, encoder_name, &encoder_index)){
+ spice_warning("spice: unknown video encoder %s", encoder_name);
+
+ } else if (!get_name_index(video_codec_names, codec_name, &codec_index)) {
+ spice_warning("spice: unknown video codec %s", codec_name);
+
+ } else if (!video_encoder_procs[encoder_index]) {
+ spice_warning("spice: unsupported video encoder %s", encoder_name);
+
+ } else {
+ RedVideoCodec new_codec;
+ new_codec.create = video_encoder_procs[encoder_index];
+ new_codec.type = video_codec_names[codec_index].id;
+ new_codec.cap = video_codec_caps[codec_index];
+ g_array_append_val(reds->config->video_codecs, new_codec);
+ }
+
+ free(encoder_name);
+ free(codec_name);
+ codecs = c;
}
- g_array_append_val(reds->config->renderers, inf->id);
- return TRUE;
}
SPICE_GNUC_VISIBLE int spice_server_init(SpiceServer *reds, SpiceCoreInterface *core)
@@ -3562,12 +3659,16 @@ SPICE_GNUC_VISIBLE int spice_server_init(SpiceServer *reds, SpiceCoreInterface *
if (reds->config->renderers->len == 0) {
reds_add_renderer(reds, default_renderer);
}
+ if (reds->config->video_codecs->len == 0) {
+ reds_set_video_codecs(reds, default_video_codecs);
+ }
return ret;
}
SPICE_GNUC_VISIBLE void spice_server_destroy(SpiceServer *reds)
{
g_array_unref(reds->config->renderers);
+ g_array_unref(reds->config->video_codecs);
free(reds->config);
if (reds->main_channel) {
main_channel_close(reds->main_channel);
@@ -3869,6 +3970,18 @@ uint32_t reds_get_streaming_video(const RedsState *reds)
return reds->config->streaming_video;
}
+SPICE_GNUC_VISIBLE int spice_server_set_video_codecs(SpiceServer *reds, const char *video_codecs)
+{
+ reds_set_video_codecs(reds, video_codecs);
+ reds_on_vc_change(reds);
+ return 0;
+}
+
+GArray* reds_get_video_codecs(const RedsState *reds)
+{
+ return reds->config->video_codecs;
+}
+
SPICE_GNUC_VISIBLE int spice_server_set_playback_compression(SpiceServer *reds, int enable)
{
reds->config->playback_compression = !!enable;
@@ -4261,6 +4374,15 @@ void reds_on_sv_change(RedsState *reds)
}
}
+void reds_on_vc_change(RedsState *reds)
+{
+ GList *l;
+
+ for (l = reds->qxl_instances; l != NULL; l = l->next) {
+ red_qxl_on_vc_change(l->data, reds_get_video_codecs(reds));
+ }
+}
+
void reds_on_vm_stop(RedsState *reds)
{
GList *l;
diff --git a/server/reds.h b/server/reds.h
index 1f05081..cd62fc1 100644
--- a/server/reds.h
+++ b/server/reds.h
@@ -93,6 +93,7 @@ void reds_on_main_channel_migrate(RedsState *reds, MainChannelClient *mcc);
void reds_set_client_mm_time_latency(RedsState *reds, RedClient *client, uint32_t latency);
uint32_t reds_get_streaming_video(const RedsState *reds);
+GArray* reds_get_video_codecs(const RedsState *reds);
spice_wan_compression_t reds_get_jpeg_state(const RedsState *reds);
spice_wan_compression_t reds_get_zlib_glz_state(const RedsState *reds);
SpiceCoreInterfaceInternal* reds_get_core_interface(RedsState *reds);
diff --git a/server/spice-server.h b/server/spice-server.h
index 87c5c59..6eb1b1d 100644
--- a/server/spice-server.h
+++ b/server/spice-server.h
@@ -114,6 +114,14 @@ enum {
};
int spice_server_set_streaming_video(SpiceServer *s, int value);
+
+enum {
+ SPICE_STREAMING_INVALID,
+ SPICE_STREAMING_SPICE,
+ SPICE_STREAMING_GSTREAMER
+};
+
+int spice_server_set_video_codecs(SpiceServer *s, const char* video_codecs);
int spice_server_set_playback_compression(SpiceServer *s, int enable);
int spice_server_set_agent_mouse(SpiceServer *s, int enable);
int spice_server_set_agent_copypaste(SpiceServer *s, int enable);
diff --git a/server/spice-server.syms b/server/spice-server.syms
index 5c3e53c..edf04a4 100644
--- a/server/spice-server.syms
+++ b/server/spice-server.syms
@@ -168,3 +168,8 @@ global:
spice_qxl_gl_scanout;
spice_qxl_gl_draw_async;
} SPICE_SERVER_0.12.6;
+
+SPICE_SERVER_0.13.2 {
+global:
+ spice_server_set_video_codecs;
+} SPICE_SERVER_0.13.1;
diff --git a/server/stream.c b/server/stream.c
index 617e33a..3d3e627 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -711,17 +711,34 @@ static VideoEncoder* dcc_create_video_encoder(DisplayChannelClient *dcc,
uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs)
{
+ DisplayChannel *display = DCC_TO_DC(dcc);
RedChannelClient *rcc = RED_CHANNEL_CLIENT(dcc);
int client_has_multi_codec = red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_MULTI_CODEC);
- if (!client_has_multi_codec || red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_CODEC_MJPEG)) {
-#ifdef HAVE_GSTREAMER_1_0
- VideoEncoder* video_encoder = gstreamer_encoder_new(starting_bit_rate, cbs);
+ int i;
+
+ for (i = 0; i < display->video_codecs->len; i++) {
+ RedVideoCodec* video_codec = &g_array_index (display->video_codecs, RedVideoCodec, i);
+
+ if (!client_has_multi_codec &&
+ video_codec->type != SPICE_VIDEO_CODEC_TYPE_MJPEG) {
+ /* Old clients only support MJPEG */
+ continue;
+ }
+ if (client_has_multi_codec &&
+ !red_channel_client_test_remote_cap(rcc, video_codec->cap)) {
+ /* The client is recent but does not support this codec */
+ continue;
+ }
+
+ VideoEncoder* video_encoder = video_codec->create(video_codec->type, starting_bit_rate, cbs);
if (video_encoder) {
return video_encoder;
}
-#endif
- /* Use the builtin MJPEG video encoder as a fallback */
- return mjpeg_encoder_new(starting_bit_rate, cbs);
+ }
+
+ /* Try to use the builtin MJPEG video encoder as a fallback */
+ if (!client_has_multi_codec || red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_CODEC_MJPEG)) {
+ return mjpeg_encoder_new(SPICE_VIDEO_CODEC_TYPE_MJPEG, starting_bit_rate, cbs);
}
return NULL;
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 708432b..7a04bc6 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -120,6 +120,9 @@ struct VideoEncoder {
* statistics.
*/
void (*get_stats)(VideoEncoder *encoder, VideoEncoderStats *stats);
+
+ /* The codec being used by the video encoder */
+ SpiceVideoCodecType codec_type;
};
@@ -152,17 +155,30 @@ typedef struct VideoEncoderRateControlCbs {
/* Instantiates the video encoder.
*
+ * @codec_type: The codec to use.
* @starting_bit_rate: An initial estimate of the available stream bit rate
* or zero if the client does not support rate control.
* @cbs: A set of callback methods to be used for rate control.
* @return: A pointer to a structure implementing the VideoEncoder
* methods.
*/
-VideoEncoder* mjpeg_encoder_new(uint64_t starting_bit_rate,
+typedef VideoEncoder* (*new_video_encoder_t)(SpiceVideoCodecType codec_type,
+ uint64_t starting_bit_rate,
+ VideoEncoderRateControlCbs *cbs);
+
+VideoEncoder* mjpeg_encoder_new(SpiceVideoCodecType codec_type,
+ uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs);
#ifdef HAVE_GSTREAMER_1_0
-VideoEncoder* gstreamer_encoder_new(uint64_t starting_bit_rate,
+VideoEncoder* gstreamer_encoder_new(SpiceVideoCodecType codec_type,
+ uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs);
#endif
+typedef struct RedVideoCodec {
+ new_video_encoder_t create;
+ SpiceVideoCodecType type;
+ uint32_t cap;
+} RedVideoCodec;
+
#endif
commit f4414af48c9c716300afbfa6ee31dced53a57038
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Wed Nov 4 19:34:49 2015 +0100
streaming: Check the client video codec capabilities
The server picks a codec supported by the client based on the following
new client capabilities:
* SPICE_DISPLAY_CAP_MULTI_CODEC which denotes a recent client that
supports multiple codecs. This capability is needed to not have to
hardcode that MJPEG is supported. This makes it possible to write
clients that don't support MJPEG.
* SPICE_DISPLAY_CAP_CODEC_XXX, where XXX is a supported codec. Note
that for now the server only supports the MJPEG codec.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/dcc-send.c b/server/dcc-send.c
index 145099d..9c5d970 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1718,7 +1718,8 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
drawable->red_drawable->mm_time :
reds_get_mm_time();
outbuf_size = dcc->send_data.stream_outbuf_size;
- ret = agent->video_encoder->encode_frame(agent->video_encoder,
+ ret = !agent->video_encoder ? VIDEO_ENCODER_FRAME_UNSUPPORTED :
+ agent->video_encoder->encode_frame(agent->video_encoder,
frame_mm_time,
©->src_bitmap->u.bitmap,
©->src_area, stream->top_down,
diff --git a/server/stream.c b/server/stream.c
index b203612..617e33a 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -707,17 +707,24 @@ static void update_client_playback_delay(void *opaque, uint32_t delay_ms)
}
/* A helper for dcc_create_stream(). */
-static VideoEncoder* dcc_create_video_encoder(uint64_t starting_bit_rate,
+static VideoEncoder* dcc_create_video_encoder(DisplayChannelClient *dcc,
+ uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs)
{
+ RedChannelClient *rcc = RED_CHANNEL_CLIENT(dcc);
+ int client_has_multi_codec = red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_MULTI_CODEC);
+ if (!client_has_multi_codec || red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_CODEC_MJPEG)) {
#ifdef HAVE_GSTREAMER_1_0
- VideoEncoder* video_encoder = gstreamer_encoder_new(starting_bit_rate, cbs);
- if (video_encoder) {
- return video_encoder;
- }
+ VideoEncoder* video_encoder = gstreamer_encoder_new(starting_bit_rate, cbs);
+ if (video_encoder) {
+ return video_encoder;
+ }
#endif
- /* Use the builtin MJPEG video encoder as a fallback */
- return mjpeg_encoder_new(starting_bit_rate, cbs);
+ /* Use the builtin MJPEG video encoder as a fallback */
+ return mjpeg_encoder_new(starting_bit_rate, cbs);
+ }
+
+ return NULL;
}
void dcc_create_stream(DisplayChannelClient *dcc, Stream *stream)
@@ -747,9 +754,9 @@ void dcc_create_stream(DisplayChannelClient *dcc, Stream *stream)
video_cbs.update_client_playback_delay = update_client_playback_delay;
initial_bit_rate = get_initial_bit_rate(dcc, stream);
- agent->video_encoder = dcc_create_video_encoder(initial_bit_rate, &video_cbs);
+ agent->video_encoder = dcc_create_video_encoder(dcc, initial_bit_rate, &video_cbs);
} else {
- agent->video_encoder = dcc_create_video_encoder(0, NULL);
+ agent->video_encoder = dcc_create_video_encoder(dcc, 0, NULL);
}
red_channel_client_pipe_add(RED_CHANNEL_CLIENT(dcc), stream_create_item_new(agent));
commit a697bd971bfd676557e1a2e5debc3b61d2037591
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Thu May 7 21:11:17 2015 +0200
streaming: Add a GStreamer 1.0 MJPEG video encoder and use it by default
This introduces a pared down GStreamer-based video encoder to serve as
the basis for later enhancements.
In this form the new encoder supports both regular and sized streams
but lacks any rate control. It should still work fine if bandwidth is
sufficient such as on LANs.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/configure.ac b/configure.ac
index 856833b..35da955 100644
--- a/configure.ac
+++ b/configure.ac
@@ -68,6 +68,28 @@ dnl =========================================================================
dnl Check optional features
SPICE_CHECK_SMARTCARD
+AC_ARG_ENABLE(gstreamer,
+ AS_HELP_STRING([--enable-gstreamer=@<:@auto/yes/no@:>@],
+ [Enable GStreamer 1.0 support]),,
+ [enable_gstreamer="auto"])
+
+if test "x$enable_gstreamer" != "xno"; then
+ SPICE_CHECK_GSTREAMER(GSTREAMER_1_0, 1.0, [gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0],
+ [enable_gstreamer="yes"
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-base 1.0], [appsrc videoconvert appsink])
+ SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gstreamer-libav 1.0], [avenc_mjpeg])
+ ],
+ [if test "x$enable_gstreamer" = "xyes"; then
+ AC_MSG_ERROR([GStreamer 1.0 support requested but not found. You may set GSTREAMER_1_0_CFLAGS and GSTREAMER_1_0_LIBS to avoid the need to call pkg-config.])
+ fi
+ ])
+fi
+AM_CONDITIONAL(HAVE_GSTREAMER_1_0, test "x$have_gstreamer_1_0" = "xyes")
+
+if test x"$gstreamer_missing" != x; then
+ SPICE_WARNING([The following GStreamer $enable_gstreamer tools/elements are missing:$gstreamer_missing. The GStreamer video encoder can be built but may not work.])
+fi
+
AC_ARG_ENABLE([automated_tests],
AS_HELP_STRING([--enable-automated-tests], [Enable automated tests using spicy-screenshot (part of spice-gtk)]),,
[enable_automated_tests="no"])
@@ -240,6 +262,7 @@ AC_MSG_NOTICE([
LZ4 support: ${enable_lz4}
Smartcard: ${have_smartcard}
+ GStreamer 1.0: ${have_gstreamer_1_0}
SASL support: ${have_sasl}
Automated tests: ${enable_automated_tests}
Manual: ${have_asciidoc}
diff --git a/server/Makefile.am b/server/Makefile.am
index cca3b9b..ea3e2ff 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -12,6 +12,7 @@ AM_CPPFLAGS = \
$(SASL_CFLAGS) \
$(SLIRP_CFLAGS) \
$(SMARTCARD_CFLAGS) \
+ $(GSTREAMER_1_0_CFLAGS) \
$(SPICE_PROTOCOL_CFLAGS) \
$(SSL_CFLAGS) \
$(VISIBILITY_HIDDEN_CFLAGS) \
@@ -45,6 +46,7 @@ libserver_la_LIBADD = \
$(PIXMAN_LIBS) \
$(SASL_LIBS) \
$(SLIRP_LIBS) \
+ $(GSTREAMER_1_0_LIBS) \
$(SSL_LIBS) \
$(Z_LIBS) \
$(SPICE_NONPKGCONFIG_LIBS) \
@@ -157,6 +159,12 @@ libserver_la_SOURCES += \
$(NULL)
endif
+if HAVE_GSTREAMER_1_0
+libserver_la_SOURCES += \
+ gstreamer-encoder.c \
+ $(NULL)
+endif
+
libspice_server_la_LIBADD = libserver.la
libspice_server_la_SOURCES =
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
new file mode 100644
index 0000000..6f748fb
--- /dev/null
+++ b/server/gstreamer-encoder.c
@@ -0,0 +1,558 @@
+/* -*- Mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ Copyright (C) 2015 Jeremy White
+ Copyright (C) 2015-2016 Francois Gouget
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <gst/gst.h>
+#include <gst/app/gstappsrc.h>
+#include <gst/app/gstappsink.h>
+
+#include "red-common.h"
+#include "video-encoder.h"
+
+
+#define SPICE_GST_DEFAULT_FPS 30
+
+
+typedef struct {
+ SpiceBitmapFmt spice_format;
+ const char *format;
+ uint32_t bpp;
+} SpiceFormatForGStreamer;
+
+typedef struct SpiceGstEncoder {
+ VideoEncoder base;
+
+ /* Rate control callbacks */
+ VideoEncoderRateControlCbs cbs;
+
+ /* Spice's initial bit rate estimation in bits per second. */
+ uint64_t starting_bit_rate;
+
+ /* ---------- Video characteristics ---------- */
+
+ uint32_t width;
+ uint32_t height;
+ const SpiceFormatForGStreamer *format;
+ SpiceBitmapFmt spice_format;
+
+ /* ---------- GStreamer pipeline ---------- */
+
+ /* Pointers to the GStreamer pipeline elements. If pipeline is NULL the
+ * other pointers are invalid.
+ */
+ GstElement *pipeline;
+ GstAppSink *appsink;
+ GstAppSrc *appsrc;
+ GstCaps *src_caps;
+ GstElement *gstenc;
+
+ /* Pipeline parameters to modify before the next frame. */
+# define SPICE_GST_VIDEO_PIPELINE_STATE 0x1
+# define SPICE_GST_VIDEO_PIPELINE_BITRATE 0x2
+# define SPICE_GST_VIDEO_PIPELINE_CAPS 0x4
+ uint32_t set_pipeline;
+
+ /* The bit rate target for the outgoing network stream. (bits per second) */
+ uint64_t bit_rate;
+
+ /* The minimum bit rate. */
+# define SPICE_GST_MIN_BITRATE (128 * 1024)
+
+ /* The default bit rate. */
+# define SPICE_GST_DEFAULT_BITRATE (8 * 1024 * 1024)
+} SpiceGstEncoder;
+
+
+/* ---------- Miscellaneous SpiceGstEncoder helpers ---------- */
+
+static inline double get_mbps(uint64_t bit_rate)
+{
+ return (double)bit_rate / 1024 / 1024;
+}
+
+/* Returns the source frame rate which may change at any time so don't store
+ * the result.
+ */
+static uint32_t get_source_fps(SpiceGstEncoder *encoder)
+{
+ return encoder->cbs.get_source_fps ?
+ encoder->cbs.get_source_fps(encoder->cbs.opaque) : SPICE_GST_DEFAULT_FPS;
+}
+
+static void set_pipeline_changes(SpiceGstEncoder *encoder, uint32_t flags)
+{
+ encoder->set_pipeline |= flags;
+}
+
+static void free_pipeline(SpiceGstEncoder *encoder)
+{
+ if (encoder->src_caps) {
+ gst_caps_unref(encoder->src_caps);
+ encoder->src_caps = NULL;
+ }
+ if (encoder->pipeline) {
+ gst_element_set_state(encoder->pipeline, GST_STATE_NULL);
+ gst_object_unref(encoder->appsrc);
+ gst_object_unref(encoder->gstenc);
+ gst_object_unref(encoder->appsink);
+ gst_object_unref(encoder->pipeline);
+ encoder->pipeline = NULL;
+ }
+}
+
+/* The maximum bit rate we will use for the current video.
+ *
+ * This is based on a 10x compression ratio which should be more than enough
+ * for even MJPEG to provide good quality.
+ */
+static uint64_t get_bit_rate_cap(SpiceGstEncoder *encoder)
+{
+ uint32_t raw_frame_bits = encoder->width * encoder->height * encoder->format->bpp;
+ return raw_frame_bits * get_source_fps(encoder) / 10;
+}
+
+static void adjust_bit_rate(SpiceGstEncoder *encoder)
+{
+ if (encoder->bit_rate == 0) {
+ /* Use the default value, */
+ encoder->bit_rate = SPICE_GST_DEFAULT_BITRATE;
+ } else if (encoder->bit_rate < SPICE_GST_MIN_BITRATE) {
+ /* don't let the bit rate go too low */
+ encoder->bit_rate = SPICE_GST_MIN_BITRATE;
+ } else {
+ /* or too high */
+ encoder->bit_rate = MIN(encoder->bit_rate, get_bit_rate_cap(encoder));
+ }
+ spice_debug("adjust_bit_rate(%.3fMbps)", get_mbps(encoder->bit_rate));
+}
+
+
+/* ---------- GStreamer pipeline ---------- */
+
+/* A helper for spice_gst_encoder_encode_frame() */
+static const SpiceFormatForGStreamer *map_format(SpiceBitmapFmt format)
+{
+ /* See GStreamer's part-mediatype-video-raw.txt and
+ * section-types-definitions.html documents.
+ */
+ static const SpiceFormatForGStreamer format_map[] = {
+ {SPICE_BITMAP_FMT_RGBA, "BGRA", 32},
+ {SPICE_BITMAP_FMT_16BIT, "RGB15", 16},
+ /* TODO: Test the other formats */
+ {SPICE_BITMAP_FMT_32BIT, "BGRx", 32},
+ {SPICE_BITMAP_FMT_24BIT, "BGR", 24},
+ };
+
+ int i;
+ for (i = 0; i < G_N_ELEMENTS(format_map); i++) {
+ if (format_map[i].spice_format == format) {
+ if (i > 1) {
+ spice_warning("The %d format has not been tested yet", format);
+ }
+ return &format_map[i];
+ }
+ }
+
+ return NULL;
+}
+
+static void set_appsrc_caps(SpiceGstEncoder *encoder)
+{
+ if (encoder->src_caps) {
+ gst_caps_unref(encoder->src_caps);
+ }
+ encoder->src_caps = gst_caps_new_simple(
+ "video/x-raw",
+ "format", G_TYPE_STRING, encoder->format->format,
+ "width", G_TYPE_INT, encoder->width,
+ "height", G_TYPE_INT, encoder->height,
+ "framerate", GST_TYPE_FRACTION, get_source_fps(encoder), 1,
+ NULL);
+ gst_app_src_set_caps(encoder->appsrc, encoder->src_caps);
+}
+
+static gboolean create_pipeline(SpiceGstEncoder *encoder)
+{
+ GError *err = NULL;
+ /* Set max-threads to ensure zero-frame latency */
+ const gchar *desc = "appsrc is-live=true format=time do-timestamp=true name=src ! videoconvert ! avenc_mjpeg max-threads=1 name=encoder ! appsink name=sink";
+ spice_debug("GStreamer pipeline: %s", desc);
+ encoder->pipeline = gst_parse_launch_full(desc, NULL, GST_PARSE_FLAG_FATAL_ERRORS, &err);
+ if (!encoder->pipeline || err) {
+ spice_warning("GStreamer error: %s", err->message);
+ g_clear_error(&err);
+ if (encoder->pipeline) {
+ gst_object_unref(encoder->pipeline);
+ encoder->pipeline = NULL;
+ }
+ return FALSE;
+ }
+ encoder->appsrc = GST_APP_SRC(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "src"));
+ encoder->gstenc = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "encoder");
+ encoder->appsink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(encoder->pipeline), "sink"));
+
+ /* See https://bugzilla.gnome.org/show_bug.cgi?id=753257 */
+ spice_debug("removing the pipeline clock");
+ gst_pipeline_use_clock(GST_PIPELINE(encoder->pipeline), NULL);
+
+ set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_STATE |
+ SPICE_GST_VIDEO_PIPELINE_BITRATE |
+ SPICE_GST_VIDEO_PIPELINE_CAPS);
+
+ return TRUE;
+}
+
+/* A helper for configure_pipeline() */
+static void set_gstenc_bitrate(SpiceGstEncoder *encoder)
+{
+ adjust_bit_rate(encoder);
+ g_object_set(G_OBJECT(encoder->gstenc),
+ "bitrate", (gint)encoder->bit_rate, NULL);
+}
+
+/* A helper for spice_gst_encoder_encode_frame() */
+static gboolean configure_pipeline(SpiceGstEncoder *encoder)
+{
+ if (!encoder->pipeline && !create_pipeline(encoder)) {
+ return FALSE;
+ }
+ if (!encoder->set_pipeline) {
+ return TRUE;
+ }
+
+ /* If the pipeline state does not need to be changed it's because it is
+ * already in the PLAYING state. So first set it to the NULL state so it
+ * can be (re)configured.
+ */
+ if (!(encoder->set_pipeline & SPICE_GST_VIDEO_PIPELINE_STATE) &&
+ gst_element_set_state(encoder->pipeline, GST_STATE_NULL) == GST_STATE_CHANGE_FAILURE) {
+ spice_debug("GStreamer error: could not stop the pipeline");
+ free_pipeline(encoder);
+ return FALSE;
+ }
+
+ /* Configure the encoder bitrate */
+ if (encoder->set_pipeline & SPICE_GST_VIDEO_PIPELINE_BITRATE) {
+ set_gstenc_bitrate(encoder);
+ }
+
+ /* Set the source caps */
+ if (encoder->set_pipeline & SPICE_GST_VIDEO_PIPELINE_CAPS) {
+ set_appsrc_caps(encoder);
+ }
+
+ /* Start playing */
+ if (gst_element_set_state(encoder->pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
+ spice_warning("GStreamer error: unable to set the pipeline to the playing state");
+ free_pipeline(encoder);
+ return FALSE;
+ }
+
+ encoder->set_pipeline = 0;
+ return TRUE;
+}
+
+/* A helper for the *_copy() functions */
+static int is_chunk_stride_aligned(const SpiceBitmap *bitmap, uint32_t index)
+{
+ SpiceChunks *chunks = bitmap->data;
+ if (chunks->chunk[index].len % bitmap->stride != 0) {
+ /* A line straddles two chunks. This is not supported */
+ spice_warning("chunk %d/%d contains an incomplete line, cannot copy",
+ index, chunks->num_chunks);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+/* A helper for push_raw_frame() */
+static inline int line_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
+ uint32_t chunk_offset, uint32_t stream_stride,
+ uint32_t height, uint8_t *buffer)
+{
+ uint8_t *dst = buffer;
+ SpiceChunks *chunks = bitmap->data;
+ uint32_t chunk_index = 0;
+ for (int l = 0; l < height; l++) {
+ /* We may have to move forward by more than one chunk the first
+ * time around. This also protects us against 0-byte chunks.
+ */
+ while (chunk_offset >= chunks->chunk[chunk_index].len) {
+ if (!is_chunk_stride_aligned(bitmap, chunk_index)) {
+ return FALSE;
+ }
+ chunk_offset -= chunks->chunk[chunk_index].len;
+ chunk_index++;
+ }
+
+ /* Copy the line */
+ uint8_t *src = chunks->chunk[chunk_index].data + chunk_offset;
+ memcpy(dst, src, stream_stride);
+ dst += stream_stride;
+ chunk_offset += bitmap->stride;
+ }
+ spice_return_val_if_fail(dst - buffer == stream_stride * height, FALSE);
+ return TRUE;
+}
+
+/* A helper for push_raw_frame() */
+static inline int chunk_copy(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
+ uint32_t chunk_offset, uint32_t len, uint8_t *dst)
+{
+ SpiceChunks *chunks = bitmap->data;
+ uint32_t chunk_index = 0;
+ /* Skip chunks until we find the start of the frame */
+ while (chunk_index < chunks->num_chunks &&
+ chunk_offset >= chunks->chunk[chunk_index].len) {
+ if (!is_chunk_stride_aligned(bitmap, chunk_index)) {
+ return FALSE;
+ }
+ chunk_offset -= chunks->chunk[chunk_index].len;
+ chunk_index++;
+ }
+
+ /* We can copy the frame chunk by chunk */
+ while (len && chunk_index < chunks->num_chunks) {
+ if (!is_chunk_stride_aligned(bitmap, chunk_index)) {
+ return FALSE;
+ }
+ uint8_t *src = chunks->chunk[chunk_index].data + chunk_offset;
+ uint32_t thislen = MIN(chunks->chunk[chunk_index].len - chunk_offset, len);
+ memcpy(dst, src, thislen);
+ dst += thislen;
+ len -= thislen;
+ chunk_offset = 0;
+ chunk_index++;
+ }
+ spice_return_val_if_fail(len == 0, FALSE);
+ return TRUE;
+}
+
+/* A helper for spice_gst_encoder_encode_frame() */
+static int push_raw_frame(SpiceGstEncoder *encoder, const SpiceBitmap *bitmap,
+ const SpiceRect *src, int top_down)
+{
+ uint32_t height = src->bottom - src->top;
+ uint32_t stream_stride = (src->right - src->left) * encoder->format->bpp / 8;
+ uint32_t len = stream_stride * height;
+ GstBuffer *buffer = gst_buffer_new_and_alloc(len);
+ GstMapInfo map;
+ gst_buffer_map(buffer, &map, GST_MAP_WRITE);
+ uint8_t *dst = map.data;
+
+ /* Note that we should not reorder the lines, even if top_down is false.
+ * It just changes the number of lines to skip at the start of the bitmap.
+ */
+ uint32_t skip_lines = top_down ? src->top : bitmap->y - (src->bottom - 0);
+ uint32_t chunk_offset = bitmap->stride * skip_lines;
+
+ if (stream_stride != bitmap->stride) {
+ /* We have to do a line-by-line copy because for each we have to
+ * leave out pixels on the left or right.
+ */
+ chunk_offset += src->left * encoder->format->bpp / 8;
+ if (!line_copy(encoder, bitmap, chunk_offset, stream_stride, height, dst)) {
+ gst_buffer_unmap(buffer, &map);
+ gst_buffer_unref(buffer);
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+ } else {
+ /* We can copy the bitmap chunk by chunk */
+ if (!chunk_copy(encoder, bitmap, chunk_offset, len, dst)) {
+ gst_buffer_unmap(buffer, &map);
+ gst_buffer_unref(buffer);
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+ }
+ gst_buffer_unmap(buffer, &map);
+
+ GstFlowReturn ret = gst_app_src_push_buffer(encoder->appsrc, buffer);
+ if (ret != GST_FLOW_OK) {
+ spice_warning("GStreamer error: unable to push source buffer (%d)", ret);
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+
+ return VIDEO_ENCODER_FRAME_ENCODE_DONE;
+}
+
+/* A helper for spice_gst_encoder_encode_frame() */
+static int pull_compressed_buffer(SpiceGstEncoder *encoder,
+ uint8_t **outbuf, size_t *outbuf_size,
+ uint32_t *data_size)
+{
+ spice_return_val_if_fail(outbuf && outbuf_size, VIDEO_ENCODER_FRAME_UNSUPPORTED);
+
+ GstSample *sample = gst_app_sink_pull_sample(encoder->appsink);
+ if (sample) {
+ GstMapInfo map;
+ GstBuffer *buffer = gst_sample_get_buffer(sample);
+ if (buffer && gst_buffer_map(buffer, &map, GST_MAP_READ)) {
+ gint size = gst_buffer_get_size(buffer);
+ if (!*outbuf || *outbuf_size < size) {
+ free(*outbuf);
+ *outbuf = spice_malloc(size);
+ *outbuf_size = size;
+ }
+ /* TODO Try to avoid this copy by changing the GstBuffer handling */
+ memcpy(*outbuf, map.data, size);
+ *data_size = size;
+ gst_buffer_unmap(buffer, &map);
+ gst_sample_unref(sample);
+ return VIDEO_ENCODER_FRAME_ENCODE_DONE;
+ }
+ gst_sample_unref(sample);
+ }
+ spice_debug("failed to pull the compressed buffer");
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+}
+
+
+/* ---------- VideoEncoder's public API ---------- */
+
+static void spice_gst_encoder_destroy(VideoEncoder *video_encoder)
+{
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ free_pipeline(encoder);
+ free(encoder);
+}
+
+static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
+ uint32_t frame_mm_time,
+ const SpiceBitmap *bitmap,
+ const SpiceRect *src, int top_down,
+ uint8_t **outbuf, size_t *outbuf_size,
+ uint32_t *data_size)
+{
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+
+ uint32_t width = src->right - src->left;
+ uint32_t height = src->bottom - src->top;
+ if (width != encoder->width || height != encoder->height ||
+ encoder->spice_format != bitmap->format) {
+ spice_debug("video format change: width %d -> %d, height %d -> %d, format %d -> %d",
+ encoder->width, width, encoder->height, height,
+ encoder->spice_format, bitmap->format);
+ encoder->format = map_format(bitmap->format);
+ if (!encoder->format) {
+ spice_warning("unable to map format type %d", bitmap->format);
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+ encoder->spice_format = bitmap->format;
+ encoder->width = width;
+ encoder->height = height;
+ if (encoder->pipeline) {
+ set_pipeline_changes(encoder, SPICE_GST_VIDEO_PIPELINE_CAPS);
+ }
+ }
+
+ if (!configure_pipeline(encoder)) {
+ return VIDEO_ENCODER_FRAME_UNSUPPORTED;
+ }
+
+ int rc = push_raw_frame(encoder, bitmap, src, top_down);
+ if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ rc = pull_compressed_buffer(encoder, outbuf, outbuf_size, data_size);
+ if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ /* The input buffer will be stuck in the pipeline, preventing
+ * later ones from being processed. Furthermore something went
+ * wrong with this pipeline, so it may be safer to rebuild it
+ * from scratch.
+ */
+ free_pipeline(encoder);
+ }
+ }
+ return rc;
+}
+
+static void spice_gst_encoder_client_stream_report(VideoEncoder *video_encoder,
+ uint32_t num_frames,
+ uint32_t num_drops,
+ uint32_t start_frame_mm_time,
+ uint32_t end_frame_mm_time,
+ int32_t end_frame_delay,
+ uint32_t audio_delay)
+{
+ spice_debug("client report: #frames %u, #drops %d, duration %u video-delay %d audio-delay %u",
+ num_frames, num_drops,
+ end_frame_mm_time - start_frame_mm_time,
+ end_frame_delay, audio_delay);
+}
+
+static void spice_gst_encoder_notify_server_frame_drop(VideoEncoder *video_encoder)
+{
+ spice_debug("server report: getting frame drops...");
+}
+
+static uint64_t spice_gst_encoder_get_bit_rate(VideoEncoder *video_encoder)
+{
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ return encoder->bit_rate;
+}
+
+static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
+ VideoEncoderStats *stats)
+{
+ SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
+ uint64_t raw_bit_rate = encoder->width * encoder->height * (encoder->format ? encoder->format->bpp : 0) * get_source_fps(encoder);
+
+ spice_return_if_fail(stats != NULL);
+ stats->starting_bit_rate = encoder->starting_bit_rate;
+ stats->cur_bit_rate = encoder->bit_rate;
+
+ /* Use the compression level as a proxy for the quality */
+ stats->avg_quality = stats->cur_bit_rate ? 100.0 - raw_bit_rate / stats->cur_bit_rate : 0;
+ if (stats->avg_quality < 0) {
+ stats->avg_quality = 0;
+ }
+}
+
+VideoEncoder *gstreamer_encoder_new(uint64_t starting_bit_rate,
+ VideoEncoderRateControlCbs *cbs)
+{
+ GError *err = NULL;
+ if (!gst_init_check(NULL, NULL, &err)) {
+ spice_warning("GStreamer error: %s", err->message);
+ g_clear_error(&err);
+ return NULL;
+ }
+
+ SpiceGstEncoder *encoder = spice_new0(SpiceGstEncoder, 1);
+ encoder->base.destroy = spice_gst_encoder_destroy;
+ encoder->base.encode_frame = spice_gst_encoder_encode_frame;
+ encoder->base.client_stream_report = spice_gst_encoder_client_stream_report;
+ encoder->base.notify_server_frame_drop = spice_gst_encoder_notify_server_frame_drop;
+ encoder->base.get_bit_rate = spice_gst_encoder_get_bit_rate;
+ encoder->base.get_stats = spice_gst_encoder_get_stats;
+
+ if (cbs) {
+ encoder->cbs = *cbs;
+ }
+ encoder->starting_bit_rate = starting_bit_rate;
+
+ /* All the other fields are initialized to zero by spice_new0(). */
+
+ if (!create_pipeline(encoder)) {
+ /* Some GStreamer dependency is probably missing */
+ free(encoder);
+ encoder = NULL;
+ }
+ return (VideoEncoder*)encoder;
+}
diff --git a/server/stream.c b/server/stream.c
index d791918..b203612 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -706,6 +706,20 @@ static void update_client_playback_delay(void *opaque, uint32_t delay_ms)
agent->dcc->streams_max_latency);
}
+/* A helper for dcc_create_stream(). */
+static VideoEncoder* dcc_create_video_encoder(uint64_t starting_bit_rate,
+ VideoEncoderRateControlCbs *cbs)
+{
+#ifdef HAVE_GSTREAMER_1_0
+ VideoEncoder* video_encoder = gstreamer_encoder_new(starting_bit_rate, cbs);
+ if (video_encoder) {
+ return video_encoder;
+ }
+#endif
+ /* Use the builtin MJPEG video encoder as a fallback */
+ return mjpeg_encoder_new(starting_bit_rate, cbs);
+}
+
void dcc_create_stream(DisplayChannelClient *dcc, Stream *stream)
{
StreamAgent *agent = &dcc->stream_agents[get_stream_id(DCC_TO_DC(dcc), stream)];
@@ -733,9 +747,9 @@ void dcc_create_stream(DisplayChannelClient *dcc, Stream *stream)
video_cbs.update_client_playback_delay = update_client_playback_delay;
initial_bit_rate = get_initial_bit_rate(dcc, stream);
- agent->video_encoder = mjpeg_encoder_new(initial_bit_rate, &video_cbs);
+ agent->video_encoder = dcc_create_video_encoder(initial_bit_rate, &video_cbs);
} else {
- agent->video_encoder = mjpeg_encoder_new(0, NULL);
+ agent->video_encoder = dcc_create_video_encoder(0, NULL);
}
red_channel_client_pipe_add(RED_CHANNEL_CLIENT(dcc), stream_create_item_new(agent));
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 7e06a13..708432b 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -45,7 +45,7 @@ struct VideoEncoder {
*
* @encoder: The video encoder.
* @frame_mm_time: The frame's mm-time timestamp in milliseconds.
- * @bitmap: The Spice screen.
+ * @bitmap: A bitmap containing the source video frame.
* @src: A rectangle specifying the area occupied by the video.
* @top_down: If true the first video line is specified by src.top.
* @outbuf: The buffer for the compressed frame. This must either
@@ -160,5 +160,9 @@ typedef struct VideoEncoderRateControlCbs {
*/
VideoEncoder* mjpeg_encoder_new(uint64_t starting_bit_rate,
VideoEncoderRateControlCbs *cbs);
+#ifdef HAVE_GSTREAMER_1_0
+VideoEncoder* gstreamer_encoder_new(uint64_t starting_bit_rate,
+ VideoEncoderRateControlCbs *cbs);
+#endif
#endif
commit 1b69198c4ec73110251e0ebf969275e98950808e
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Jun 3 16:07:44 2016 +0200
streaming: Remove the width/height encode_frame() parameters
encode_frame() needs the QXL_DRAW_COPY operation's SpiceCopy.src_area
field anyway, so the width and height parameters were redundant.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/dcc-send.c b/server/dcc-send.c
index 0486959..145099d 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1680,7 +1680,7 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
SpiceCopy *copy;
uint32_t frame_mm_time;
uint32_t n;
- int is_sized, width, height;
+ int is_sized;
int ret;
spice_assert(drawable->red_drawable->type == QXL_DRAW_COPY);
@@ -1690,9 +1690,8 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
return FALSE;
}
- width = copy->src_area.right - copy->src_area.left;
- height = copy->src_area.bottom - copy->src_area.top;
- is_sized = (width != stream->width) || (height != stream->height) ||
+ is_sized = (copy->src_area.right - copy->src_area.left != stream->width) ||
+ (copy->src_area.bottom - copy->src_area.top != stream->height) ||
!rect_is_equal(&drawable->red_drawable->bbox, &stream->dest_area);
if (is_sized &&
@@ -1722,8 +1721,7 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
ret = agent->video_encoder->encode_frame(agent->video_encoder,
frame_mm_time,
©->src_bitmap->u.bitmap,
- width, height, ©->src_area,
- stream->top_down,
+ ©->src_area, stream->top_down,
&dcc->send_data.stream_outbuf,
&outbuf_size, &n);
switch (ret) {
@@ -1761,8 +1759,8 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
stream_data.base.id = get_stream_id(display, stream);
stream_data.base.multi_media_time = frame_mm_time;
stream_data.data_size = n;
- stream_data.width = width;
- stream_data.height = height;
+ stream_data.width = copy->src_area.right - copy->src_area.left;
+ stream_data.height = copy->src_area.bottom - copy->src_area.top;
stream_data.dest = drawable->red_drawable->bbox;
spice_debug("stream %d: sized frame: dest ==> ", stream_data.base.id);
diff --git a/server/mjpeg-encoder.c b/server/mjpeg-encoder.c
index 7dcea50..57708cd 100644
--- a/server/mjpeg-encoder.c
+++ b/server/mjpeg-encoder.c
@@ -927,7 +927,6 @@ static int encode_frame(MJpegEncoder *encoder, const SpiceRect *src,
static int mjpeg_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
- int width, int height,
const SpiceRect *src, int top_down,
uint8_t **outbuf, size_t *outbuf_size,
uint32_t *data_size)
diff --git a/server/video-encoder.h b/server/video-encoder.h
index 8aa7783..7e06a13 100644
--- a/server/video-encoder.h
+++ b/server/video-encoder.h
@@ -60,7 +60,7 @@ struct VideoEncoder {
* only happen if rate control is active.
*/
int (*encode_frame)(VideoEncoder *encoder, uint32_t frame_mm_time,
- const SpiceBitmap *bitmap, int width, int height,
+ const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
uint8_t **outbuf, size_t *outbuf_size,
uint32_t *data_size);
commit 032cb0ce85b44da3ee5d0308909164452e25bff5
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Tue May 24 17:25:58 2016 +0200
mjpeg: Use src_area as the authoritative source for the frame dimensions
Video frames correspond to QXL_DRAW_COPY operations where the frame area
is defined by the SpiceCopy.src_area field.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/mjpeg-encoder.c b/server/mjpeg-encoder.c
index e3646db..7dcea50 100644
--- a/server/mjpeg-encoder.c
+++ b/server/mjpeg-encoder.c
@@ -706,7 +706,7 @@ static void mjpeg_encoder_adjust_fps(MJpegEncoder *encoder, uint64_t now)
*/
static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
SpiceBitmapFmt format,
- int width, int height,
+ const SpiceRect *src,
uint8_t **dest, size_t *dest_len,
uint32_t frame_mm_time)
{
@@ -777,10 +777,12 @@ static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
+ encoder->cinfo.image_width = src->right - src->left;
+ encoder->cinfo.image_height = src->bottom - src->top;
if (encoder->pixel_converter != NULL) {
- unsigned int stride = width * 3;
+ JDIMENSION stride = encoder->cinfo.image_width * 3;
/* check for integer overflow */
- if (stride < width) {
+ if (stride < encoder->cinfo.image_width) {
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
if (encoder->row_size < stride) {
@@ -790,9 +792,6 @@ static int mjpeg_encoder_start_frame(MJpegEncoder *encoder,
}
spice_jpeg_mem_dest(&encoder->cinfo, dest, dest_len);
-
- encoder->cinfo.image_width = width;
- encoder->cinfo.image_height = height;
jpeg_set_defaults(&encoder->cinfo);
encoder->cinfo.dct_method = JDCT_IFAST;
quality = mjpeg_quality_samples[encoder->rate_control.quality_id];
@@ -935,8 +934,7 @@ static int mjpeg_encoder_encode_frame(VideoEncoder *video_encoder,
{
MJpegEncoder *encoder = (MJpegEncoder*)video_encoder;
- int ret = mjpeg_encoder_start_frame(encoder, bitmap->format,
- width, height,
+ int ret = mjpeg_encoder_start_frame(encoder, bitmap->format, src,
outbuf, outbuf_size,
frame_mm_time);
if (ret != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
commit 618255168e28f8d5d6db6aa3ede94446095ec311
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Jun 10 12:43:57 2016 +0200
streaming: Simplify is_next_stream_frame()
After the removal of Drawable::sized_stream, we no longer need to detect
if the stream changes size in is_next_stream_frame() so it can return a
boolean rather than a value from an enum.
diff --git a/server/stream.c b/server/stream.c
index 457ee47..d791918 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -216,24 +216,24 @@ static void update_copy_graduality(DisplayChannel *display, Drawable *drawable)
}
}
-static int is_next_stream_frame(DisplayChannel *display,
- const Drawable *candidate,
- const int other_src_width,
- const int other_src_height,
- const SpiceRect *other_dest,
- const red_time_t other_time,
- const Stream *stream,
- int container_candidate_allowed)
+static bool is_next_stream_frame(DisplayChannel *display,
+ const Drawable *candidate,
+ const int other_src_width,
+ const int other_src_height,
+ const SpiceRect *other_dest,
+ const red_time_t other_time,
+ const Stream *stream,
+ int container_candidate_allowed)
{
RedDrawable *red_drawable;
if (!candidate->streamable) {
- return STREAM_FRAME_NONE;
+ return FALSE;
}
if (candidate->creation_time - other_time >
(stream ? RED_STREAM_CONTINUS_MAX_DELTA : RED_STREAM_DETACTION_MAX_DELTA)) {
- return STREAM_FRAME_NONE;
+ return FALSE;
}
red_drawable = candidate->red_drawable;
@@ -241,13 +241,13 @@ static int is_next_stream_frame(DisplayChannel *display,
SpiceRect* candidate_src;
if (!rect_is_equal(&red_drawable->bbox, other_dest)) {
- return STREAM_FRAME_NONE;
+ return FALSE;
}
candidate_src = &red_drawable->u.copy.src_area;
if (candidate_src->right - candidate_src->left != other_src_width ||
candidate_src->bottom - candidate_src->top != other_src_height) {
- return STREAM_FRAME_NONE;
+ return FALSE;
}
} else {
if (rect_contains(&red_drawable->bbox, other_dest)) {
@@ -261,20 +261,20 @@ static int is_next_stream_frame(DisplayChannel *display,
rect_debug(other_dest);
spice_debug("new box ==>");
rect_debug(&red_drawable->bbox);
- return STREAM_FRAME_NONE;
+ return FALSE;
}
} else {
- return STREAM_FRAME_NONE;
+ return FALSE;
}
}
if (stream) {
SpiceBitmap *bitmap = &red_drawable->u.copy.src_bitmap->u.bitmap;
if (stream->top_down != !!(bitmap->flags & SPICE_BITMAP_FLAGS_TOP_DOWN)) {
- return STREAM_FRAME_NONE;
+ return FALSE;
}
}
- return STREAM_FRAME_NATIVE;
+ return TRUE;
}
static void attach_stream(DisplayChannel *display, Drawable *drawable, Stream *stream)
@@ -513,15 +513,15 @@ void stream_trace_update(DisplayChannel *display, Drawable *drawable)
FOREACH_STREAMS(display, item) {
Stream *stream = SPICE_CONTAINEROF(item, Stream, link);
- int is_next_frame = is_next_stream_frame(display,
- drawable,
- stream->width,
- stream->height,
- &stream->dest_area,
- stream->last_time,
- stream,
- TRUE);
- if (is_next_frame != STREAM_FRAME_NONE) {
+ bool is_next_frame = is_next_stream_frame(display,
+ drawable,
+ stream->width,
+ stream->height,
+ &stream->dest_area,
+ stream->last_time,
+ stream,
+ TRUE);
+ if (is_next_frame) {
if (stream->current) {
stream->current->streamable = FALSE; //prevent item trace
before_reattach_stream(display, stream, drawable);
@@ -536,8 +536,7 @@ void stream_trace_update(DisplayChannel *display, Drawable *drawable)
trace_end = trace + NUM_TRACE_ITEMS;
for (; trace < trace_end; trace++) {
if (is_next_stream_frame(display, drawable, trace->width, trace->height,
- &trace->dest_area, trace->time, NULL, FALSE) !=
- STREAM_FRAME_NONE) {
+ &trace->dest_area, trace->time, NULL, FALSE)) {
if (stream_add_frame(display, drawable,
trace->first_frame_time,
trace->frames_count,
@@ -552,7 +551,7 @@ void stream_trace_update(DisplayChannel *display, Drawable *drawable)
void stream_maintenance(DisplayChannel *display,
Drawable *candidate, Drawable *prev)
{
- int is_next_frame;
+ bool is_next_frame;
if (candidate->stream) {
return;
@@ -565,7 +564,7 @@ void stream_maintenance(DisplayChannel *display,
stream->width, stream->height,
&stream->dest_area, stream->last_time,
stream, TRUE);
- if (is_next_frame != STREAM_FRAME_NONE) {
+ if (is_next_frame) {
before_reattach_stream(display, stream, candidate);
detach_stream(display, stream);
prev->streamable = FALSE; //prevent item trace
@@ -580,7 +579,7 @@ void stream_maintenance(DisplayChannel *display,
&prev->red_drawable->bbox, prev->creation_time,
prev->stream,
FALSE);
- if (is_next_frame != STREAM_FRAME_NONE) {
+ if (is_next_frame) {
stream_add_frame(display, candidate,
prev->first_frame_time,
prev->frames_count,
diff --git a/server/stream.h b/server/stream.h
index 94ced44..9dcb8f7 100644
--- a/server/stream.h
+++ b/server/stream.h
@@ -52,12 +52,6 @@ typedef struct RedStreamActivateReportItem {
uint32_t stream_id;
} RedStreamActivateReportItem;
-enum {
- STREAM_FRAME_NONE,
- STREAM_FRAME_NATIVE,
- STREAM_FRAME_CONTAINER,
-};
-
#define STREAM_STATS
#ifdef STREAM_STATS
typedef struct StreamStats {
commit 47509b1e6ef5783f0c843a2a6fc4fe3d85a5bb01
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Jun 10 12:43:16 2016 +0200
streaming: Remove unused detach_stream() argument
After the removal of Drawable::sized_stream, the last argument to
detach_stream() is no longer used.
diff --git a/server/display-channel.c b/server/display-channel.c
index f1e8ba1..76d22f9 100644
--- a/server/display-channel.c
+++ b/server/display-channel.c
@@ -1337,7 +1337,7 @@ void drawable_unref(Drawable *drawable)
spice_warn_if_fail(ring_is_empty(&drawable->pipes));
if (drawable->stream) {
- detach_stream(display, drawable->stream, TRUE);
+ detach_stream(display, drawable->stream);
}
region_destroy(&drawable->tree_item.base.rgn);
diff --git a/server/stream.c b/server/stream.c
index f527c71..457ee47 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -321,8 +321,7 @@ static void attach_stream(DisplayChannel *display, Drawable *drawable, Stream *s
}
}
-void detach_stream(DisplayChannel *display, Stream *stream,
- int detach_sized)
+void detach_stream(DisplayChannel *display, Stream *stream)
{
spice_assert(stream->current && stream->current->stream);
spice_assert(stream->current->stream == stream);
@@ -526,7 +525,7 @@ void stream_trace_update(DisplayChannel *display, Drawable *drawable)
if (stream->current) {
stream->current->streamable = FALSE; //prevent item trace
before_reattach_stream(display, stream, drawable);
- detach_stream(display, stream, FALSE);
+ detach_stream(display, stream);
}
attach_stream(display, drawable, stream);
return;
@@ -568,7 +567,7 @@ void stream_maintenance(DisplayChannel *display,
stream, TRUE);
if (is_next_frame != STREAM_FRAME_NONE) {
before_reattach_stream(display, stream, candidate);
- detach_stream(display, stream, FALSE);
+ detach_stream(display, stream);
prev->streamable = FALSE; //prevent item trace
attach_stream(display, candidate, stream);
}
@@ -859,7 +858,7 @@ static void detach_stream_gracefully(DisplayChannel *display, Stream *stream,
dcc_detach_stream_gracefully(dcc, stream, update_area_limit);
}
if (stream->current) {
- detach_stream(display, stream, TRUE);
+ detach_stream(display, stream);
}
}
@@ -896,11 +895,11 @@ void stream_detach_behind(DisplayChannel *display, QRegion *region, Drawable *dr
}
}
if (detach && stream->current) {
- detach_stream(display, stream, TRUE);
+ detach_stream(display, stream);
} else if (!is_connected) {
if (stream->current &&
region_intersects(&stream->current->tree_item.base.rgn, region)) {
- detach_stream(display, stream, TRUE);
+ detach_stream(display, stream);
}
}
}
diff --git a/server/stream.h b/server/stream.h
index 715f920..94ced44 100644
--- a/server/stream.h
+++ b/server/stream.h
@@ -161,6 +161,6 @@ void stream_agent_unref (DisplayChan
StreamAgent *agent);
void stream_agent_stop (StreamAgent *agent);
-void detach_stream(DisplayChannel *display, Stream *stream, int detach_sized);
+void detach_stream(DisplayChannel *display, Stream *stream);
#endif /* STREAM_H */
commit 42a5794845d0ee4b34ac523b8ad5a6c453d2203c
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Jun 10 12:43:12 2016 +0200
streaming: Remove the Drawable.sized_stream field
Only red_marshall_stream_data() needs to know whether to send the frame
using a SpiceMsgDisplayStreamDataSized or a regular StreamData message.
So check whether we have a sized frame there and simplify the rest of
the code.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/dcc-send.c b/server/dcc-send.c
index bab1217..0486959 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1683,10 +1683,6 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
int is_sized, width, height;
int ret;
- if (!stream) {
- spice_assert(drawable->sized_stream);
- stream = drawable->sized_stream;
- }
spice_assert(drawable->red_drawable->type == QXL_DRAW_COPY);
copy = &drawable->red_drawable->u.copy;
@@ -1696,16 +1692,13 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
width = copy->src_area.right - copy->src_area.left;
height = copy->src_area.bottom - copy->src_area.top;
- is_sized = (drawable->sized_stream != NULL);
+ is_sized = (width != stream->width) || (height != stream->height) ||
+ !rect_is_equal(&drawable->red_drawable->bbox, &stream->dest_area);
if (is_sized &&
!red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_SIZED_STREAM)) {
return FALSE;
}
- if (!is_sized) {
- width = stream->width;
- height = stream->height;
- }
StreamAgent *agent = &dcc->stream_agents[get_stream_id(display, stream)];
uint64_t time_now = spice_get_monotonic_time_ns();
@@ -2150,7 +2143,7 @@ static void marshall_qxl_drawable(RedChannelClient *rcc,
spice_return_if_fail(display);
/* allow sized frames to be streamed, even if they where replaced by another frame, since
* newer frames might not cover sized frames completely if they are bigger */
- if ((item->stream || item->sized_stream) && red_marshall_stream_data(rcc, m, item)) {
+ if (item->stream && red_marshall_stream_data(rcc, m, item)) {
return;
}
if (display->enable_jpeg)
diff --git a/server/display-channel.h b/server/display-channel.h
index 8588583..0baad62 100644
--- a/server/display-channel.h
+++ b/server/display-channel.h
@@ -70,7 +70,6 @@ struct Drawable {
int gradual_frames_count;
int last_gradual_frame;
Stream *stream;
- Stream *sized_stream;
int streamable;
BitmapGradualType copy_bitmap_graduality;
DependItem depend_items[3];
diff --git a/server/stream.c b/server/stream.c
index d8be148..f527c71 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -226,7 +226,6 @@ static int is_next_stream_frame(DisplayChannel *display,
int container_candidate_allowed)
{
RedDrawable *red_drawable;
- int is_frame_container = FALSE;
if (!candidate->streamable) {
return STREAM_FRAME_NONE;
@@ -252,7 +251,6 @@ static int is_next_stream_frame(DisplayChannel *display,
}
} else {
if (rect_contains(&red_drawable->bbox, other_dest)) {
- SpiceRect* candidate_src;
int candidate_area = rect_get_area(&red_drawable->bbox);
int other_area = rect_get_area(other_dest);
/* do not stream drawables that are significantly
@@ -265,13 +263,6 @@ static int is_next_stream_frame(DisplayChannel *display,
rect_debug(&red_drawable->bbox);
return STREAM_FRAME_NONE;
}
-
- candidate_src = &red_drawable->u.copy.src_area;
- if (candidate_area > other_area ||
- candidate_src->right - candidate_src->left != other_src_width ||
- candidate_src->bottom - candidate_src->top != other_src_height) {
- is_frame_container = TRUE;
- }
} else {
return STREAM_FRAME_NONE;
}
@@ -283,11 +274,7 @@ static int is_next_stream_frame(DisplayChannel *display,
return STREAM_FRAME_NONE;
}
}
- if (is_frame_container) {
- return STREAM_FRAME_CONTAINER;
- } else {
- return STREAM_FRAME_NATIVE;
- }
+ return STREAM_FRAME_NATIVE;
}
static void attach_stream(DisplayChannel *display, Drawable *drawable, Stream *stream)
@@ -340,9 +327,6 @@ void detach_stream(DisplayChannel *display, Stream *stream,
spice_assert(stream->current && stream->current->stream);
spice_assert(stream->current->stream == stream);
stream->current->stream = NULL;
- if (detach_sized) {
- stream->current->sized_stream = NULL;
- }
stream->current = NULL;
}
@@ -545,9 +529,6 @@ void stream_trace_update(DisplayChannel *display, Drawable *drawable)
detach_stream(display, stream, FALSE);
}
attach_stream(display, drawable, stream);
- if (is_next_frame == STREAM_FRAME_CONTAINER) {
- drawable->sized_stream = stream;
- }
return;
}
}
@@ -590,9 +571,6 @@ void stream_maintenance(DisplayChannel *display,
detach_stream(display, stream, FALSE);
prev->streamable = FALSE; //prevent item trace
attach_stream(display, candidate, stream);
- if (is_next_frame == STREAM_FRAME_CONTAINER) {
- candidate->sized_stream = stream;
- }
}
} else if (candidate->streamable) {
SpiceRect* prev_src = &prev->red_drawable->u.copy.src_area;
@@ -833,13 +811,12 @@ static void dcc_detach_stream_gracefully(DisplayChannelClient *dcc,
/* (1) The caller should detach the drawable from the stream. This will
* lead to sending the drawable losslessly, as an ordinary drawable. */
if (dcc_drawable_is_in_pipe(dcc, stream->current)) {
- spice_debug("stream %d: upgrade by linked drawable. sized %d, box ==>",
- stream_id, stream->current->sized_stream != NULL);
+ spice_debug("stream %d: upgrade by linked drawable. box ==>",
+ stream_id);
rect_debug(&stream->current->red_drawable->bbox);
goto clear_vis_region;
}
- spice_debug("stream %d: upgrade by drawable. sized %d, box ==>",
- stream_id, stream->current->sized_stream != NULL);
+ spice_debug("stream %d: upgrade by drawable. box ==>", stream_id);
rect_debug(&stream->current->red_drawable->bbox);
rcc = RED_CHANNEL_CLIENT(dcc);
upgrade_item = spice_new(RedUpgradeItem, 1);
commit 28f2e425c4e9d86570970d49a7a3eee43e24134e
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Jun 10 12:39:24 2016 +0200
streaming: Rework red_marshall_stream_data a bit
This code just refactors the function without doing any functional
changes. The actual changes will be in the next commit, and this will
make the next commit much more obvious.
diff --git a/server/dcc-send.c b/server/dcc-send.c
index c0c7573..bab1217 100644
--- a/server/dcc-send.c
+++ b/server/dcc-send.c
@@ -1677,10 +1677,10 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
DisplayChannelClient *dcc = RCC_TO_DCC(rcc);
DisplayChannel *display = DCC_TO_DC(dcc);
Stream *stream = drawable->stream;
- SpiceImage *image;
+ SpiceCopy *copy;
uint32_t frame_mm_time;
uint32_t n;
- int width, height;
+ int is_sized, width, height;
int ret;
if (!stream) {
@@ -1689,22 +1689,20 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
}
spice_assert(drawable->red_drawable->type == QXL_DRAW_COPY);
- image = drawable->red_drawable->u.copy.src_bitmap;
-
- if (image->descriptor.type != SPICE_IMAGE_TYPE_BITMAP) {
+ copy = &drawable->red_drawable->u.copy;
+ if (copy->src_bitmap->descriptor.type != SPICE_IMAGE_TYPE_BITMAP) {
return FALSE;
}
- if (drawable->sized_stream) {
- if (red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_SIZED_STREAM)) {
- SpiceRect *src_rect = &drawable->red_drawable->u.copy.src_area;
+ width = copy->src_area.right - copy->src_area.left;
+ height = copy->src_area.bottom - copy->src_area.top;
+ is_sized = (drawable->sized_stream != NULL);
- width = src_rect->right - src_rect->left;
- height = src_rect->bottom - src_rect->top;
- } else {
- return FALSE;
- }
- } else {
+ if (is_sized &&
+ !red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_SIZED_STREAM)) {
+ return FALSE;
+ }
+ if (!is_sized) {
width = stream->width;
height = stream->height;
}
@@ -1730,8 +1728,8 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
outbuf_size = dcc->send_data.stream_outbuf_size;
ret = agent->video_encoder->encode_frame(agent->video_encoder,
frame_mm_time,
- &image->u.bitmap, width, height,
- &drawable->red_drawable->u.copy.src_area,
+ ©->src_bitmap->u.bitmap,
+ width, height, ©->src_area,
stream->top_down,
&dcc->send_data.stream_outbuf,
&outbuf_size, &n);
@@ -1752,7 +1750,7 @@ static int red_marshall_stream_data(RedChannelClient *rcc,
}
dcc->send_data.stream_outbuf_size = outbuf_size;
- if (!drawable->sized_stream) {
+ if (!is_sized) {
SpiceMsgDisplayStreamData stream_data;
red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DATA, NULL);
commit 709c9d71f1dea7a9992174f64d13061016247273
Author: Francois Gouget <fgouget at codeweavers.com>
Date: Fri Jun 3 15:51:30 2016 +0200
streaming: Better check for sized frames
Usually the RedDrawable bbox dimensions match the src_area dimensions
so that checking that the bbox matches the stream's original dest_area
should be enough to determine if sized stream support is needed to
send the frame.
But making the bbox different could be used to have the scaling be
performed on the client side. So it's better not to assume the bbox and
src_area have the same dimensions.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
diff --git a/server/stream.c b/server/stream.c
index 2aa13d1..d8be148 100644
--- a/server/stream.c
+++ b/server/stream.c
@@ -252,6 +252,7 @@ static int is_next_stream_frame(DisplayChannel *display,
}
} else {
if (rect_contains(&red_drawable->bbox, other_dest)) {
+ SpiceRect* candidate_src;
int candidate_area = rect_get_area(&red_drawable->bbox);
int other_area = rect_get_area(other_dest);
/* do not stream drawables that are significantly
@@ -265,7 +266,10 @@ static int is_next_stream_frame(DisplayChannel *display,
return STREAM_FRAME_NONE;
}
- if (candidate_area > other_area) {
+ candidate_src = &red_drawable->u.copy.src_area;
+ if (candidate_area > other_area ||
+ candidate_src->right - candidate_src->left != other_src_width ||
+ candidate_src->bottom - candidate_src->top != other_src_height) {
is_frame_container = TRUE;
}
} else {
More information about the Spice-commits
mailing list