[Spice-devel] [PATCH v5 11/20] server: Shape the bit rate of the GStreamer video encoders output.
Francois Gouget
fgouget at codeweavers.com
Thu Aug 27 12:01:19 PDT 2015
The GStreamer encoders don't follow the specified bit rate very closely: they can decide to exceed it for ten seconds or more if they consider the scene deserves it. Such long bursts are enough to cause network congestion, resulting in many lost frames which cause significant display corruption.
So the GStreamer video encoder now uses a short 300ms virtual buffer to shape the compressed video output and ensure we don't exceed the target bit rate for any significant length of time.
It could instead rely on the network feedback (when available) to lower the bit rate. However frequent GStreamer bit rate changes lower the overall video compression level and also result in a lower average bit rate, both of which result in lower video quality.
The GStreamer video encoder also keeps track of the encoded frame size so it can gather statistics and call update_client_playback_delay() with accurate information and also annotate the client report debug traces with the corresponding bit rate information.
Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
---
Changes since take 4:
- Disable the bit rate shaping if rate_control_is_active() is false.
server/gstreamer_encoder.c | 293 +++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 285 insertions(+), 8 deletions(-)
diff --git a/server/gstreamer_encoder.c b/server/gstreamer_encoder.c
index e42641b..6f111fd 100644
--- a/server/gstreamer_encoder.c
+++ b/server/gstreamer_encoder.c
@@ -39,6 +39,9 @@ typedef struct GstEncoder GstEncoder;
# define DO_ZERO_COPY
#endif
+#define NANO_SECOND (1000000000LL)
+#define MILLI_SECOND (1000LL)
+#define NANO_MS (NANO_SECOND / MILLI_SECOND)
typedef struct {
SpiceBitmapFmt spice_format;
@@ -60,6 +63,11 @@ struct GstVideoBuffer {
gboolean persistent;
};
+typedef struct {
+ uint32_t mm_time;
+ uint32_t size;
+} GstFrameInformation;
+
struct GstEncoder {
VideoEncoder base;
@@ -99,6 +107,45 @@ struct GstEncoder {
/* The frame counter for GStreamer buffers */
uint32_t frame;
+
+ /* ---------- Encoded frame statistics ---------- */
+
+ /* Should be >= than FRAME_STATISTICS_COUNT. This is also used to annotate
+ * the client report debug traces with bit rate information.
+ */
+# define GSTE_HISTORY_SIZE 60
+
+ /* A circular buffer containing the past encoded frames information. */
+ GstFrameInformation history[GSTE_HISTORY_SIZE];
+
+ /* The indices of the oldest and newest frames in the history buffer. */
+ uint32_t history_first;
+ uint32_t history_last;
+
+ /* How many frames to take into account when computing the effective
+ * bit rate, average frame size, etc. This should be large enough so the
+ * I and P frames average out, and short enough for it to reflect the
+ * current situation.
+ */
+# define GSTE_FRAME_STATISTICS_COUNT 21
+
+ /* The index of the oldest frame taken into account for the statistics. */
+ uint32_t stat_first;
+
+ /* Used to compute the average frame size. */
+ uint64_t stat_sum;
+
+ /* Tracks the maximum frame size. */
+ uint32_t stat_maximum;
+
+
+ /* ---------- Encoder bit rate control ----------
+ *
+ * GStreamer encoders don't follow the specified bit rate very
+ * closely. These fields are used to ensure we don't exceed the desired
+ * stream bit rate, regardless of the GStreamer encoder's output.
+ */
+
/* The bit rate target for the outgoing network stream. (bits per second) */
uint64_t bit_rate;
@@ -107,6 +154,27 @@ struct GstEncoder {
/* The default bit rate */
# define GSTE_DEFAULT_BITRATE (8 * 1024 * 1024)
+
+ /* The bit rate control is performed using a virtual buffer to allow short
+ * term variations: bursts are allowed until the virtual buffer is full.
+ * Then frames are dropped to limit the bit rate. VBUFFER_SIZE defines the
+ * size of the virtual buffer in milliseconds worth of data.
+ */
+# define GSTE_VBUFFER_SIZE 300
+
+ int32_t vbuffer_size;
+ int32_t vbuffer_free;
+
+ /* When dropping frames, this is set to the minimum mm_time of the next
+ * frame to encode. Otherwise set to zero.
+ */
+ uint32_t next_frame;
+
+ /* Defines the minimum allowed fps. */
+# define GSTE_MAX_PERIOD (NANO_SECOND / 3)
+
+ /* How big of a margin to take to cover for latency jitter. */
+# define GSTE_LATENCY_MARGIN 0.1
};
@@ -158,6 +226,18 @@ static uint32_t get_source_fps(GstEncoder *encoder)
encoder->cbs.get_source_fps(encoder->cbs_opaque) : GSTE_DEFAULT_FPS;
}
+static uint32_t get_network_latency(GstEncoder *encoder)
+{
+ /* Assume that the network latency is symmetric */
+ return encoder->cbs.get_roundtrip_ms ?
+ encoder->cbs.get_roundtrip_ms(encoder->cbs_opaque) / 2 : 0;
+}
+
+static inline int rate_control_is_active(GstEncoder* encoder)
+{
+ return encoder->cbs.get_roundtrip_ms != NULL;
+}
+
static void reset_pipeline(GstEncoder *encoder)
{
if (!encoder->pipeline) {
@@ -175,6 +255,180 @@ static void reset_pipeline(GstEncoder *encoder)
encoder->pipeline = NULL;
}
+
+/* ---------- Encoded frame statistics ---------- */
+
+static inline uint32_t get_last_frame_mm_time(GstEncoder *encoder)
+{
+ return encoder->history[encoder->history_last].mm_time;
+}
+
+/* Returns the current bit rate based on the last GSTE_FRAME_STATISTICS_COUNT
+ * frames.
+ */
+static uint64_t get_effective_bit_rate(GstEncoder *encoder)
+{
+ uint32_t elapsed = encoder->history[encoder->history_last].mm_time -
+ encoder->history[encoder->stat_first].mm_time;
+ if (encoder->next_frame) {
+ elapsed += encoder->next_frame - get_last_frame_mm_time(encoder);
+ } else {
+ elapsed += MILLI_SECOND / get_source_fps(encoder);
+ }
+ return elapsed ? encoder->stat_sum * 8 * MILLI_SECOND / elapsed : 0;
+}
+
+static uint64_t get_average_frame_size(GstEncoder *encoder)
+{
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? GSTE_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ return encoder->stat_sum / count;
+}
+
+static uint32_t get_maximum_frame_size(GstEncoder *encoder)
+{
+ if (encoder->stat_maximum == 0) {
+ uint32_t index = encoder->history_last;
+ while (1) {
+ encoder->stat_maximum = MAX(encoder->stat_maximum,
+ encoder->history[index].size);
+ if (index == encoder->stat_first) {
+ break;
+ }
+ index = (index ? index : GSTE_HISTORY_SIZE) - 1;
+ }
+ }
+ return encoder->stat_maximum;
+}
+
+/* Returns the bit rate of the specified period. from and to must be the
+ * mm time of the first and last frame to consider.
+ */
+static uint64_t get_period_bit_rate(GstEncoder *encoder, uint32_t from, uint32_t to)
+{
+ uint32_t sum = 0;
+ uint32_t last_mm_time = 0;
+ uint32_t index = encoder->history_last;
+ while (1) {
+ if (encoder->history[index].mm_time == to) {
+ if (last_mm_time == 0) {
+ /* We don't know how much time elapsed between the period's
+ * last frame and the next so we cannot include it.
+ */
+ sum = 1;
+ last_mm_time = to;
+ } else {
+ sum = encoder->history[index].size + 1;
+ }
+
+ } else if (encoder->history[index].mm_time == from) {
+ sum += encoder->history[index].size;
+ return (sum - 1) * 8 * MILLI_SECOND / (last_mm_time - from);
+
+ } else if (index == encoder->history_first) {
+ /* This period is outside the recorded history */
+ spice_debug("period (%u-%u) outside known history (%u-%u)",
+ from, to,
+ encoder->history[encoder->history_first].mm_time,
+ encoder->history[encoder->history_last].mm_time);
+ return 0;
+
+ } else if (sum > 0) {
+ sum += encoder->history[index].size;
+
+ } else {
+ last_mm_time = encoder->history[index].mm_time;
+ }
+ index = (index ? index : GSTE_HISTORY_SIZE) - 1;
+ }
+
+}
+
+static void add_frame(GstEncoder *encoder, uint32_t frame_mm_time, uint32_t size)
+{
+ /* Update the statistics */
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? GSTE_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ if (count == GSTE_FRAME_STATISTICS_COUNT) {
+ encoder->stat_sum -= encoder->history[encoder->stat_first].size;
+ if (encoder->stat_maximum == encoder->history[encoder->stat_first].size) {
+ encoder->stat_maximum = 0;
+ }
+ encoder->stat_first = (encoder->stat_first + 1) % GSTE_HISTORY_SIZE;
+ }
+ encoder->stat_sum += size;
+ if (encoder->stat_maximum > 0 && size > encoder->stat_maximum) {
+ encoder->stat_maximum = size;
+ }
+
+ /* Update the frame history */
+ encoder->history_last = (encoder->history_last + 1) % GSTE_HISTORY_SIZE;
+ if (encoder->history_last == encoder->history_first) {
+ encoder->history_first = (encoder->history_first + 1) % GSTE_HISTORY_SIZE;
+ }
+ encoder->history[encoder->history_last].mm_time = frame_mm_time;
+ encoder->history[encoder->history_last].size = size;
+}
+
+
+/* ---------- Encoder bit rate control ---------- */
+
+static uint32_t get_min_playback_delay(GstEncoder *encoder)
+{
+ /* Make sure the delay is large enough to send a large frame (typically an
+ * I frame) and an average frame. This also takes into account the frames
+ * dropped by the encoder bit rate control.
+ */
+ uint32_t size = get_maximum_frame_size(encoder) + get_average_frame_size(encoder);
+ uint32_t send_time = MILLI_SECOND * size * 8 / encoder->bit_rate;
+
+ /* Also factor in the network latency with a margin for jitter. */
+ uint32_t net_latency = get_network_latency(encoder) * (1.0 + GSTE_LATENCY_MARGIN);
+
+ return send_time + net_latency;
+}
+
+static void update_client_playback_delay(GstEncoder *encoder)
+{
+ if (encoder->cbs.update_client_playback_delay) {
+ uint32_t min_delay = get_min_playback_delay(encoder);
+ encoder->cbs.update_client_playback_delay(encoder->cbs_opaque, min_delay);
+ }
+}
+
+static void update_next_frame(GstEncoder *encoder)
+{
+ if (encoder->vbuffer_free >= 0) {
+ encoder->next_frame = 0;
+ return;
+ }
+
+ /* Figure out how many frames to drop to not exceed the current bit rate.
+ * Use nanoseconds to avoid precision loss.
+ */
+ uint64_t delay_ns = -encoder->vbuffer_free * 8 * NANO_SECOND / encoder->bit_rate;
+ uint64_t period_ns = NANO_SECOND / get_source_fps(encoder);
+ uint32_t drops = (delay_ns + period_ns - 1) / period_ns; /* round up */
+ spice_debug("drops=%u vbuffer %d/%d", drops, encoder->vbuffer_free,
+ encoder->vbuffer_size);
+
+ delay_ns = drops * period_ns + period_ns / 2;
+ if (delay_ns > GSTE_MAX_PERIOD) {
+ delay_ns = GSTE_MAX_PERIOD;
+ }
+ encoder->next_frame = get_last_frame_mm_time(encoder) + delay_ns / NANO_MS;
+
+ /* Drops mean a higher delay between encoded frames so update the playback
+ * delay.
+ */
+ update_client_playback_delay(encoder);
+}
+
+
+/* ---------- Network bit rate control ---------- */
+
/* The maximum bit rate we will use for the current video.
*
* This is based on a 10x compression ratio which should be more than enough
@@ -676,10 +930,22 @@ static int gst_encoder_encode_frame(GstEncoder *encoder,
encoder->spice_format = bitmap->format;
encoder->width = width;
encoder->height = height;
- if (encoder->pipeline) {
+ if (encoder->bit_rate == 0) {
+ encoder->history[0].mm_time = frame_mm_time;
+ encoder->bit_rate = encoder->starting_bit_rate;
+ adjust_bit_rate(encoder);
+ encoder->vbuffer_free = 0; /* Slow start */
+ } else if (encoder->pipeline) {
reconfigure_pipeline(encoder);
}
}
+
+ if (rate_control_is_active(encoder) &&
+ frame_mm_time < encoder->next_frame) {
+ /* Drop the frame to limit the outgoing bit rate. */
+ return VIDEO_ENCODER_FRAME_DROP;
+ }
+
if (!encoder->pipeline && !construct_pipeline(encoder, bitmap)) {
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
@@ -692,6 +958,13 @@ static int gst_encoder_encode_frame(GstEncoder *encoder,
spice_assert(!encoder->needs_bitmap);
#endif
}
+ if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ return rc;
+ }
+ add_frame(encoder, frame_mm_time, (*buffer)->base.size);
+
+ update_next_frame(encoder);
+
return rc;
}
@@ -703,10 +976,12 @@ static void gst_encoder_client_stream_report(GstEncoder *encoder,
int32_t end_frame_delay,
uint32_t audio_delay)
{
- spice_debug("client report: #frames %u, #drops %d, duration %u video-delay %d audio-delay %u",
- num_frames, num_drops,
- end_frame_mm_time - start_frame_mm_time,
- end_frame_delay, audio_delay);
+ uint64_t period_bit_rate = get_period_bit_rate(encoder, start_frame_mm_time, end_frame_mm_time);
+ spice_debug("client report: %u/%u drops in %ums margins video %3d audio %3u bw %.3f/%.3fMbps",
+ num_drops, num_frames, end_frame_mm_time - start_frame_mm_time,
+ end_frame_delay, audio_delay,
+ get_mbps(period_bit_rate),
+ get_mbps(get_effective_bit_rate(encoder)));
}
static void gst_encoder_notify_server_frame_drop(GstEncoder *encoder)
@@ -716,7 +991,7 @@ static void gst_encoder_notify_server_frame_drop(GstEncoder *encoder)
static uint64_t gst_encoder_get_bit_rate(GstEncoder *encoder)
{
- return encoder->bit_rate;
+ return get_effective_bit_rate(encoder);
}
static void gst_encoder_get_stats(GstEncoder *encoder, VideoEncoderStats *stats)
@@ -725,7 +1000,7 @@ static void gst_encoder_get_stats(GstEncoder *encoder, VideoEncoderStats *stats)
spice_assert(encoder != NULL && stats != NULL);
stats->starting_bit_rate = encoder->starting_bit_rate;
- stats->cur_bit_rate = encoder->bit_rate;
+ stats->cur_bit_rate = get_effective_bit_rate(encoder);
/* Use the compression level as a proxy for the quality */
stats->avg_quality = stats->cur_bit_rate ? 100.0 - raw_bit_rate / stats->cur_bit_rate : 0;
@@ -741,6 +1016,8 @@ GstEncoder *create_gstreamer_encoder(SpiceVideoCodecType codec_type,
{
GstEncoder *encoder;
+ spice_assert(GSTE_FRAME_STATISTICS_COUNT <= GSTE_HISTORY_SIZE);
+
spice_assert(!cbs || (cbs && cbs->get_roundtrip_ms && cbs->get_source_fps));
if (codec_type != SPICE_VIDEO_CODEC_TYPE_MJPEG &&
codec_type != SPICE_VIDEO_CODEC_TYPE_VP8 &&
@@ -765,7 +1042,7 @@ GstEncoder *create_gstreamer_encoder(SpiceVideoCodecType codec_type,
encoder->cbs = *cbs;
}
encoder->cbs_opaque = cbs_opaque;
- encoder->bit_rate = encoder->starting_bit_rate = starting_bit_rate;
+ encoder->starting_bit_rate = starting_bit_rate;
/* All the other fields are initialized to zero by spice_new0(). */
--
2.5.0
More information about the Spice-devel
mailing list