[Spice-devel] [PATCH spice 10/11] server/video-streams: adjust mjpeg quality and frame rate according to the current bit rate
Yonit Halperin
yhalperi at redhat.com
Sun Apr 8 08:43:19 PDT 2012
Previously, the mjpeg quality was always 70. The frame rate was tuned
according to the frames' congestion in the pipe.
This patch sets the mjpeg quality and frame rate according
to the compressed size of the frames and the currently available bit
rate.
The compression size is estimated for different jpeg qualities,
and the bit rate is evaluated using qos queries (see red_channel).
The bit rate and compression size are monitored for major changes, and
when they occur, the mjpeg settings are re-evaluated.
In addition, the settings are fine-tuned correspondingly to the frames
pipe congestion.
Signed-off-by: Yonit Halperin <yhalperi at redhat.com>
---
server/red_worker.c | 385 ++++++++++++++++++++++++++++++++++++++++++++++-----
1 files changed, 347 insertions(+), 38 deletions(-)
diff --git a/server/red_worker.c b/server/red_worker.c
index a9942cf..92e1197 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -114,7 +114,8 @@
#define RED_STREAM_MIN_SIZE (96 * 96)
#define FPS_TEST_INTERVAL 1
-#define MAX_FPS 30
+#define STREAM_MAX_FPS 30
+#define STREAM_MIN_FPS 1
//best bit rate per pixel base on 13000000 bps for frame size 720x576 pixels and 25 fps
#define BEST_BIT_RATE_PER_PIXEL 38
@@ -396,6 +397,13 @@ struct Stream {
int bit_rate;
};
+#define STREAM_JPEG_QUALITY_SAMPLE_NUM 4
+static const int stream_jpeg_quality_samples[STREAM_JPEG_QUALITY_SAMPLE_NUM] = {15, 25, 50, 70};
+
+#define STREAM_FRAME_SIZE_CHANGE_TH 1.5
+#define STREAM_BIT_RATE_CHANGE_TH 1.25
+#define STREAM_AVERAGE_SIZE_WINDOW 3
+
typedef struct StreamAgent {
QRegion vis_region;
PipeItem create_item;
@@ -403,6 +411,34 @@ typedef struct StreamAgent {
Stream *stream;
uint64_t last_send_time;
+ /*
+ Adjusting the stream jpeg quality and frame rate (fps):
+ When during_sampling=TRUE, we compress different frames with different
+ jpeg quality. By using (1) the resulting compression ratio, (2) the current
+ channel bandwidth, and (3) the existance of other streams,
+ we evaulate the max frame frequency for the stream with the given quality,
+ and we choose the highest quality that will allow a reasonable frame rate.
+ during_sampling is set for new streams and also when the bandwidth and/or
+ average frame size significantly change.
+ */
+ struct {
+ int quality_id;
+ uint64_t quality_sample_size[STREAM_JPEG_QUALITY_SAMPLE_NUM];
+ int during_sampling;
+ /* low limit for the the current sampling */
+ int min_sample_quality_id;
+ int min_sample_quality_fps; // min fps for the given quality
+ /* tracking the best sampled fps so far */
+ int max_sampled_fps;
+ int max_sampled_fps_quality_id;
+ int byte_rate;
+ /* tracking the average frame size with the current jpeg quality */
+ uint64_t size_sum;
+ int size_summed_count;
+ uint64_t recent_size_sum;
+ int recent_size_summed_count;
+ } jpeg;
+
int frames;
int drops;
int fps;
@@ -938,6 +974,7 @@ typedef struct RedWorker {
Ring streams;
ItemTrace items_trace[NUM_TRACE_ITEMS];
uint32_t next_item_trace;
+ uint64_t streams_size_total;
QuicData quic_data;
QuicContext *quic;
@@ -2512,6 +2549,7 @@ static void red_attach_stream(RedWorker *worker, Drawable *drawable, Stream *str
region_clone(&agent->vis_region, &drawable->tree_item.base.rgn);
push_stream_clip_by_drawable(dcc, agent, drawable);
}
+ agent->frames++;
}
}
@@ -2522,6 +2560,7 @@ static void red_stop_stream(RedWorker *worker, Stream *stream)
spice_assert(ring_item_is_linked(&stream->link));
spice_assert(!stream->current);
+ spice_debug("id %ld", stream - worker->streams_buf);
WORKER_FOREACH_DCC(worker, item, dcc) {
StreamAgent *stream_agent;
stream_agent = &dcc->stream_agents[stream - worker->streams_buf];
@@ -2530,6 +2569,7 @@ static void red_stop_stream(RedWorker *worker, Stream *stream)
stream->refs++;
red_channel_client_pipe_add(&dcc->common.base, &stream_agent->destroy_item);
}
+ worker->streams_size_total -= stream->width * stream->height;
ring_remove(&stream->link);
red_release_stream(worker, stream);
}
@@ -2741,11 +2781,10 @@ static int get_minimal_bit_rate(RedWorker *worker, int width, int height)
return ret;
}
-static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
+static void red_stream_agent_init(DisplayChannelClient *dcc, Stream *stream)
{
StreamAgent *agent = &dcc->stream_agents[stream - dcc->common.worker->streams_buf];
- stream->refs++;
spice_assert(region_is_empty(&agent->vis_region));
if (stream->current) {
agent->frames = 1;
@@ -2754,8 +2793,36 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
agent->frames = 0;
}
agent->drops = 0;
- agent->fps = MAX_FPS;
+ agent->fps = STREAM_MAX_FPS;
reset_rate(dcc, agent);
+ agent->jpeg.quality_id = STREAM_JPEG_QUALITY_SAMPLE_NUM / 2;
+ memset(agent->jpeg.quality_sample_size, 0,
+ sizeof(agent->jpeg.quality_sample_size[0]) * STREAM_JPEG_QUALITY_SAMPLE_NUM);
+ agent->jpeg.byte_rate = 0;
+ agent->jpeg.max_sampled_fps = 0;
+ agent->jpeg.max_sampled_fps_quality_id = 0;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ agent->jpeg.size_sum = 0;
+ agent->jpeg.size_summed_count = 0;
+ agent->jpeg.recent_size_sum = 0;
+ agent->jpeg.recent_size_summed_count = 0;
+ agent->jpeg.during_sampling = FALSE;
+}
+
+static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
+{
+ StreamAgent *agent = &dcc->stream_agents[stream - dcc->common.worker->streams_buf];
+
+ stream->refs++;
+
+ spice_debug("id %ld %dx%d dest (%d,%d), (%d, %d)",
+ stream - dcc->common.worker->streams_buf,
+ stream->width, stream->height,
+ stream->dest_area.left, stream->dest_area.top,
+ stream->dest_area.right, stream->dest_area.bottom);
+ red_stream_agent_init(dcc, stream);
+
red_channel_client_pipe_add(&dcc->common.base, &agent->create_item);
}
@@ -2794,6 +2861,7 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
SpiceBitmap *bitmap = &drawable->red_drawable->u.copy.src_bitmap->u.bitmap;
stream->top_down = !!(bitmap->flags & SPICE_BITMAP_FLAGS_TOP_DOWN);
drawable->stream = stream;
+ worker->streams_size_total += stream->width * stream->height;
WORKER_FOREACH_DCC(worker, dcc_ring_item, dcc) {
red_display_create_stream(dcc, stream);
@@ -2945,16 +3013,9 @@ static void reset_rate(DisplayChannelClient *dcc, StreamAgent *stream_agent)
/* MJpeg has no rate limiting anyway, so do nothing */
}
-static int display_channel_client_is_low_bandwidth(DisplayChannelClient *dcc)
-{
- return main_channel_client_is_low_bandwidth(
- red_client_get_main(red_channel_client_get_client(&dcc->common.base)));
-}
-
static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
{
DrawablePipeItem *dpi;
- DisplayChannelClient *dcc;
int index;
StreamAgent *agent;
RingItem *ring_item;
@@ -2967,35 +3028,10 @@ static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
index = stream - worker->streams_buf;
DRAWABLE_FOREACH_DPI(stream->current, ring_item, dpi) {
- dcc = dpi->dcc;
- if (!display_channel_client_is_low_bandwidth(dcc)) {
- continue;
- }
- agent = &dcc->stream_agents[index];
-
+ agent = &dpi->dcc->stream_agents[index];
if (pipe_item_is_linked(&dpi->dpi_pipe_item)) {
++agent->drops;
}
-
- if (agent->frames / agent->fps < FPS_TEST_INTERVAL) {
- agent->frames++;
- return;
- }
-
- double drop_factor = ((double)agent->frames - (double)agent->drops) /
- (double)agent->frames;
-
- if (drop_factor == 1) {
- if (agent->fps < MAX_FPS) {
- agent->fps++;
- }
- } else if (drop_factor < 0.9) {
- if (agent->fps > 1) {
- agent->fps--;
- }
- }
- agent->frames = 1;
- agent->drops = 0;
}
}
@@ -8078,6 +8114,274 @@ static int encode_frame (RedWorker *worker, const SpiceRect *src,
return TRUE;
}
+static inline void red_stream_agent_jpeg_quality_set(StreamAgent *agent, int quality_id)
+{
+ if (!agent->jpeg.during_sampling) {
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = 0;
+ }
+ agent->jpeg.size_sum = 0;
+ agent->jpeg.size_summed_count = 0;
+ agent->jpeg.recent_size_sum = 0;
+ agent->jpeg.recent_size_summed_count = 0;
+ agent->jpeg.quality_id = quality_id;
+}
+
+/*
+ Adjust the stream's jpeg quality and frame rate.
+ We evaluate the compression ratio of different jpeg qualities;
+ We compress successive frames with different qualities,
+ and then we estimate the stream frame rate with the current jpeg quality
+ and availalbe bit rate.
+*/
+static inline void red_stream_do_quality_size_sampling(DisplayChannelClient *dcc, StreamAgent *agent)
+{
+ int fps;
+ int stream_id = agent - dcc->stream_agents;
+
+ spice_assert(agent->jpeg.during_sampling);
+ if (agent->jpeg.quality_sample_size[agent->jpeg.quality_id] == 0) {
+ return;
+ }
+
+ fps = agent->jpeg.byte_rate / agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ spice_debug("stream %d: jpeg %d: %.2f (KB) fps %d",
+ stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id]/1000.0,
+ fps);
+
+ if (fps > agent->jpeg.max_sampled_fps ||
+ (fps == agent->jpeg.max_sampled_fps &&
+ agent->jpeg.quality_id > agent->jpeg.max_sampled_fps_quality_id)) {
+ agent->jpeg.max_sampled_fps = fps;
+ agent->jpeg.max_sampled_fps_quality_id = agent->jpeg.quality_id;
+ }
+
+ // assuming monotonicity
+ if (fps > 5 && fps >= 0.75 * agent->jpeg.min_sample_quality_fps) {
+ if (agent->jpeg.quality_id + 1 == STREAM_JPEG_QUALITY_SAMPLE_NUM ||
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id + 1] != 0) {
+ /* best quality has been reached, or the next better quality was
+ * already sampled and didn't pass the fps threshold */
+ goto complete_sample;
+ } else {
+ agent->jpeg.quality_id++;
+ }
+ } else {
+ if (agent->jpeg.quality_id == 0 ||
+ agent->jpeg.quality_id <= agent->jpeg.min_sample_quality_id) {
+ goto complete_sample;
+ } else if (agent->jpeg.quality_sample_size[agent->jpeg.quality_id - 1] != 0) {
+ agent->jpeg.quality_id--;
+ goto complete_sample;
+ } else {
+ agent->jpeg.quality_id--;
+ }
+ }
+ return;
+complete_sample:
+ agent->jpeg.quality_id = MAX(agent->jpeg.quality_id,
+ agent->jpeg.max_sampled_fps_quality_id);
+ agent->fps = agent->jpeg.byte_rate /
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ if (agent->jpeg.quality_id == agent->jpeg.min_sample_quality_id) {
+ agent->fps = MAX(agent->fps, agent->jpeg.min_sample_quality_fps);
+ }
+ agent->fps = MIN(STREAM_MAX_FPS, agent->fps);
+ agent->fps = MAX(STREAM_MIN_FPS, agent->fps);
+ agent->jpeg.during_sampling = FALSE;
+ agent->frames = 1;
+ agent->drops = 0;
+ agent->jpeg.max_sampled_fps = 0;
+ agent->jpeg.max_sampled_fps_quality_id = 0;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ agent->jpeg.size_sum = 0;
+ agent->jpeg.size_summed_count = 0;
+ agent->jpeg.recent_size_sum = agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ agent->jpeg.recent_size_summed_count = 1;
+ memset(agent->jpeg.quality_sample_size, 0,
+ sizeof(agent->jpeg.quality_sample_size[0]) * STREAM_JPEG_QUALITY_SAMPLE_NUM);
+ spice_debug("STREAM QUALITY SAMPLE END %d: quality %d fps %d",
+ stream_id, stream_jpeg_quality_samples[agent->jpeg.quality_id], agent->fps);
+}
+
+/*
+ Fine tuning of the stream's frame rate and quality using
+ the frames congestion in the pipe.
+*/
+static inline void red_stream_update_quality_by_drops(DisplayChannelClient *dcc,
+ StreamAgent *agent)
+{
+ int stream_id = agent - dcc->stream_agents;
+
+ if (agent->drops) {
+ if (agent->frames / agent->fps >= FPS_TEST_INTERVAL) {
+ double drop_factor = ((double)agent->frames - (double)agent->drops) /
+ (double)agent->frames;
+ if (drop_factor <= 0.9) {
+ if (agent->fps <= 10 && agent->jpeg.quality_id > 0 ) {
+ red_stream_agent_jpeg_quality_set(agent, agent->jpeg.quality_id - 1);
+ spice_debug("stream %d quality--: jpeg %d fps %d", stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ } else {
+ agent->fps--;
+ agent->fps = MAX(STREAM_MIN_FPS, agent->fps);
+ spice_debug("stream %d fps--: jpeg %d fps %d", stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ }
+ }
+ agent->frames = 1;
+ agent->drops = 0;
+ }
+ } else {
+ if (agent->frames / agent->fps >= FPS_TEST_INTERVAL) {
+ if (agent->fps >= 15 && agent->jpeg.quality_id < STREAM_JPEG_QUALITY_SAMPLE_NUM - 1) {
+ /* being more strict when we want to increase quality */
+ if (agent->frames / agent->fps >= 2 * FPS_TEST_INTERVAL) {
+ agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id;
+ agent->jpeg.min_sample_quality_fps = agent->fps;
+ agent->jpeg.during_sampling = TRUE;
+ red_stream_do_quality_size_sampling(dcc, agent);
+ spice_debug("stream %d quality resampling: jpeg %d fps %d", stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->frames = 1;
+ agent->drops = 0;
+ } else {
+ agent->fps++;
+ agent->fps = MIN(STREAM_MAX_FPS, agent->fps);
+ spice_debug("stream %d fps++: jpeg %d fps %d", stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ }
+ } else {
+ agent->fps++;
+ agent->fps = MIN(STREAM_MAX_FPS, agent->fps);
+ spice_debug("stream %d fps++: jpeg %d fps %d", stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->frames = 1;
+ agent->drops = 0;
+ }
+ }
+ }
+}
+
+/*
+ Monitor changes in the available byte rate for the stream,
+ and/or in the stream's compressed frames size. If the changes
+ pass a predefined threshold, we re-evaluate the stream's jpeg
+ quality and frame rate.
+*/
+static void red_stream_update_quality(RedChannelClient *rcc, StreamAgent *agent)
+{
+ DisplayChannelClient *dcc = RCC_TO_DCC(rcc);
+ RedWorker *worker = dcc->common.worker;
+ int stream_id = agent->stream - dcc->common.worker->streams_buf;
+ int channel_bit_rate;
+ int stream_byte_rate;
+ float byte_rate_change;
+ float size_change = 1;
+ double size_avg_old = 0.0;
+ double size_avg_new = 0.0;
+
+ channel_bit_rate = red_channel_client_get_qos_bit_rate(rcc);
+
+ if (channel_bit_rate == 0) {
+ spice_assert(agent->jpeg.byte_rate == 0);
+ red_stream_update_quality_by_drops(dcc, agent);
+ return;
+ }
+
+ stream_byte_rate = 0.8 * channel_bit_rate * (agent->stream->width * agent->stream->height) /
+ worker->streams_size_total / 8;
+ if (agent->jpeg.byte_rate == 0) { // new stream
+ agent->jpeg.byte_rate = stream_byte_rate;
+ agent->jpeg.during_sampling = TRUE;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ }
+
+ if (agent->jpeg.during_sampling) {
+ red_stream_do_quality_size_sampling(dcc, agent);
+ return;
+ }
+ spice_assert(agent->jpeg.quality_sample_size[agent->jpeg.quality_id]);
+ byte_rate_change = (stream_byte_rate + 0.0) / agent->jpeg.byte_rate;
+
+ agent->jpeg.recent_size_sum += agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ agent->jpeg.recent_size_summed_count++;
+ size_avg_new = (agent->jpeg.recent_size_sum + 0.0) / agent->jpeg.recent_size_summed_count;
+ if (agent->jpeg.recent_size_summed_count >= STREAM_AVERAGE_SIZE_WINDOW &&
+ agent->jpeg.size_summed_count > 0) {
+ size_avg_old = (agent->jpeg.size_sum + 0.0) / agent->jpeg.size_summed_count;
+ size_change = size_avg_new / size_avg_old;
+ }
+
+ if (byte_rate_change > STREAM_BIT_RATE_CHANGE_TH) {
+ spice_debug("stream %d BYTE RATE CHANGE >>: %.2f (%d-->%d prev-quality %d prev-fps %d",
+ stream_id, byte_rate_change, agent->jpeg.byte_rate, stream_byte_rate,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->jpeg.during_sampling = TRUE;
+ /* byte rate has improved --> don't allow stream to deteriorate */
+ agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id;
+ agent->jpeg.min_sample_quality_fps = agent->fps;
+ } else if (1.0 / byte_rate_change > STREAM_BIT_RATE_CHANGE_TH) {
+ spice_debug("stream %d BYTE RATE CHANGE <<: %.2f (%d-->%d) prev-quality %d prev-fps %d",
+ stream_id, byte_rate_change, agent->jpeg.byte_rate, stream_byte_rate,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ if (agent->fps * size_avg_new > stream_byte_rate) {
+ agent->jpeg.during_sampling = TRUE;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ } else {
+ spice_debug("stream byte rate is not limiting the current setting");
+ }
+ }
+
+ if (size_change > STREAM_FRAME_SIZE_CHANGE_TH) {
+ spice_debug("stream %d SIZE CHANGE >>: %.2f (%.2f-->%.2f) prev-quality %d prev-fps %d",
+ stream_id, size_change, size_avg_old, size_avg_new,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ if (agent->fps * size_avg_new > stream_byte_rate) {
+ agent->jpeg.during_sampling = TRUE;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ } else {
+ spice_debug("stream frame size is not limiting current setting");
+ }
+ } else if (1.0 / size_change > STREAM_FRAME_SIZE_CHANGE_TH) {
+ spice_debug("stream %d SIZE CHANGE <<: %.2f (%.2f-->%.2f) prev-quality %d prev-fps %d",
+ stream_id, size_change, size_avg_old, size_avg_new,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->jpeg.during_sampling = TRUE;
+ /* compression ratio has improved --> don't allow stream to deteriorate */
+ agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id;
+ agent->jpeg.min_sample_quality_fps = agent->fps;
+ }
+ agent->jpeg.byte_rate = stream_byte_rate;
+
+ if (agent->jpeg.recent_size_summed_count >= STREAM_AVERAGE_SIZE_WINDOW) {
+ agent->jpeg.size_sum += agent->jpeg.recent_size_sum;
+ agent->jpeg.size_summed_count += agent->jpeg.recent_size_summed_count;
+ agent->jpeg.recent_size_sum = 0;
+ agent->jpeg.recent_size_summed_count = 0;
+ }
+ if (agent->jpeg.during_sampling) {
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = size_avg_new;
+ red_stream_do_quality_size_sampling(dcc, agent);
+ } else {
+ red_stream_update_quality_by_drops(dcc, agent);
+ }
+}
+
static inline int red_marshall_stream_data(RedChannelClient *rcc,
SpiceMarshaller *base_marshaller, Drawable *drawable)
{
@@ -8124,21 +8428,26 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
return TRUE;
}
+ red_stream_update_quality(rcc, agent);
outbuf_size = dcc->send_data.stream_outbuf_size;
if (!mjpeg_encoder_start_frame(stream->mjpeg_encoder, image->u.bitmap.format,
- 70,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
width, height,
&dcc->send_data.stream_outbuf,
&outbuf_size)) {
+ spice_debug("start frame failed");
return FALSE;
}
if (!encode_frame(worker, &drawable->red_drawable->u.copy.src_area,
&image->u.bitmap, stream)) {
+ spice_debug("encode frame failed");
return FALSE;
}
n = mjpeg_encoder_end_frame(stream->mjpeg_encoder);
dcc->send_data.stream_outbuf_size = outbuf_size;
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = n;
+
if (!drawable->sized_stream) {
SpiceMsgDisplayStreamData stream_data;
--
1.7.7.6
More information about the Spice-devel
mailing list