[Spice-commits] 28 commits - server/dispatcher.h server/inputs_channel.c server/main_channel.c server/main_channel.h server/main_dispatcher.c server/main_dispatcher.h server/Makefile.am server/mjpeg_encoder.c server/mjpeg_encoder.h server/red_channel.c server/red_channel.h server/red_dispatcher.c server/reds.c server/reds.h server/reds-private.h server/red_worker.c server/smartcard.c server/snd_worker.c server/snd_worker.h server/spice_timer_queue.c server/spice_timer_queue.h server/spicevmc.c spice-common
Yonit Halperin
yhalperi at kemper.freedesktop.org
Mon Apr 22 13:32:08 PDT 2013
server/Makefile.am | 2
server/dispatcher.h | 6
server/inputs_channel.c | 1
server/main_channel.c | 7
server/main_channel.h | 1
server/main_dispatcher.c | 32 +
server/main_dispatcher.h | 2
server/mjpeg_encoder.c | 991 ++++++++++++++++++++++++++++++++++++++++++++-
server/mjpeg_encoder.h | 70 +++
server/red_channel.c | 228 ++++++++++
server/red_channel.h | 18
server/red_dispatcher.c | 1
server/red_worker.c | 520 +++++++++++++++++++----
server/reds-private.h | 2
server/reds.c | 28 +
server/reds.h | 2
server/smartcard.c | 1
server/snd_worker.c | 44 +
server/snd_worker.h | 2
server/spice_timer_queue.c | 268 ++++++++++++
server/spice_timer_queue.h | 43 +
server/spicevmc.c | 1
spice-common | 2
23 files changed, 2163 insertions(+), 109 deletions(-)
New commits:
commit 1013b7a5e4dc0805a4fa2b254b87cd56aa3bd157
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Tue Feb 26 11:39:24 2013 -0500
red_worker: assign mm_time to vga frames
diff --git a/server/red_worker.c b/server/red_worker.c
index fdf25e2..4c9a7b0 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -8570,6 +8570,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
Stream *stream = drawable->stream;
SpiceImage *image;
RedWorker *worker = dcc->common.worker;
+ uint32_t frame_mm_time;
int n;
int width, height;
int ret;
@@ -8615,12 +8616,16 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
}
}
+ /* workaround for vga streams */
+ frame_mm_time = drawable->red_drawable->mm_time ?
+ drawable->red_drawable->mm_time :
+ reds_get_mm_time();
outbuf_size = dcc->send_data.stream_outbuf_size;
ret = mjpeg_encoder_start_frame(agent->mjpeg_encoder, image->u.bitmap.format,
width, height,
&dcc->send_data.stream_outbuf,
&outbuf_size,
- drawable->red_drawable->mm_time);
+ frame_mm_time);
switch (ret) {
case MJPEG_ENCODER_FRAME_DROP:
spice_assert(dcc->use_mjpeg_encoder_rate_control);
@@ -8650,7 +8655,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DATA, NULL);
stream_data.base.id = get_stream_id(worker, stream);
- stream_data.base.multi_media_time = drawable->red_drawable->mm_time;
+ stream_data.base.multi_media_time = frame_mm_time;
stream_data.data_size = n;
spice_marshall_msg_display_stream_data(base_marshaller, &stream_data);
@@ -8660,7 +8665,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DATA_SIZED, NULL);
stream_data.base.id = get_stream_id(worker, stream);
- stream_data.base.multi_media_time = drawable->red_drawable->mm_time;
+ stream_data.base.multi_media_time = frame_mm_time;
stream_data.data_size = n;
stream_data.width = width;
stream_data.height = height;
@@ -8676,7 +8681,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
#ifdef STREAM_STATS
agent->stats.num_frames_sent++;
agent->stats.size_sent += n;
- agent->stats.end = drawable->red_drawable->mm_time;
+ agent->stats.end = frame_mm_time;
#endif
return TRUE;
commit 473d41b9f22e9ea730091e276f829c00de19a8d9
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Wed Feb 20 22:19:08 2013 -0500
red_worker: increase the interval limit for stream frames
diff --git a/server/red_worker.c b/server/red_worker.c
index b188dac..fdf25e2 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -110,7 +110,7 @@
#define DISPLAY_FREE_LIST_DEFAULT_SIZE 128
#define RED_STREAM_DETACTION_MAX_DELTA ((1000 * 1000 * 1000) / 5) // 1/5 sec
-#define RED_STREAM_CONTINUS_MAX_DELTA ((1000 * 1000 * 1000) / 2) // 1/2 sec
+#define RED_STREAM_CONTINUS_MAX_DELTA (1000 * 1000 * 1000)
#define RED_STREAM_TIMOUT (1000 * 1000 * 1000)
#define RED_STREAM_FRAMES_START_CONDITION 20
#define RED_STREAM_GRADUAL_FRAMES_START_CONDITION 0.2
commit 1d760551ec6f5152461112a804cfe40053d9da25
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Jan 28 09:11:26 2013 -0500
collect and print video stream statistics
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index b1010e0..4460322 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -167,6 +167,10 @@ struct MJpegEncoder {
MJpegEncoderRateControl rate_control;
MJpegEncoderRateControlCbs cbs;
void *cbs_opaque;
+
+ /* stats */
+ uint64_t avg_quality;
+ uint32_t num_frames;
};
static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
@@ -214,6 +218,7 @@ MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate
void mjpeg_encoder_destroy(MJpegEncoder *encoder)
{
+ spice_debug("avg-quality %.2f", (double)encoder->avg_quality / encoder->num_frames);
jpeg_destroy_compress(&encoder->cinfo);
free(encoder->row);
free(encoder);
@@ -815,6 +820,8 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
jpeg_set_quality(&encoder->cinfo, quality, TRUE);
jpeg_start_compress(&encoder->cinfo, encoder->first_frame);
+ encoder->num_frames++;
+ encoder->avg_quality += quality;
return MJPEG_ENCODER_FRAME_ENCODE_START;
}
diff --git a/server/red_worker.c b/server/red_worker.c
index 21b55c9..b188dac 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -449,6 +449,20 @@ struct Stream {
uint32_t input_fps;
};
+#define STREAM_STATS
+#ifdef STREAM_STATS
+typedef struct StreamStats {
+ uint64_t num_drops_pipe;
+ uint64_t num_drops_fps;
+ uint64_t num_frames_sent;
+ uint64_t num_input_frames;
+ uint64_t size_sent;
+
+ uint64_t start;
+ uint64_t end;
+} StreamStats;
+#endif
+
typedef struct StreamAgent {
QRegion vis_region; /* the part of the surface area that is currently occupied by video
fragments */
@@ -472,6 +486,9 @@ typedef struct StreamAgent {
uint32_t report_id;
uint32_t client_required_latency;
+#ifdef STREAM_STATS
+ StreamStats stats;
+#endif
} StreamAgent;
typedef struct StreamClipItem {
@@ -2599,9 +2616,40 @@ static void red_attach_stream(RedWorker *worker, Drawable *drawable, Stream *str
region_or(&agent->clip, &drawable->tree_item.base.rgn);
push_stream_clip(dcc, agent);
}
+#ifdef STREAM_STATS
+ agent->stats.num_input_frames++;
+#endif
}
}
+static void red_print_stream_stats(DisplayChannelClient *dcc, StreamAgent *agent)
+{
+#ifdef STREAM_STATS
+ StreamStats *stats = &agent->stats;
+ double passed_mm_time = (stats->end - stats->start) / 1000.0;
+
+ spice_debug("stream %ld (%dx%d): #frames-in %lu, #in-avg-fps %.2f, #frames-sent %lu, "
+ "#drops %lu (pipe %lu, fps %lu), avg_fps %.2f, "
+ "ratio(#frames-out/#frames-in) %.2f, "
+ "passed-mm-time %.2f (sec), size-total %.2f (MB), size-per-sec %.2f (Mbps), "
+ "size-per-frame %.2f (KBpf)",
+ agent - dcc->stream_agents, agent->stream->width, agent->stream->height,
+ stats->num_input_frames,
+ stats->num_input_frames / passed_mm_time,
+ stats->num_frames_sent,
+ stats->num_drops_pipe +
+ stats->num_drops_fps,
+ stats->num_drops_pipe,
+ stats->num_drops_fps,
+ stats->num_frames_sent / passed_mm_time,
+ (stats->num_frames_sent + 0.0) / stats->num_input_frames,
+ passed_mm_time,
+ stats->size_sent / 1024.0 / 1024.0,
+ ((stats->size_sent * 8.0) / (1024.0 * 1024)) / passed_mm_time,
+ stats->size_sent / 1000.0 / stats->num_frames_sent);
+#endif
+}
+
static void red_stop_stream(RedWorker *worker, Stream *stream)
{
DisplayChannelClient *dcc;
@@ -2629,6 +2677,7 @@ static void red_stop_stream(RedWorker *worker, Stream *stream)
}
stream->refs++;
red_channel_client_pipe_add(&dcc->common.base, &stream_agent->destroy_item);
+ red_print_stream_stats(dcc, stream_agent);
}
worker->streams_size_total -= stream->width * stream->height;
ring_remove(&stream->link);
@@ -3027,7 +3076,12 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
report_pipe_item->stream_id = get_stream_id(dcc->common.worker, stream);
red_channel_client_pipe_add(&dcc->common.base, &report_pipe_item->pipe_item);
}
-
+#ifdef STREAM_STATS
+ memset(&agent->stats, 0, sizeof(StreamStats));
+ if (stream->current) {
+ agent->stats.start = stream->current->red_drawable->mm_time;
+ }
+#endif
}
static void red_stream_input_fps_timer_cb(void *opaque)
@@ -3271,6 +3325,9 @@ static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream, Drawa
}
if (pipe_item_is_linked(&dpi->dpi_pipe_item)) {
+#ifdef STREAM_STATS
+ agent->stats.num_drops_pipe++;
+#endif
if (dcc->use_mjpeg_encoder_rate_control) {
mjpeg_encoder_notify_server_frame_drop(agent->mjpeg_encoder);
} else {
@@ -8551,6 +8608,9 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
if (!dcc->use_mjpeg_encoder_rate_control) {
if (time_now - agent->last_send_time < (1000 * 1000 * 1000) / agent->fps) {
agent->frames--;
+#ifdef STREAM_STATS
+ agent->stats.num_drops_fps++;
+#endif
return TRUE;
}
}
@@ -8564,6 +8624,9 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
switch (ret) {
case MJPEG_ENCODER_FRAME_DROP:
spice_assert(dcc->use_mjpeg_encoder_rate_control);
+#ifdef STREAM_STATS
+ agent->stats.num_drops_fps++;
+#endif
return TRUE;
case MJPEG_ENCODER_FRAME_UNSUPPORTED:
return FALSE;
@@ -8610,6 +8673,12 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
spice_marshaller_add_ref(base_marshaller,
dcc->send_data.stream_outbuf, n);
agent->last_send_time = time_now;
+#ifdef STREAM_STATS
+ agent->stats.num_frames_sent++;
+ agent->stats.size_sent += n;
+ agent->stats.end = drawable->red_drawable->mm_time;
+#endif
+
return TRUE;
}
commit 167f999992da730f90079238447deae11629c250
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Tue Feb 19 08:34:18 2013 -0500
server/red_worker: add an option to supply the bandwidth via env var
SPICE_BIT_RATE can be set for supplying red_worker the available
bandwidth (in Mbps).
diff --git a/server/red_worker.c b/server/red_worker.c
index 8e30ef3..21b55c9 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -2879,19 +2879,38 @@ static inline Stream *red_alloc_stream(RedWorker *worker)
static uint64_t red_stream_get_initial_bit_rate(DisplayChannelClient *dcc,
Stream *stream)
{
- uint64_t max_bit_rate;
MainChannelClient *mcc;
+ char *env_bit_rate_str;
+ uint64_t bit_rate = 0;
mcc = red_client_get_main(dcc->common.base.client);
- max_bit_rate = main_channel_client_get_bitrate_per_sec(mcc);
+ env_bit_rate_str = getenv("SPICE_BIT_RATE");
+ if (env_bit_rate_str != NULL) {
+ double env_bit_rate;
+
+ errno = 0;
+ env_bit_rate = strtod(env_bit_rate_str, NULL);
+ if (errno == 0) {
+ bit_rate = env_bit_rate * 1024 * 1024;
+ } else {
+ spice_warning("error parsing SPICE_BIT_RATE: %s", strerror(errno));
+ }
+ }
- if (max_bit_rate > dcc->streams_max_bit_rate) {
- dcc->streams_max_bit_rate = max_bit_rate;
+ if (!bit_rate) {
+ bit_rate = main_channel_client_get_bitrate_per_sec(mcc);
+
+ if (bit_rate > dcc->streams_max_bit_rate) {
+ dcc->streams_max_bit_rate = bit_rate;
+ } else {
+ bit_rate = dcc->streams_max_bit_rate;
+ }
}
+ spice_debug("base-bit-rate %.2f (Mbps)", bit_rate / 1024.0 /1024.0);
/* dividing the available bandwidth among the active streams, and saving
* (1-RED_STREAM_CHANNEL_CAPACITY) of it for other messages */
- return (RED_STREAM_CHANNEL_CAPACITY * dcc->streams_max_bit_rate *
+ return (RED_STREAM_CHANNEL_CAPACITY * bit_rate *
stream->width * stream->height) / dcc->common.worker->streams_size_total;
}
commit 4c79f325e2b2f1355850a70cee68d613346a2cbf
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Feb 18 18:55:03 2013 -0500
server/red_worker.c: use the bit rate of old streams as a start point for new streams
mjpeg_encoder modify the initial bit we supply it, according to the
client feedback. If it reaches a bit rate which is higher than the
initial one, we use the higher bit rate as the new bit rate estimation.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index 7328ea2..b1010e0 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -1247,3 +1247,8 @@ static void mjpeg_encoder_process_server_drops(MJpegEncoder *encoder)
server_state->num_frames_encoded = 0;
server_state->num_frames_dropped = 0;
}
+
+uint64_t mjpeg_encoder_get_bit_rate(MJpegEncoder *encoder)
+{
+ return encoder->rate_control.byte_rate * 8;
+}
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index 0ee2e96..310d289 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -102,4 +102,6 @@ void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
*/
void mjpeg_encoder_notify_server_frame_drop(MJpegEncoder *encoder);
+uint64_t mjpeg_encoder_get_bit_rate(MJpegEncoder *encoder);
+
#endif
diff --git a/server/red_worker.c b/server/red_worker.c
index f96ff07..8e30ef3 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -693,6 +693,7 @@ struct DisplayChannelClient {
StreamAgent stream_agents[NUM_STREAMS];
int use_mjpeg_encoder_rate_control;
uint32_t streams_max_latency;
+ uint64_t streams_max_bit_rate;
};
struct DisplayChannel {
@@ -2616,6 +2617,16 @@ static void red_stop_stream(RedWorker *worker, Stream *stream)
region_clear(&stream_agent->vis_region);
region_clear(&stream_agent->clip);
spice_assert(!pipe_item_is_linked(&stream_agent->destroy_item));
+ if (stream_agent->mjpeg_encoder && dcc->use_mjpeg_encoder_rate_control) {
+ uint64_t stream_bit_rate = mjpeg_encoder_get_bit_rate(stream_agent->mjpeg_encoder);
+
+ if (stream_bit_rate > dcc->streams_max_bit_rate) {
+ spice_debug("old max-bit-rate=%.2f new=%.2f",
+ dcc->streams_max_bit_rate / 8.0 / 1024.0 / 1024.0,
+ stream_bit_rate / 8.0 / 1024.0 / 1024.0);
+ dcc->streams_max_bit_rate = stream_bit_rate;
+ }
+ }
stream->refs++;
red_channel_client_pipe_add(&dcc->common.base, &stream_agent->destroy_item);
}
@@ -2874,10 +2885,13 @@ static uint64_t red_stream_get_initial_bit_rate(DisplayChannelClient *dcc,
mcc = red_client_get_main(dcc->common.base.client);
max_bit_rate = main_channel_client_get_bitrate_per_sec(mcc);
+ if (max_bit_rate > dcc->streams_max_bit_rate) {
+ dcc->streams_max_bit_rate = max_bit_rate;
+ }
/* dividing the available bandwidth among the active streams, and saving
* (1-RED_STREAM_CHANNEL_CAPACITY) of it for other messages */
- return (RED_STREAM_CHANNEL_CAPACITY * max_bit_rate *
+ return (RED_STREAM_CHANNEL_CAPACITY * dcc->streams_max_bit_rate *
stream->width * stream->height) / dcc->common.worker->streams_size_total;
}
commit bf9e210b21a66210b19f69fcaa4542b393b7dc22
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Fri Jan 25 09:34:19 2013 -0500
red_worker: video streams - adjust client playback latency
diff --git a/server/red_worker.c b/server/red_worker.c
index 0249bcb..f96ff07 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -81,6 +81,7 @@
#include "main_channel.h"
#include "migration_protocol.h"
#include "spice_timer_queue.h"
+#include "main_dispatcher.h"
//#define COMPRESS_STAT
//#define DUMP_BITMAP
@@ -470,6 +471,7 @@ typedef struct StreamAgent {
int fps;
uint32_t report_id;
+ uint32_t client_required_latency;
} StreamAgent;
typedef struct StreamClipItem {
@@ -690,6 +692,7 @@ struct DisplayChannelClient {
StreamAgent stream_agents[NUM_STREAMS];
int use_mjpeg_encoder_rate_control;
+ uint32_t streams_max_latency;
};
struct DisplayChannel {
@@ -2901,6 +2904,55 @@ static uint32_t red_stream_mjpeg_encoder_get_source_fps(void *opaque)
return agent->stream->input_fps;
}
+static void red_display_update_streams_max_latency(DisplayChannelClient *dcc, StreamAgent *remove_agent)
+{
+ uint32_t new_max_latency = 0;
+ int i;
+
+ if (dcc->streams_max_latency != remove_agent->client_required_latency) {
+ return;
+ }
+
+ dcc->streams_max_latency = 0;
+ if (dcc->common.worker->stream_count == 1) {
+ return;
+ }
+ for (i = 0; i < NUM_STREAMS; i++) {
+ StreamAgent *other_agent = &dcc->stream_agents[i];
+ if (other_agent == remove_agent || !other_agent->mjpeg_encoder) {
+ continue;
+ }
+ if (other_agent->client_required_latency > new_max_latency) {
+ new_max_latency = other_agent->client_required_latency;
+ }
+ }
+ dcc->streams_max_latency = new_max_latency;
+}
+
+static void red_display_stream_agent_stop(DisplayChannelClient *dcc, StreamAgent *agent)
+{
+ red_display_update_streams_max_latency(dcc, agent);
+ if (agent->mjpeg_encoder) {
+ mjpeg_encoder_destroy(agent->mjpeg_encoder);
+ agent->mjpeg_encoder = NULL;
+ }
+}
+
+static void red_stream_update_client_playback_latency(void *opaque, uint32_t delay_ms)
+{
+ StreamAgent *agent = opaque;
+ DisplayChannelClient *dcc = agent->dcc;
+
+ red_display_update_streams_max_latency(dcc, agent);
+
+ agent->client_required_latency = delay_ms;
+ if (delay_ms > agent->dcc->streams_max_latency) {
+ agent->dcc->streams_max_latency = delay_ms;
+ }
+ spice_debug("reseting client latency: %u", agent->dcc->streams_max_latency);
+ main_dispatcher_set_mm_time_latency(agent->dcc->common.base.client, agent->dcc->streams_max_latency);
+}
+
static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
{
StreamAgent *agent = &dcc->stream_agents[get_stream_id(dcc->common.worker, stream)];
@@ -2924,6 +2976,7 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
mjpeg_cbs.get_roundtrip_ms = red_stream_mjpeg_encoder_get_roundtrip;
mjpeg_cbs.get_source_fps = red_stream_mjpeg_encoder_get_source_fps;
+ mjpeg_cbs.update_client_playback_delay = red_stream_update_client_playback_latency;
initial_bit_rate = red_stream_get_initial_bit_rate(dcc, stream);
agent->mjpeg_encoder = mjpeg_encoder_new(TRUE, initial_bit_rate, &mjpeg_cbs, agent);
@@ -8895,11 +8948,7 @@ static void red_display_marshall_stream_end(RedChannelClient *rcc,
red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DESTROY, NULL);
destroy.id = get_stream_id(dcc->common.worker, agent->stream);
-
- if (agent->mjpeg_encoder) {
- mjpeg_encoder_destroy(agent->mjpeg_encoder);
- agent->mjpeg_encoder = NULL;
- }
+ red_display_stream_agent_stop(dcc, agent);
spice_marshall_msg_display_stream_destroy(base_marshaller, &destroy);
}
commit 073aeec569e2bee3feec8042986b8434db8afad3
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Fri Jan 25 09:27:33 2013 -0500
reds: support mm_time latency adjustments
When there is no audio playback, we set the mm_time in the client to be older
than the one in the server by at least the requested latency (the delta is
actually bigger, due to the network latency).
When there is an audio playback, we adjust the mm_time in the client by
adjusting the playback buffer using SPICE_MSG_PLAYBACK_LATENCY.
diff --git a/server/main_dispatcher.c b/server/main_dispatcher.c
index 1126ec0..8402402 100644
--- a/server/main_dispatcher.c
+++ b/server/main_dispatcher.c
@@ -40,6 +40,7 @@ MainDispatcher main_dispatcher;
enum {
MAIN_DISPATCHER_CHANNEL_EVENT = 0,
MAIN_DISPATCHER_MIGRATE_SEAMLESS_DST_COMPLETE,
+ MAIN_DISPATCHER_SET_MM_TIME_LATENCY,
MAIN_DISPATCHER_NUM_MESSAGES
};
@@ -53,6 +54,11 @@ typedef struct MainDispatcherMigrateSeamlessDstCompleteMessage {
RedClient *client;
} MainDispatcherMigrateSeamlessDstCompleteMessage;
+typedef struct MainDispatcherMmTimeLatencyMessage {
+ RedClient *client;
+ uint32_t latency;
+} MainDispatcherMmTimeLatencyMessage;
+
/* channel_event - calls core->channel_event, must be done in main thread */
static void main_dispatcher_self_handle_channel_event(
int event,
@@ -96,6 +102,13 @@ static void main_dispatcher_handle_migrate_complete(void *opaque,
reds_on_client_seamless_migrate_complete(mig_complete->client);
}
+static void main_dispatcher_handle_mm_time_latency(void *opaque,
+ void *payload)
+{
+ MainDispatcherMmTimeLatencyMessage *msg = payload;
+ reds_set_client_mm_time_latency(msg->client, msg->latency);
+}
+
void main_dispatcher_seamless_migrate_dst_complete(RedClient *client)
{
MainDispatcherMigrateSeamlessDstCompleteMessage msg;
@@ -109,6 +122,22 @@ void main_dispatcher_seamless_migrate_dst_complete(RedClient *client)
dispatcher_send_message(&main_dispatcher.base, MAIN_DISPATCHER_MIGRATE_SEAMLESS_DST_COMPLETE,
&msg);
}
+
+void main_dispatcher_set_mm_time_latency(RedClient *client, uint32_t latency)
+{
+ MainDispatcherMmTimeLatencyMessage msg;
+
+ if (pthread_self() == main_dispatcher.base.self) {
+ reds_set_client_mm_time_latency(client, latency);
+ return;
+ }
+
+ msg.client = client;
+ msg.latency = latency;
+ dispatcher_send_message(&main_dispatcher.base, MAIN_DISPATCHER_SET_MM_TIME_LATENCY,
+ &msg);
+}
+
static void dispatcher_handle_read(int fd, int event, void *opaque)
{
Dispatcher *dispatcher = opaque;
@@ -129,4 +158,7 @@ void main_dispatcher_init(SpiceCoreInterface *core)
dispatcher_register_handler(&main_dispatcher.base, MAIN_DISPATCHER_MIGRATE_SEAMLESS_DST_COMPLETE,
main_dispatcher_handle_migrate_complete,
sizeof(MainDispatcherMigrateSeamlessDstCompleteMessage), 0 /* no ack */);
+ dispatcher_register_handler(&main_dispatcher.base, MAIN_DISPATCHER_SET_MM_TIME_LATENCY,
+ main_dispatcher_handle_mm_time_latency,
+ sizeof(MainDispatcherMmTimeLatencyMessage), 0 /* no ack */);
}
diff --git a/server/main_dispatcher.h b/server/main_dispatcher.h
index d44ee3a..0c79ca8 100644
--- a/server/main_dispatcher.h
+++ b/server/main_dispatcher.h
@@ -6,6 +6,7 @@
void main_dispatcher_channel_event(int event, SpiceChannelEventInfo *info);
void main_dispatcher_seamless_migrate_dst_complete(RedClient *client);
+void main_dispatcher_set_mm_time_latency(RedClient *client, uint32_t latency);
void main_dispatcher_init(SpiceCoreInterface *core);
#endif //MAIN_DISPATCHER_H
diff --git a/server/reds-private.h b/server/reds-private.h
index 3db6565..9358d27 100644
--- a/server/reds-private.h
+++ b/server/reds-private.h
@@ -177,6 +177,8 @@ typedef struct RedsState {
int allow_multiple_clients;
RedsClientMonitorsConfig client_monitors_config;
+ int mm_timer_enabled;
+ uint32_t mm_time_latency;
} RedsState;
#endif
diff --git a/server/reds.c b/server/reds.c
index 822289b..c3b5518 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -3031,6 +3031,29 @@ listen:
return slisten;
}
+static void reds_send_mm_time(void)
+{
+ main_channel_push_multi_media_time(reds->main_channel,
+ reds_get_mm_time() - reds->mm_time_latency);
+}
+
+void reds_set_client_mm_time_latency(RedClient *client, uint32_t latency)
+{
+ // TODO: multi-client support for mm_time
+ if (reds->mm_timer_enabled) {
+ // TODO: consider network latency
+ if (latency > reds->mm_time_latency) {
+ reds->mm_time_latency = latency;
+ reds_send_mm_time();
+ } else {
+ spice_debug("new latency %u is smaller than existing %u",
+ latency, reds->mm_time_latency);
+ }
+ } else {
+ snd_set_playback_latency(client, latency);
+ }
+}
+
static int reds_init_net(void)
{
if (spice_port != -1) {
@@ -3463,12 +3486,15 @@ void reds_enable_mm_timer(void)
if (!reds_main_channel_connected()) {
return;
}
- main_channel_push_multi_media_time(reds->main_channel, reds_get_mm_time() - MM_TIME_DELTA);
+ reds->mm_timer_enabled = TRUE;
+ reds->mm_time_latency = MM_TIME_DELTA;
+ reds_send_mm_time();
}
void reds_disable_mm_timer(void)
{
core->timer_cancel(reds->mm_timer);
+ reds->mm_timer_enabled = FALSE;
}
static void mm_timer_proc(void *opaque)
diff --git a/server/reds.h b/server/reds.h
index f8e8d56..59f13ce 100644
--- a/server/reds.h
+++ b/server/reds.h
@@ -164,4 +164,6 @@ void reds_on_client_seamless_migrate_complete(RedClient *client);
void reds_on_main_channel_migrate(MainChannelClient *mcc);
void reds_on_char_device_state_destroy(SpiceCharDeviceState *dev);
+void reds_set_client_mm_time_latency(RedClient *client, uint32_t latency);
+
#endif
commit a9087c4c8a1cf38c9dae331a9255f0dc6823e7b8
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Fri Jan 25 09:21:22 2013 -0500
snd_worker: support sending SPICE_MSG_PLAYBACK_LATENCY
also update spice-common submodule
diff --git a/server/snd_worker.c b/server/snd_worker.c
index 2647d87..d6ec47a 100644
--- a/server/snd_worker.c
+++ b/server/snd_worker.c
@@ -58,6 +58,7 @@ enum PlaybackeCommand {
SND_PLAYBACK_CTRL,
SND_PLAYBACK_PCM,
SND_PLAYBACK_VOLUME,
+ SND_PLAYBACK_LATENCY,
};
enum RecordCommand {
@@ -71,6 +72,7 @@ enum RecordCommand {
#define SND_PLAYBACK_CTRL_MASK (1 << SND_PLAYBACK_CTRL)
#define SND_PLAYBACK_PCM_MASK (1 << SND_PLAYBACK_PCM)
#define SND_PLAYBACK_VOLUME_MASK (1 << SND_PLAYBACK_VOLUME)
+#define SND_PLAYBACK_LATENCY_MASK ( 1 << SND_PLAYBACK_LATENCY)
#define SND_RECORD_MIGRATE_MASK (1 << SND_RECORD_MIGRATE)
#define SND_RECORD_CTRL_MASK (1 << SND_RECORD_CTRL)
@@ -144,6 +146,7 @@ struct PlaybackChannel {
struct {
uint8_t celt_buf[CELT_COMPRESSED_FRAME_BYTES];
} send_data;
+ uint32_t latency;
};
struct SndWorker {
@@ -610,6 +613,20 @@ static int snd_playback_send_mute(PlaybackChannel *playback_channel)
return snd_send_mute(channel, &st->volume, SPICE_MSG_PLAYBACK_MUTE);
}
+static int snd_playback_send_latency(PlaybackChannel *playback_channel)
+{
+ SndChannel *channel = &playback_channel->base;
+ SpiceMsgPlaybackLatency latency_msg;
+
+ spice_debug("latency %u", playback_channel->latency);
+ if (!snd_reset_send_data(channel, SPICE_MSG_PLAYBACK_LATENCY)) {
+ return FALSE;
+ }
+ latency_msg.latency_ms = playback_channel->latency;
+ spice_marshall_msg_playback_latency(channel->send_data.marshaller, &latency_msg);
+
+ return snd_begin_send_message(channel);
+}
static int snd_playback_send_start(PlaybackChannel *playback_channel)
{
SndChannel *channel = (SndChannel *)playback_channel;
@@ -819,6 +836,12 @@ static void snd_playback_send(void* data)
}
channel->command &= ~SND_PLAYBACK_MIGRATE_MASK;
}
+ if (channel->command & SND_PLAYBACK_LATENCY_MASK) {
+ if (!snd_playback_send_latency(playback_channel)) {
+ return;
+ }
+ channel->command &= ~SND_PLAYBACK_LATENCY_MASK;
+ }
}
}
@@ -1096,6 +1119,27 @@ SPICE_GNUC_VISIBLE void spice_server_playback_put_samples(SpicePlaybackInstance
snd_playback_send(&playback_channel->base);
}
+void snd_set_playback_latency(RedClient *client, uint32_t latency)
+{
+ SndWorker *now = workers;
+
+ for (; now; now = now->next) {
+ if (now->base_channel->type == SPICE_CHANNEL_PLAYBACK && now->connection &&
+ now->connection->channel_client->client == client) {
+
+ if (red_channel_client_test_remote_cap(now->connection->channel_client,
+ SPICE_PLAYBACK_CAP_LATENCY)) {
+ PlaybackChannel* playback = (PlaybackChannel*)now->connection;
+
+ playback->latency = latency;
+ snd_set_command(now->connection, SND_PLAYBACK_LATENCY_MASK);
+ snd_playback_send(now->connection);
+ } else {
+ spice_debug("client doesn't not support SPICE_PLAYBACK_CAP_LATENCY");
+ }
+ }
+ }
+}
static void on_new_playback_channel(SndWorker *worker)
{
PlaybackChannel *playback_channel =
diff --git a/server/snd_worker.h b/server/snd_worker.h
index 1811a61..8de746d 100644
--- a/server/snd_worker.h
+++ b/server/snd_worker.h
@@ -29,4 +29,6 @@ void snd_detach_record(SpiceRecordInstance *sin);
void snd_set_playback_compression(int on);
int snd_get_playback_compression(void);
+void snd_set_playback_latency(RedClient *client, uint32_t latency);
+
#endif
diff --git a/spice-common b/spice-common
index 7cdf8de..30e8478 160000
--- a/spice-common
+++ b/spice-common
@@ -1 +1 @@
-Subproject commit 7cdf8de00a573b6bdb4ec4582c87aa79b25796d3
+Subproject commit 30e84783cad7a5d4cd345367a267cafbfd0571af
commit ba1aaef0fea69dc46297a5f85d50020fb728355b
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Fri Jan 25 09:18:00 2013 -0500
dispatcher.h: fix - s/#define MAIN_DISPATCHER_H/#define DISPATCHER_H
diff --git a/server/dispatcher.h b/server/dispatcher.h
index a468c58..1b389bd 100644
--- a/server/dispatcher.h
+++ b/server/dispatcher.h
@@ -1,5 +1,5 @@
-#ifndef MAIN_DISPATCHER_H
-#define MAIN_DISPATCHER_H
+#ifndef DISPATCHER_H
+#define DISPATCHER_H
#include <spice.h>
@@ -103,4 +103,4 @@ int dispatcher_get_recv_fd(Dispatcher *);
*/
void dispatcher_set_opaque(Dispatcher *dispatcher, void *opaque);
-#endif //MAIN_DISPATCHER_H
+#endif //DISPATCHER_H
diff --git a/server/main_dispatcher.h b/server/main_dispatcher.h
index ec4a6b4..d44ee3a 100644
--- a/server/main_dispatcher.h
+++ b/server/main_dispatcher.h
@@ -2,6 +2,7 @@
#define MAIN_DISPATCHER_H
#include <spice.h>
+#include "red_channel.h"
void main_dispatcher_channel_event(int event, SpiceChannelEventInfo *info);
void main_dispatcher_seamless_migrate_dst_complete(RedClient *client);
commit 23730a6c800fa00c6975b4c36a46b2b427aac35c
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Thu Jan 24 17:21:58 2013 -0500
red_worker: ignoring video frame drops that are not due to pipe congestion
A frame can be dropped if a new frame was added during the same
call to red_process_command (we didn't attempt to send the older
frame). Such drops are ignored.
diff --git a/server/red_worker.c b/server/red_worker.c
index 2fe9a3a..0249bcb 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -846,6 +846,8 @@ struct Drawable {
int surface_id;
int surfaces_dest[3];
+
+ uint32_t process_commands_generation;
};
typedef struct _Drawable _Drawable;
@@ -997,6 +999,7 @@ typedef struct RedWorker {
ZlibData zlib_data;
ZlibEncoder *zlib;
+ uint32_t process_commands_generation;
#ifdef PIPE_DEBUG
uint32_t last_id;
#endif
@@ -3152,7 +3155,7 @@ static int display_channel_client_is_low_bandwidth(DisplayChannelClient *dcc)
red_client_get_main(red_channel_client_get_client(&dcc->common.base)));
}
-static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
+static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream, Drawable *new_frame)
{
DrawablePipeItem *dpi;
DisplayChannelClient *dcc;
@@ -3166,6 +3169,11 @@ static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
return;
}
+ if (new_frame->process_commands_generation == stream->current->process_commands_generation) {
+ spice_debug("ignoring drop, same process_commands_generation as previous frame");
+ return;
+ }
+
index = get_stream_id(worker, stream);
DRAWABLE_FOREACH_DPI(stream->current, ring_item, dpi) {
dcc = dpi->dcc;
@@ -3299,7 +3307,7 @@ static inline void red_stream_maintenance(RedWorker *worker, Drawable *candidate
stream,
TRUE);
if (is_next_frame != STREAM_FRAME_NONE) {
- pre_stream_item_swap(worker, stream);
+ pre_stream_item_swap(worker, stream, candidate);
red_detach_stream(worker, stream, FALSE);
prev->streamable = FALSE; //prevent item trace
red_attach_stream(worker, candidate, stream);
@@ -3452,7 +3460,7 @@ static inline void red_use_stream_trace(RedWorker *worker, Drawable *drawable)
if (is_next_frame != STREAM_FRAME_NONE) {
if (stream->current) {
stream->current->streamable = FALSE; //prevent item trace
- pre_stream_item_swap(worker, stream);
+ pre_stream_item_swap(worker, stream, drawable);
red_detach_stream(worker, stream, FALSE);
}
red_attach_stream(worker, drawable, stream);
@@ -3965,6 +3973,7 @@ static Drawable *get_drawable(RedWorker *worker, uint8_t effect, RedDrawable *re
ring_init(&drawable->pipes);
ring_init(&drawable->glz_ring);
+ drawable->process_commands_generation = worker->process_commands_generation;
return drawable;
}
@@ -5013,6 +5022,7 @@ static int red_process_commands(RedWorker *worker, uint32_t max_pipe_size, int *
return n;
}
+ worker->process_commands_generation++;
*ring_is_empty = FALSE;
while (!display_is_connected(worker) ||
// TODO: change to average pipe size?
commit ed1654eb11fbed9db48ea04fe19b08cad61e4087
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Thu Jan 24 17:09:15 2013 -0500
red_worker: notify mjpeg_encoder on server frame drops
diff --git a/server/red_worker.c b/server/red_worker.c
index 27e5380..2fe9a3a 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -3169,13 +3169,19 @@ static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
index = get_stream_id(worker, stream);
DRAWABLE_FOREACH_DPI(stream->current, ring_item, dpi) {
dcc = dpi->dcc;
- if (!display_channel_client_is_low_bandwidth(dcc)) {
+ agent = &dcc->stream_agents[index];
+
+ if (!dcc->use_mjpeg_encoder_rate_control &&
+ !display_channel_client_is_low_bandwidth(dcc)) {
continue;
}
- agent = &dcc->stream_agents[index];
if (pipe_item_is_linked(&dpi->dpi_pipe_item)) {
- ++agent->drops;
+ if (dcc->use_mjpeg_encoder_rate_control) {
+ mjpeg_encoder_notify_server_frame_drop(agent->mjpeg_encoder);
+ } else {
+ ++agent->drops;
+ }
}
}
@@ -3184,11 +3190,14 @@ static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
double drop_factor;
agent = &dcc->stream_agents[index];
+
+ if (dcc->use_mjpeg_encoder_rate_control) {
+ continue;
+ }
if (agent->frames / agent->fps < FPS_TEST_INTERVAL) {
agent->frames++;
continue;
}
-
drop_factor = ((double)agent->frames - (double)agent->drops) /
(double)agent->frames;
spice_debug("stream %d: #frames %u #drops %u", index, agent->frames, agent->drops);
commit 0df94503996664212e927e2c5dea7c33f65eae12
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Thu Jan 24 16:57:15 2013 -0500
red_worker: support SPICE_MSGC_DISPLAY_STREAM_REPORT
update mjpeg_encoder with reports from the client about
the playback quality.
The patch also updates the spice-common submodule.
diff --git a/server/red_dispatcher.c b/server/red_dispatcher.c
index 97e9737..e054b5a 100644
--- a/server/red_dispatcher.c
+++ b/server/red_dispatcher.c
@@ -1143,6 +1143,7 @@ RedDispatcher *red_dispatcher_init(QXLInstance *qxl)
red_channel_register_client_cbs(display_channel, &client_cbs);
red_channel_set_data(display_channel, red_dispatcher);
red_channel_set_cap(display_channel, SPICE_DISPLAY_CAP_MONITORS_CONFIG);
+ red_channel_set_cap(display_channel, SPICE_DISPLAY_CAP_STREAM_REPORT);
reds_register_channel(display_channel);
}
diff --git a/server/red_worker.c b/server/red_worker.c
index 2a1030e..27e5380 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -117,6 +117,9 @@
#define RED_STREAM_MIN_SIZE (96 * 96)
#define RED_STREAM_INPUT_FPS_TIMEOUT (5 * 1000) // 5 sec
#define RED_STREAM_CHANNEL_CAPACITY 0.8
+/* the client's stream report frequency is the minimum of the 2 values below */
+#define RED_STREAM_CLIENT_REPORT_WINDOW 5 // #frames
+#define RED_STREAM_CLIENT_REPORT_TIMEOUT 1000 // milliseconds
#define FPS_TEST_INTERVAL 1
#define MAX_FPS 30
@@ -292,6 +295,7 @@ enum {
PIPE_ITEM_TYPE_CREATE_SURFACE,
PIPE_ITEM_TYPE_DESTROY_SURFACE,
PIPE_ITEM_TYPE_MONITORS_CONFIG,
+ PIPE_ITEM_TYPE_STREAM_ACTIVATE_REPORT,
};
typedef struct VerbItem {
@@ -351,6 +355,11 @@ typedef struct MonitorsConfigItem {
MonitorsConfig *monitors_config;
} MonitorsConfigItem;
+typedef struct StreamActivateReportItem {
+ PipeItem pipe_item;
+ uint32_t stream_id;
+} StreamActivateReportItem;
+
typedef struct CursorItem {
uint32_t group_id;
int refs;
@@ -459,6 +468,8 @@ typedef struct StreamAgent {
int frames;
int drops;
int fps;
+
+ uint32_t report_id;
} StreamAgent;
typedef struct StreamClipItem {
@@ -2917,6 +2928,17 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
agent->mjpeg_encoder = mjpeg_encoder_new(FALSE, 0, NULL, NULL);
}
red_channel_client_pipe_add(&dcc->common.base, &agent->create_item);
+
+ if (red_channel_client_test_remote_cap(&dcc->common.base, SPICE_DISPLAY_CAP_STREAM_REPORT)) {
+ StreamActivateReportItem *report_pipe_item = spice_malloc0(sizeof(*report_pipe_item));
+
+ agent->report_id = rand();
+ red_channel_pipe_item_init(dcc->common.base.channel, &report_pipe_item->pipe_item,
+ PIPE_ITEM_TYPE_STREAM_ACTIVATE_REPORT);
+ report_pipe_item->stream_id = get_stream_id(dcc->common.worker, stream);
+ red_channel_client_pipe_add(&dcc->common.base, &report_pipe_item->pipe_item);
+ }
+
}
static void red_stream_input_fps_timer_cb(void *opaque)
@@ -3007,7 +3029,8 @@ static void red_display_client_init_streams(DisplayChannelClient *dcc)
red_channel_pipe_item_init(channel, &agent->create_item, PIPE_ITEM_TYPE_STREAM_CREATE);
red_channel_pipe_item_init(channel, &agent->destroy_item, PIPE_ITEM_TYPE_STREAM_DESTROY);
}
- dcc->use_mjpeg_encoder_rate_control = TRUE;
+ dcc->use_mjpeg_encoder_rate_control =
+ red_channel_client_test_remote_cap(&dcc->common.base, SPICE_DISPLAY_CAP_STREAM_REPORT);
}
static void red_display_destroy_streams_agents(DisplayChannelClient *dcc)
@@ -9000,6 +9023,22 @@ static void red_marshall_monitors_config(RedChannelClient *rcc, SpiceMarshaller
free(msg);
}
+static void red_marshall_stream_activate_report(RedChannelClient *rcc,
+ SpiceMarshaller *base_marshaller,
+ uint32_t stream_id)
+{
+ DisplayChannelClient *dcc = RCC_TO_DCC(rcc);
+ StreamAgent *agent = &dcc->stream_agents[stream_id];
+ SpiceMsgDisplayStreamActivateReport msg;
+
+ red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_ACTIVATE_REPORT, NULL);
+ msg.stream_id = stream_id;
+ msg.unique_id = agent->report_id;
+ msg.max_window_size = RED_STREAM_CLIENT_REPORT_WINDOW;
+ msg.timeout_ms = RED_STREAM_CLIENT_REPORT_TIMEOUT;
+ spice_marshall_msg_display_stream_activate_report(base_marshaller, &msg);
+}
+
static void display_channel_send_item(RedChannelClient *rcc, PipeItem *pipe_item)
{
SpiceMarshaller *m = red_channel_client_get_marshaller(rcc);
@@ -9070,6 +9109,13 @@ static void display_channel_send_item(RedChannelClient *rcc, PipeItem *pipe_item
red_marshall_monitors_config(rcc, m, monconf_item->monitors_config);
break;
}
+ case PIPE_ITEM_TYPE_STREAM_ACTIVATE_REPORT: {
+ StreamActivateReportItem *report_item = SPICE_CONTAINEROF(pipe_item,
+ StreamActivateReportItem,
+ pipe_item);
+ red_marshall_stream_activate_report(rcc, m, report_item->stream_id);
+ break;
+ }
default:
spice_error("invalid pipe item type");
}
@@ -10010,6 +10056,37 @@ static int display_channel_handle_migrate_data(RedChannelClient *rcc, uint32_t s
return TRUE;
}
+static int display_channel_handle_stream_report(DisplayChannelClient *dcc,
+ SpiceMsgcDisplayStreamReport *stream_report)
+{
+ StreamAgent *stream_agent;
+
+ if (stream_report->stream_id >= NUM_STREAMS) {
+ spice_warning("stream_report: invalid stream id %u", stream_report->stream_id);
+ return FALSE;
+ }
+ stream_agent = &dcc->stream_agents[stream_report->stream_id];
+ if (!stream_agent->mjpeg_encoder) {
+ spice_info("stream_report: no encoder for stream id %u."
+ "Probably the stream has been destroyed", stream_report->stream_id);
+ return TRUE;
+ }
+
+ if (stream_report->unique_id != stream_agent->report_id) {
+ spice_warning("local reoprt-id (%u) != msg report-id (%u)",
+ stream_agent->report_id, stream_report->unique_id);
+ return TRUE;
+ }
+ mjpeg_encoder_client_stream_report(stream_agent->mjpeg_encoder,
+ stream_report->num_frames,
+ stream_report->num_drops,
+ stream_report->start_frame_mm_time,
+ stream_report->end_frame_mm_time,
+ stream_report->last_frame_delay,
+ stream_report->audio_delay);
+ return TRUE;
+}
+
static int display_channel_handle_message(RedChannelClient *rcc, uint32_t size, uint16_t type,
void *message)
{
@@ -10023,6 +10100,9 @@ static int display_channel_handle_message(RedChannelClient *rcc, uint32_t size,
}
dcc->expect_init = FALSE;
return display_channel_init(dcc, (SpiceMsgcDisplayInit *)message);
+ case SPICE_MSGC_DISPLAY_STREAM_REPORT:
+ return display_channel_handle_stream_report(dcc,
+ (SpiceMsgcDisplayStreamReport *)message);
default:
return red_channel_client_handle_message(rcc, size, type, message);
}
@@ -10372,6 +10452,7 @@ static void display_channel_client_release_item_before_push(DisplayChannelClient
case PIPE_ITEM_TYPE_PIXMAP_SYNC:
case PIPE_ITEM_TYPE_PIXMAP_RESET:
case PIPE_ITEM_TYPE_INVAL_PALLET_CACHE:
+ case PIPE_ITEM_TYPE_STREAM_ACTIVATE_REPORT:
free(item);
break;
default:
@@ -10541,7 +10622,7 @@ static void handle_new_display_channel(RedWorker *worker, RedClient *client, Red
spice_info("zlib-over-glz %s", display_channel->enable_zlib_glz_wrap ? "enabled" : "disabled");
guest_set_client_capabilities(worker);
-
+
// todo: tune level according to bandwidth
display_channel->zlib_level = ZLIB_DEFAULT_COMPRESSION_LEVEL;
red_display_client_init_streams(dcc);
@@ -11909,6 +11990,7 @@ static void red_init(RedWorker *worker, WorkerInitData *init_data)
if (!spice_timer_queue_create()) {
spice_error("failed to create timer queue");
}
+ srand(time(NULL));
message = RED_WORKER_MESSAGE_READY;
write_message(worker->channel, &message);
diff --git a/spice-common b/spice-common
index 5ebeee5..7cdf8de 160000
--- a/spice-common
+++ b/spice-common
@@ -1 +1 @@
-Subproject commit 5ebeee51146f9441474377e77bbc15ec69530d54
+Subproject commit 7cdf8de00a573b6bdb4ec4582c87aa79b25796d3
commit ade45ed93abbb9b88d9e0a6e0e12073ea5bcfc36
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Feb 18 14:05:43 2013 -0500
red_worker: start using mjpeg_encoder rate control capabilities
This patch only employs setting the stream parameters based on
the initial given bit-rate, the latency, and the encoding size.
Later patches will also employ mjpeg_encoder response to client reports,
and its control over frame drops.
The patch also removes old stream bit rate calculations that weren't
used.
diff --git a/server/main_channel.c b/server/main_channel.c
index a065ffb..dd927ab 100644
--- a/server/main_channel.c
+++ b/server/main_channel.c
@@ -1150,6 +1150,11 @@ uint64_t main_channel_client_get_bitrate_per_sec(MainChannelClient *mcc)
return mcc->bitrate_per_sec;
}
+uint64_t main_channel_client_get_roundtrip_ms(MainChannelClient *mcc)
+{
+ return mcc->latency / 1000;
+}
+
static void main_channel_client_migrate(RedChannelClient *rcc)
{
reds_on_main_channel_migrate(SPICE_CONTAINEROF(rcc, MainChannelClient, base));
diff --git a/server/main_channel.h b/server/main_channel.h
index 56663b7..b2f0e6f 100644
--- a/server/main_channel.h
+++ b/server/main_channel.h
@@ -69,6 +69,7 @@ uint32_t main_channel_client_get_link_id(MainChannelClient *mcc);
int main_channel_client_is_low_bandwidth(MainChannelClient *mcc);
uint64_t main_channel_client_get_bitrate_per_sec(MainChannelClient *mcc);
+uint64_t main_channel_client_get_roundtrip_ms(MainChannelClient *mcc);
int main_channel_is_connected(MainChannel *main_chan);
RedChannelClient* main_channel_client_get_base(MainChannelClient* mcc);
diff --git a/server/red_worker.c b/server/red_worker.c
index 2f40635..2a1030e 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -116,14 +116,11 @@
#define RED_STREAM_FRAMES_RESET_CONDITION 100
#define RED_STREAM_MIN_SIZE (96 * 96)
#define RED_STREAM_INPUT_FPS_TIMEOUT (5 * 1000) // 5 sec
+#define RED_STREAM_CHANNEL_CAPACITY 0.8
#define FPS_TEST_INTERVAL 1
#define MAX_FPS 30
-//best bit rate per pixel base on 13000000 bps for frame size 720x576 pixels and 25 fps
-#define BEST_BIT_RATE_PER_PIXEL 38
-#define WORST_BIT_RATE_PER_PIXEL 4
-
#define RED_COMPRESS_BUF_SIZE (1024 * 64)
#define ZLIB_DEFAULT_COMPRESSION_LEVEL 3
@@ -415,6 +412,9 @@ typedef struct ImageItem {
typedef struct Drawable Drawable;
+typedef struct DisplayChannel DisplayChannel;
+typedef struct DisplayChannelClient DisplayChannelClient;
+
enum {
STREAM_FRAME_NONE,
STREAM_FRAME_NATIVE,
@@ -432,7 +432,6 @@ struct Stream {
int top_down;
Stream *next;
RingItem link;
- int bit_rate;
SpiceTimer *input_fps_timer;
uint32_t num_input_frames;
@@ -455,6 +454,7 @@ typedef struct StreamAgent {
Stream *stream;
uint64_t last_send_time;
MJpegEncoder *mjpeg_encoder;
+ DisplayChannelClient *dcc;
int frames;
int drops;
@@ -526,9 +526,6 @@ typedef struct FreeList {
WaitForChannels wait;
} FreeList;
-typedef struct DisplayChannel DisplayChannel;
-typedef struct DisplayChannelClient DisplayChannelClient;
-
typedef struct {
DisplayChannelClient *dcc;
RedCompressBuf *bufs_head;
@@ -681,6 +678,7 @@ struct DisplayChannelClient {
QRegion surface_client_lossy_region[NUM_SURFACES];
StreamAgent stream_agents[NUM_STREAMS];
+ int use_mjpeg_encoder_rate_control;
};
struct DisplayChannel {
@@ -974,6 +972,7 @@ typedef struct RedWorker {
Ring streams;
ItemTrace items_trace[NUM_TRACE_ITEMS];
uint32_t next_item_trace;
+ uint64_t streams_size_total;
QuicData quic_data;
QuicContext *quic;
@@ -1049,7 +1048,6 @@ static int red_display_free_some_independent_glz_drawables(DisplayChannelClient
static void red_display_free_glz_drawable(DisplayChannelClient *dcc, RedGlzDrawable *drawable);
static ImageItem *red_add_surface_area_image(DisplayChannelClient *dcc, int surface_id,
SpiceRect *area, PipeItem *pos, int can_lossy);
-static void reset_rate(DisplayChannelClient *dcc, StreamAgent *stream_agent);
static BitmapGradualType _get_bitmap_graduality_level(RedWorker *worker, SpiceBitmap *bitmap,
uint32_t group_id);
static inline int _stride_is_extra(SpiceBitmap *bitmap);
@@ -2604,6 +2602,7 @@ static void red_stop_stream(RedWorker *worker, Stream *stream)
stream->refs++;
red_channel_client_pipe_add(&dcc->common.base, &stream_agent->destroy_item);
}
+ worker->streams_size_total -= stream->width * stream->height;
ring_remove(&stream->link);
red_release_stream(worker, stream);
}
@@ -2849,38 +2848,43 @@ static inline Stream *red_alloc_stream(RedWorker *worker)
return stream;
}
-static int get_bit_rate(DisplayChannelClient *dcc,
- int width, int height)
+static uint64_t red_stream_get_initial_bit_rate(DisplayChannelClient *dcc,
+ Stream *stream)
{
- uint64_t bit_rate = width * height * BEST_BIT_RATE_PER_PIXEL;
+ uint64_t max_bit_rate;
MainChannelClient *mcc;
- int is_low_bandwidth = 0;
- if (dcc) {
- mcc = red_client_get_main(dcc->common.base.client);
- is_low_bandwidth = main_channel_client_is_low_bandwidth(mcc);
- }
+ mcc = red_client_get_main(dcc->common.base.client);
+ max_bit_rate = main_channel_client_get_bitrate_per_sec(mcc);
- if (is_low_bandwidth) {
- bit_rate = MIN(main_channel_client_get_bitrate_per_sec(mcc) * 70 / 100, bit_rate);
- bit_rate = MAX(bit_rate, width * height * WORST_BIT_RATE_PER_PIXEL);
- }
- return bit_rate;
+
+ /* dividing the available bandwidth among the active streams, and saving
+ * (1-RED_STREAM_CHANNEL_CAPACITY) of it for other messages */
+ return (RED_STREAM_CHANNEL_CAPACITY * max_bit_rate *
+ stream->width * stream->height) / dcc->common.worker->streams_size_total;
}
-static int get_minimal_bit_rate(RedWorker *worker, int width, int height)
+static uint32_t red_stream_mjpeg_encoder_get_roundtrip(void *opaque)
{
- RingItem *item;
- DisplayChannelClient *dcc;
- int ret = INT_MAX;
+ StreamAgent *agent = opaque;
+ int roundtrip;
- WORKER_FOREACH_DCC(worker, item, dcc) {
- int bit_rate = get_bit_rate(dcc, width, height);
- if (bit_rate < ret) {
- ret = bit_rate;
- }
+ spice_assert(agent);
+ roundtrip = red_channel_client_get_roundtrip_ms(&agent->dcc->common.base);
+ if (roundtrip < 0) {
+ MainChannelClient *mcc = red_client_get_main(agent->dcc->common.base.client);
+ roundtrip = main_channel_client_get_roundtrip_ms(mcc);
}
- return ret;
+
+ return roundtrip;
+}
+
+static uint32_t red_stream_mjpeg_encoder_get_source_fps(void *opaque)
+{
+ StreamAgent *agent = opaque;
+
+ spice_assert(agent);
+ return agent->stream->input_fps;
}
static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
@@ -2898,8 +2902,20 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
}
agent->drops = 0;
agent->fps = MAX_FPS;
- reset_rate(dcc, agent);
- agent->mjpeg_encoder = mjpeg_encoder_new(FALSE, 0, NULL, NULL);
+ agent->dcc = dcc;
+
+ if (dcc->use_mjpeg_encoder_rate_control) {
+ MJpegEncoderRateControlCbs mjpeg_cbs;
+ uint64_t initial_bit_rate;
+
+ mjpeg_cbs.get_roundtrip_ms = red_stream_mjpeg_encoder_get_roundtrip;
+ mjpeg_cbs.get_source_fps = red_stream_mjpeg_encoder_get_source_fps;
+
+ initial_bit_rate = red_stream_get_initial_bit_rate(dcc, stream);
+ agent->mjpeg_encoder = mjpeg_encoder_new(TRUE, initial_bit_rate, &mjpeg_cbs, agent);
+ } else {
+ agent->mjpeg_encoder = mjpeg_encoder_new(FALSE, 0, NULL, NULL);
+ }
red_channel_client_pipe_add(&dcc->common.base, &agent->create_item);
}
@@ -2929,8 +2945,6 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
RingItem *dcc_ring_item;
Stream *stream;
SpiceRect* src_rect;
- int stream_width;
- int stream_height;
spice_assert(!drawable->stream);
@@ -2940,8 +2954,6 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
spice_assert(drawable->red_drawable->type == QXL_DRAW_COPY);
src_rect = &drawable->red_drawable->u.copy.src_area;
- stream_width = src_rect->right - src_rect->left;
- stream_height = src_rect->bottom - src_rect->top;
ring_add(&worker->streams, &stream->link);
stream->current = drawable;
@@ -2950,7 +2962,6 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
stream->height = src_rect->bottom - src_rect->top;
stream->dest_area = drawable->red_drawable->bbox;
stream->refs = 1;
- stream->bit_rate = get_minimal_bit_rate(worker, stream_width, stream_height);
SpiceBitmap *bitmap = &drawable->red_drawable->u.copy.src_bitmap->u.bitmap;
stream->top_down = !!(bitmap->flags & SPICE_BITMAP_FLAGS_TOP_DOWN);
drawable->stream = stream;
@@ -2960,13 +2971,14 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
stream->num_input_frames = 0;
stream->input_fps_timer_start = red_now();
stream->input_fps = MAX_FPS;
+ worker->streams_size_total += stream->width * stream->height;
+ worker->stream_count++;
WORKER_FOREACH_DCC(worker, dcc_ring_item, dcc) {
red_display_create_stream(dcc, stream);
}
- worker->stream_count++;
spice_debug("stream %d %dx%d (%d, %d) (%d, %d)", (int)(stream - worker->streams_buf), stream->width,
- stream->height, stream->dest_area.left, stream->dest_area.top,
- stream->dest_area.right, stream->dest_area.bottom);
+ stream->height, stream->dest_area.left, stream->dest_area.top,
+ stream->dest_area.right, stream->dest_area.bottom);
return;
}
@@ -2995,6 +3007,7 @@ static void red_display_client_init_streams(DisplayChannelClient *dcc)
red_channel_pipe_item_init(channel, &agent->create_item, PIPE_ITEM_TYPE_STREAM_CREATE);
red_channel_pipe_item_init(channel, &agent->destroy_item, PIPE_ITEM_TYPE_STREAM_DESTROY);
}
+ dcc->use_mjpeg_encoder_rate_control = TRUE;
}
static void red_display_destroy_streams_agents(DisplayChannelClient *dcc)
@@ -3110,19 +3123,6 @@ static inline int red_is_next_stream_frame(RedWorker *worker, const Drawable *ca
FALSE);
}
-static void reset_rate(DisplayChannelClient *dcc, StreamAgent *stream_agent)
-{
- Stream *stream = stream_agent->stream;
- int rate;
-
- rate = get_bit_rate(dcc, stream->width, stream->height);
- if (rate == stream->bit_rate) {
- return;
- }
-
- /* MJpeg has no rate limiting anyway, so do nothing */
-}
-
static int display_channel_client_is_low_bandwidth(DisplayChannelClient *dcc)
{
return main_channel_client_is_low_bandwidth(
@@ -8419,9 +8419,12 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
StreamAgent *agent = &dcc->stream_agents[get_stream_id(worker, stream)];
uint64_t time_now = red_now();
size_t outbuf_size;
- if (time_now - agent->last_send_time < (1000 * 1000 * 1000) / agent->fps) {
- agent->frames--;
- return TRUE;
+
+ if (!dcc->use_mjpeg_encoder_rate_control) {
+ if (time_now - agent->last_send_time < (1000 * 1000 * 1000) / agent->fps) {
+ agent->frames--;
+ return TRUE;
+ }
}
outbuf_size = dcc->send_data.stream_outbuf_size;
@@ -8432,7 +8435,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
drawable->red_drawable->mm_time);
switch (ret) {
case MJPEG_ENCODER_FRAME_DROP:
- spice_warning("mjpeg rate control is not supported yet");
+ spice_assert(dcc->use_mjpeg_encoder_rate_control);
return TRUE;
case MJPEG_ENCODER_FRAME_UNSUPPORTED:
return FALSE;
commit b0fb03f0aefcb461fabc9880190eb8bb7796dbbd
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Feb 18 13:11:31 2013 -0500
server/red_worker: enable latency monitoring in the display channel
diff --git a/server/red_worker.c b/server/red_worker.c
index b22a595..2f40635 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -10140,6 +10140,7 @@ static CommonChannelClient *common_channel_client_create(int size,
RedClient *client,
RedsStream *stream,
int mig_target,
+ int monitor_latency,
uint32_t *common_caps,
int num_common_caps,
uint32_t *caps,
@@ -10147,7 +10148,7 @@ static CommonChannelClient *common_channel_client_create(int size,
{
MainChannelClient *mcc = red_client_get_main(client);
RedChannelClient *rcc =
- red_channel_client_create(size, &common->base, client, stream, FALSE,
+ red_channel_client_create(size, &common->base, client, stream, monitor_latency,
num_common_caps, common_caps, num_caps, caps);
if (!rcc) {
return NULL;
@@ -10175,6 +10176,7 @@ DisplayChannelClient *display_channel_client_create(CommonChannel *common,
(DisplayChannelClient*)common_channel_client_create(
sizeof(DisplayChannelClient), common, client, stream,
mig_target,
+ TRUE,
common_caps, num_common_caps,
caps, num_caps);
@@ -10196,6 +10198,7 @@ CursorChannelClient *cursor_channel_create_rcc(CommonChannel *common,
(CursorChannelClient*)common_channel_client_create(
sizeof(CursorChannelClient), common, client, stream,
mig_target,
+ FALSE,
common_caps,
num_common_caps,
caps,
commit 86fbcf1ddb939ed1833495e12303491910c02d9e
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Thu Jan 24 16:20:27 2013 -0500
red_worker: stream - update periodically the input frame rate
Periodically calculate the rate of frames arriving from the guest to the
server.
diff --git a/server/red_worker.c b/server/red_worker.c
index 1ce8523..b22a595 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -115,6 +115,7 @@
#define RED_STREAM_GRADUAL_FRAMES_START_CONDITION 0.2
#define RED_STREAM_FRAMES_RESET_CONDITION 100
#define RED_STREAM_MIN_SIZE (96 * 96)
+#define RED_STREAM_INPUT_FPS_TIMEOUT (5 * 1000) // 5 sec
#define FPS_TEST_INTERVAL 1
#define MAX_FPS 30
@@ -432,6 +433,11 @@ struct Stream {
Stream *next;
RingItem link;
int bit_rate;
+
+ SpiceTimer *input_fps_timer;
+ uint32_t num_input_frames;
+ uint64_t input_fps_timer_start;
+ uint32_t input_fps;
};
typedef struct StreamAgent {
@@ -1063,6 +1069,7 @@ static void dump_bitmap(RedWorker *worker, SpiceBitmap *bitmap, uint32_t group_i
#endif
static void red_push_monitors_config(DisplayChannelClient *dcc);
+static inline uint64_t red_now(void);
/*
* Macros to make iterating over stuff easier
@@ -2477,6 +2484,9 @@ static int is_same_drawable(RedWorker *worker, Drawable *d1, Drawable *d2)
static inline void red_free_stream(RedWorker *worker, Stream *stream)
{
+ if (stream->input_fps_timer) {
+ spice_timer_remove(stream->input_fps_timer);
+ }
stream->next = worker->free_streams;
worker->free_streams = stream;
}
@@ -2555,6 +2565,7 @@ static void red_attach_stream(RedWorker *worker, Drawable *drawable, Stream *str
stream->current = drawable;
drawable->stream = stream;
stream->last_time = drawable->creation_time;
+ stream->num_input_frames++;
WORKER_FOREACH_DCC(worker, item, dcc) {
StreamAgent *agent;
@@ -2892,6 +2903,24 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
red_channel_client_pipe_add(&dcc->common.base, &agent->create_item);
}
+static void red_stream_input_fps_timer_cb(void *opaque)
+{
+ Stream *stream = opaque;
+ uint64_t now = red_now();
+ double duration_sec;
+
+ spice_assert(opaque);
+ if (now == stream->input_fps_timer_start) {
+ spice_warning("timer start and expiry time are equal");
+ return;
+ }
+ duration_sec = (now - stream->input_fps_timer_start)/(1000.0*1000*1000);
+ stream->input_fps = stream->num_input_frames / duration_sec;
+ spice_debug("input-fps=%u", stream->input_fps);
+ stream->num_input_frames = 0;
+ stream->input_fps_timer_start = now;
+}
+
/* TODO: we create the stream even if dcc is NULL, i.e. no client - or
* maybe we can't reach this function in that case? question: do we want to? */
static void red_create_stream(RedWorker *worker, Drawable *drawable)
@@ -2925,7 +2954,12 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
SpiceBitmap *bitmap = &drawable->red_drawable->u.copy.src_bitmap->u.bitmap;
stream->top_down = !!(bitmap->flags & SPICE_BITMAP_FLAGS_TOP_DOWN);
drawable->stream = stream;
-
+ stream->input_fps_timer = spice_timer_queue_add(red_stream_input_fps_timer_cb, stream);
+ spice_assert(stream->input_fps_timer);
+ spice_timer_set(stream->input_fps_timer, RED_STREAM_INPUT_FPS_TIMEOUT);
+ stream->num_input_frames = 0;
+ stream->input_fps_timer_start = red_now();
+ stream->input_fps = MAX_FPS;
WORKER_FOREACH_DCC(worker, dcc_ring_item, dcc) {
red_display_create_stream(dcc, stream);
}
commit 9a62a9a809eaf018707a2a8d790b7be4f3608e3a
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Feb 18 12:42:56 2013 -0500
red_channel: monitor connection latency using MSG_PING
diff --git a/server/inputs_channel.c b/server/inputs_channel.c
index c968bb2..931dac1 100644
--- a/server/inputs_channel.c
+++ b/server/inputs_channel.c
@@ -532,6 +532,7 @@ static void inputs_connect(RedChannel *channel, RedClient *client,
channel,
client,
stream,
+ FALSE,
num_common_caps, common_caps,
num_caps, caps);
if (!icc) {
diff --git a/server/main_channel.c b/server/main_channel.c
index 618f5bf..a065ffb 100644
--- a/server/main_channel.c
+++ b/server/main_channel.c
@@ -1086,7 +1086,7 @@ static MainChannelClient *main_channel_client_create(MainChannel *main_chan, Red
{
MainChannelClient *mcc = (MainChannelClient*)
red_channel_client_create(sizeof(MainChannelClient), &main_chan->base,
- client, stream, num_common_caps,
+ client, stream, FALSE, num_common_caps,
common_caps, num_caps, caps);
spice_assert(mcc != NULL);
mcc->connection_id = connection_id;
diff --git a/server/red_channel.c b/server/red_channel.c
index b52f9e6..b1a6d57 100644
--- a/server/red_channel.c
+++ b/server/red_channel.c
@@ -29,6 +29,7 @@
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
+#include <sys/ioctl.h>
#include "common/generated_server_marshallers.h"
#include "common/ring.h"
@@ -43,6 +44,20 @@ typedef struct EmptyMsgPipeItem {
int msg;
} EmptyMsgPipeItem;
+#define PING_TEST_TIMEOUT_MS 15000
+#define PING_TEST_IDLE_NET_TIMEOUT_MS 100
+
+enum QosPingState {
+ PING_STATE_NONE,
+ PING_STATE_TIMER,
+ PING_STATE_WARMUP,
+ PING_STATE_LATENCY,
+};
+
+static void red_channel_client_start_ping_timer(RedChannelClient *rcc, uint32_t timeout);
+static void red_channel_client_cancel_ping_timer(RedChannelClient *rcc);
+static void red_channel_client_restart_ping_timer(RedChannelClient *rcc);
+
static void red_channel_client_event(int fd, int event, void *data);
static void red_client_add_channel(RedClient *client, RedChannelClient *rcc);
static void red_client_remove_channel(RedChannelClient *rcc);
@@ -481,6 +496,49 @@ static void red_channel_client_send_empty_msg(RedChannelClient *rcc, PipeItem *b
red_channel_client_begin_send_message(rcc);
}
+static void red_channel_client_send_ping(RedChannelClient *rcc)
+{
+ SpiceMsgPing ping;
+ struct timespec ts;
+
+ if (!rcc->latency_monitor.warmup_was_sent) { // latency test start
+ int delay_val;
+ socklen_t opt_size = sizeof(delay_val);
+
+ rcc->latency_monitor.warmup_was_sent = TRUE;
+ /*
+ * When testing latency, TCP_NODELAY must be switched on, otherwise,
+ * sending the ping message is delayed by Nagle algorithm, and the
+ * roundtrip measurment is less accurate (bigger).
+ */
+ rcc->latency_monitor.tcp_nodelay = 1;
+ if (getsockopt(rcc->stream->socket, IPPROTO_TCP, TCP_NODELAY, &delay_val,
+ &opt_size) == -1) {
+ spice_warning("getsockopt failed, %s", strerror(errno));
+ } else {
+ rcc->latency_monitor.tcp_nodelay = delay_val;
+ if (!delay_val) {
+ spice_debug("switching to TCP_NODELAY");
+ delay_val = 1;
+ if (setsockopt(rcc->stream->socket, IPPROTO_TCP, TCP_NODELAY, &delay_val,
+ sizeof(delay_val)) == -1) {
+ if (errno != ENOTSUP) {
+ spice_warning("setsockopt failed, %s", strerror(errno));
+ }
+ }
+ }
+ }
+ }
+
+ red_channel_client_init_send_data(rcc, SPICE_MSG_PING, NULL);
+ ping.id = rcc->latency_monitor.id;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ ping.timestamp = ts.tv_sec * 1000000000LL + ts.tv_nsec;
+ spice_marshall_msg_ping(rcc->send_data.marshaller, &ping);
+ spice_debug("time %lu", ping.timestamp);
+ red_channel_client_begin_send_message(rcc);
+}
+
static void red_channel_client_send_item(RedChannelClient *rcc, PipeItem *item)
{
int handled = TRUE;
@@ -500,6 +558,10 @@ static void red_channel_client_send_item(RedChannelClient *rcc, PipeItem *item)
red_channel_client_send_empty_msg(rcc, item);
free(item);
break;
+ case PIPE_ITEM_TYPE_PING:
+ red_channel_client_send_ping(rcc);
+ free(item);
+ break;
default:
handled = FALSE;
}
@@ -549,7 +611,13 @@ static void red_channel_peer_on_out_msg_done(void *opaque)
red_channel_client_restore_main_sender(rcc);
spice_assert(rcc->send_data.header.data != NULL);
red_channel_client_begin_send_message(rcc);
+ } else {
+ if (rcc->latency_monitor.timer && !rcc->send_data.blocked && rcc->pipe_size == 0) {
+ /* It is possible that the socket will become idle, so we may be able to test latency */
+ red_channel_client_restart_ping_timer(rcc);
+ }
}
+
}
static void red_channel_client_pipe_remove(RedChannelClient *rcc, PipeItem *item)
@@ -636,8 +704,39 @@ static int red_channel_client_pre_create_validate(RedChannel *channel, RedClient
return TRUE;
}
+static void red_channel_client_push_ping(RedChannelClient *rcc)
+{
+ spice_debug(NULL);
+ spice_assert(rcc->latency_monitor.state == PING_STATE_NONE);
+ rcc->latency_monitor.state = PING_STATE_WARMUP;
+ rcc->latency_monitor.warmup_was_sent = FALSE;
+ rcc->latency_monitor.id = rand();
+ red_channel_client_pipe_add_type(rcc, PIPE_ITEM_TYPE_PING);
+ red_channel_client_pipe_add_type(rcc, PIPE_ITEM_TYPE_PING);
+}
+
+static void red_channel_client_ping_timer(void *opaque)
+{
+ int so_unsent_size = 0;
+ RedChannelClient *rcc = opaque;
+
+ spice_assert(rcc->latency_monitor.state == PING_STATE_TIMER);
+ red_channel_client_cancel_ping_timer(rcc);
+ /* retrieving the occupied size of the socket's tcp snd buffer (unacked + unsent) */
+ if (ioctl(rcc->stream->socket, TIOCOUTQ, &so_unsent_size) == -1) {
+ spice_printerr("ioctl(TIOCOUTQ) failed, %s", strerror(errno));
+ }
+ if (so_unsent_size > 0) {
+ spice_debug("tcp snd buffer is still occupied. rescheduling ping");
+ red_channel_client_start_ping_timer(rcc, PING_TEST_IDLE_NET_TIMEOUT_MS);
+ } else {
+ red_channel_client_push_ping(rcc);
+ }
+}
+
RedChannelClient *red_channel_client_create(int size, RedChannel *channel, RedClient *client,
RedsStream *stream,
+ int monitor_latency,
int num_common_caps, uint32_t *common_caps,
int num_caps, uint32_t *caps)
{
@@ -699,6 +798,14 @@ RedChannelClient *red_channel_client_create(int size, RedChannel *channel, RedCl
red_client_add_channel(client, rcc);
red_channel_ref(channel);
pthread_mutex_unlock(&client->lock);
+
+ if (monitor_latency) {
+ rcc->latency_monitor.timer = channel->core->timer_add(
+ red_channel_client_ping_timer, rcc);
+ red_channel_client_start_ping_timer(rcc, PING_TEST_IDLE_NET_TIMEOUT_MS);
+ rcc->latency_monitor.roundtrip = -1;
+ }
+
return rcc;
error:
free(rcc);
@@ -1106,6 +1213,14 @@ void red_channel_push(RedChannel *channel)
}
}
+int red_channel_client_get_roundtrip_ms(RedChannelClient *rcc)
+{
+ if (rcc->latency_monitor.roundtrip < 0) {
+ return rcc->latency_monitor.roundtrip;
+ }
+ return rcc->latency_monitor.roundtrip / 1000 / 1000;
+}
+
static void red_channel_client_init_outgoing_messages_window(RedChannelClient *rcc)
{
rcc->ack_data.messages_window = 0;
@@ -1158,6 +1273,108 @@ static void red_channel_handle_migrate_data(RedChannelClient *rcc, uint32_t size
red_channel_client_seamless_migration_done(rcc);
}
+static void red_channel_client_restart_ping_timer(RedChannelClient *rcc)
+{
+ struct timespec ts;
+ uint64_t passed, timeout;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+
+ passed = ts.tv_sec * 1000000000LL + ts.tv_nsec;
+ passed = passed - rcc->latency_monitor.last_pong_time;
+ passed /= 1000*1000;
+ timeout = PING_TEST_IDLE_NET_TIMEOUT_MS;
+ if (passed < PING_TEST_TIMEOUT_MS) {
+ timeout += PING_TEST_TIMEOUT_MS - passed;
+ }
+
+ red_channel_client_start_ping_timer(rcc, timeout);
+}
+
+static void red_channel_client_start_ping_timer(RedChannelClient *rcc, uint32_t timeout)
+{
+ if (!rcc->latency_monitor.timer) {
+ return;
+ }
+ if (rcc->latency_monitor.state != PING_STATE_NONE) {
+ return;
+ }
+ rcc->latency_monitor.state = PING_STATE_TIMER;
+ rcc->channel->core->timer_start(rcc->latency_monitor.timer, timeout);
+}
+
+static void red_channel_client_cancel_ping_timer(RedChannelClient *rcc)
+{
+ if (!rcc->latency_monitor.timer) {
+ return;
+ }
+ if (rcc->latency_monitor.state != PING_STATE_TIMER) {
+ return;
+ }
+
+ rcc->channel->core->timer_cancel(rcc->latency_monitor.timer);
+ rcc->latency_monitor.state = PING_STATE_NONE;
+}
+
+static void red_channel_client_handle_pong(RedChannelClient *rcc, SpiceMsgPing *ping)
+{
+ uint64_t now;
+ struct timespec ts;
+
+ /* ignoring unexpected pongs, or post-migration pongs for pings that
+ * started just before migration */
+ if (ping->id != rcc->latency_monitor.id) {
+ spice_warning("ping-id (%u)!= pong-id %u",
+ rcc->latency_monitor.id, ping->id);
+ return;
+ }
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ now = ts.tv_sec * 1000000000LL + ts.tv_nsec;
+
+ spice_debug("now %lu", now);
+ if (rcc->latency_monitor.state == PING_STATE_WARMUP) {
+ rcc->latency_monitor.state = PING_STATE_LATENCY;
+ spice_debug("warmup roundtrip %.2f (ms)", (now - ping->timestamp)/1000.0/1000.0);
+ return;
+ } else if (rcc->latency_monitor.state != PING_STATE_LATENCY) {
+ spice_warning("unexpected");
+ return;
+ }
+
+ /* set TCO_NODELAY=0, in case we reverted it for the test*/
+ if (!rcc->latency_monitor.tcp_nodelay) {
+ int delay_val = 0;
+
+ spice_debug("switching to back TCP_NODELAY=0");
+ if (setsockopt(rcc->stream->socket, IPPROTO_TCP, TCP_NODELAY, &delay_val,
+ sizeof(delay_val)) == -1) {
+ if (errno != ENOTSUP) {
+ spice_warning("setsockopt failed, %s", strerror(errno));
+ }
+ }
+ }
+
+ /*
+ * The real network latency shouldn't change during the connection. However,
+ * the measurements can be bigger than the real roundtrip due to other
+ * threads or processes that are utilizing the network. We update the roundtrip
+ * measurement with the minimal value we encountered till now.
+ */
+ if (rcc->latency_monitor.roundtrip < 0 ||
+ now - ping->timestamp < rcc->latency_monitor.roundtrip) {
+ rcc->latency_monitor.roundtrip = now - ping->timestamp;
+ spice_debug("roundtrip ms %.2f (ms)", rcc->latency_monitor.roundtrip/1000.0/1000.0);
+ } else {
+ spice_debug("not updating roundtrip. The latest latency measured was bigger (%.2f)",
+ (now - ping->timestamp)/1000.0/1000.0);
+ }
+
+ rcc->latency_monitor.last_pong_time = now;
+ rcc->latency_monitor.state = PING_STATE_NONE;
+ red_channel_client_start_ping_timer(rcc, PING_TEST_TIMEOUT_MS);
+}
+
int red_channel_client_handle_message(RedChannelClient *rcc, uint32_t size,
uint16_t type, void *message)
{
@@ -1188,6 +1405,9 @@ int red_channel_client_handle_message(RedChannelClient *rcc, uint32_t size,
case SPICE_MSGC_MIGRATE_DATA:
red_channel_handle_migrate_data(rcc, size, message);
break;
+ case SPICE_MSGC_PONG:
+ red_channel_client_handle_pong(rcc, message);
+ break;
default:
spice_printerr("invalid message type %u", type);
return FALSE;
@@ -1229,6 +1449,10 @@ void red_channel_client_begin_send_message(RedChannelClient *rcc)
spice_printerr("BUG: header->type == 0");
return;
}
+
+ /* canceling the latency test timer till the nework is idle */
+ red_channel_client_cancel_ping_timer(rcc);
+
spice_marshaller_flush(m);
rcc->send_data.size = spice_marshaller_get_total_size(m);
rcc->send_data.header.set_msg_size(&rcc->send_data.header,
@@ -1459,6 +1683,10 @@ void red_channel_client_disconnect(RedChannelClient *rcc)
}
reds_stream_free(rcc->stream);
rcc->stream = NULL;
+ if (rcc->latency_monitor.timer) {
+ rcc->channel->core->timer_remove(rcc->latency_monitor.timer);
+ rcc->latency_monitor.timer = NULL;
+ }
red_channel_remove_client(rcc);
rcc->channel->channel_cbs.on_disconnect(rcc);
}
diff --git a/server/red_channel.h b/server/red_channel.h
index 0bd4cb1..f770510 100644
--- a/server/red_channel.h
+++ b/server/red_channel.h
@@ -144,6 +144,7 @@ enum {
PIPE_ITEM_TYPE_SET_ACK=1,
PIPE_ITEM_TYPE_MIGRATE,
PIPE_ITEM_TYPE_EMPTY_MSG,
+ PIPE_ITEM_TYPE_PING,
PIPE_ITEM_TYPE_CHANNEL_BASE=101,
};
@@ -222,6 +223,17 @@ typedef struct RedChannelCapabilities {
int test_capabilty(uint32_t *caps, int num_caps, uint32_t cap);
+typedef struct RedChannelClientLatencyMonitor {
+ int state;
+ uint64_t last_pong_time;
+ SpiceTimer *timer;
+ uint32_t id;
+ int tcp_nodelay;
+ int warmup_was_sent;
+
+ int64_t roundtrip;
+} RedChannelClientLatencyMonitor;
+
struct RedChannelClient {
RingItem channel_link;
RingItem client_link;
@@ -273,6 +285,8 @@ struct RedChannelClient {
int wait_migrate_data;
int wait_migrate_flush_mark;
+
+ RedChannelClientLatencyMonitor latency_monitor;
};
struct RedChannel {
@@ -343,6 +357,7 @@ void red_channel_set_data(RedChannel *channel, void *data);
RedChannelClient *red_channel_client_create(int size, RedChannel *channel, RedClient *client,
RedsStream *stream,
+ int monitor_latency,
int num_common_caps, uint32_t *common_caps,
int num_caps, uint32_t *caps);
// TODO: tmp, for channels that don't use RedChannel yet (e.g., snd channel), but
@@ -417,6 +432,9 @@ void red_channel_client_begin_send_message(RedChannelClient *rcc);
*/
SpiceMarshaller *red_channel_client_switch_to_urgent_sender(RedChannelClient *rcc);
+/* returns -1 if we don't have an estimation */
+int red_channel_client_get_roundtrip_ms(RedChannelClient *rcc);
+
void red_channel_pipe_item_init(RedChannel *channel, PipeItem *item, int type);
// TODO: add back the channel_pipe_add functionality - by adding reference counting
diff --git a/server/red_worker.c b/server/red_worker.c
index ab21c54..1ce8523 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -10113,7 +10113,7 @@ static CommonChannelClient *common_channel_client_create(int size,
{
MainChannelClient *mcc = red_client_get_main(client);
RedChannelClient *rcc =
- red_channel_client_create(size, &common->base, client, stream,
+ red_channel_client_create(size, &common->base, client, stream, FALSE,
num_common_caps, common_caps, num_caps, caps);
if (!rcc) {
return NULL;
diff --git a/server/smartcard.c b/server/smartcard.c
index f1e6244..aad22aa 100644
--- a/server/smartcard.c
+++ b/server/smartcard.c
@@ -805,6 +805,7 @@ static void smartcard_connect_client(RedChannel *channel, RedClient *client,
channel,
client,
stream,
+ FALSE,
num_common_caps, common_caps,
num_caps, caps);
diff --git a/server/spicevmc.c b/server/spicevmc.c
index aba2a5d..e10f183 100644
--- a/server/spicevmc.c
+++ b/server/spicevmc.c
@@ -474,6 +474,7 @@ static void spicevmc_connect(RedChannel *channel, RedClient *client,
}
rcc = red_channel_client_create(sizeof(RedChannelClient), channel, client, stream,
+ FALSE,
num_common_caps, common_caps,
num_caps, caps);
if (!rcc) {
commit d146ae0d926075bcedf93a654c793065c3bebf66
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Sun Apr 1 21:30:52 2012 +0300
server/red_worker: assign timer callbacks to worker_core, using spice_timer_queue
display channel - supplying timeouts interface to red_channel, in order to allow
periodic latency monitoring (see next patch).
diff --git a/server/red_worker.c b/server/red_worker.c
index 3d27f41..ab21c54 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -80,6 +80,7 @@
#include "dispatcher.h"
#include "main_channel.h"
#include "migration_protocol.h"
+#include "spice_timer_queue.h"
//#define COMPRESS_STAT
//#define DUMP_BITMAP
@@ -10090,6 +10091,11 @@ static void worker_watch_remove(SpiceWatch *watch)
}
SpiceCoreInterface worker_core = {
+ .timer_add = spice_timer_queue_add,
+ .timer_start = spice_timer_set,
+ .timer_cancel = spice_timer_cancel,
+ .timer_remove = spice_timer_remove,
+
.watch_update_mask = worker_watch_update_mask,
.watch_add = worker_watch_add,
.watch_remove = worker_watch_remove,
@@ -11860,6 +11866,10 @@ static void red_init(RedWorker *worker, WorkerInitData *init_data)
spice_warn_if(init_data->n_surfaces > NUM_SURFACES);
worker->n_surfaces = init_data->n_surfaces;
+ if (!spice_timer_queue_create()) {
+ spice_error("failed to create timer queue");
+ }
+
message = RED_WORKER_MESSAGE_READY;
write_message(worker->channel, &message);
}
@@ -11893,10 +11903,14 @@ SPICE_GNUC_NORETURN void *red_worker_main(void *arg)
worker->event_timeout = INF_EVENT_WAIT;
for (;;) {
int i, num_events;
+ unsigned int timers_queue_timeout;
+ timers_queue_timeout = spice_timer_queue_get_timeout_ms();
worker->event_timeout = MIN(red_get_streams_timout(worker), worker->event_timeout);
+ worker->event_timeout = MIN(timers_queue_timeout, worker->event_timeout);
num_events = poll(worker->poll_fds, MAX_EVENT_SOURCES, worker->event_timeout);
red_handle_streams_timout(worker);
+ spice_timer_queue_cb();
if (worker->display_channel) {
/* during migration, in the dest, the display channel can be initialized
commit e3bc21957086ce03edcb3867affc2d820291fe75
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Sun Apr 1 14:48:13 2012 +0300
server: spice_timer_queue
Each thread can create a spice_timer_queue, for managing its
own timers.
diff --git a/server/Makefile.am b/server/Makefile.am
index 8b380fc..7a52b17 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -93,6 +93,8 @@ libspice_server_la_SOURCES = \
spice.h \
stat.h \
spicevmc.c \
+ spice_timer_queue.c \
+ spice_timer_queue.h \
zlib_encoder.c \
zlib_encoder.h \
$(NULL)
diff --git a/server/spice_timer_queue.c b/server/spice_timer_queue.c
new file mode 100644
index 0000000..690ab83
--- /dev/null
+++ b/server/spice_timer_queue.c
@@ -0,0 +1,268 @@
+/* -*- Mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ Copyright (C) 2013 Red Hat, Inc.
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include <pthread.h>
+#include "red_common.h"
+#include "spice_timer_queue.h"
+#include "common/ring.h"
+
+static Ring timer_queue_list;
+static int queue_count = 0;
+static pthread_mutex_t queue_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static void spice_timer_queue_init(void)
+{
+ ring_init(&timer_queue_list);
+}
+
+struct SpiceTimer {
+ RingItem link;
+ RingItem active_link;
+
+ SpiceTimerFunc func;
+ void *opaque;
+
+ SpiceTimerQueue *queue;
+
+ int is_active;
+ uint32_t ms;
+ uint64_t expiry_time;
+};
+
+struct SpiceTimerQueue {
+ RingItem link;
+ pthread_t thread;
+ Ring timers;
+ Ring active_timers;
+};
+
+static SpiceTimerQueue *spice_timer_queue_find(void)
+{
+ pthread_t self = pthread_self();
+ RingItem *queue_item;
+
+ RING_FOREACH(queue_item, &timer_queue_list) {
+ SpiceTimerQueue *queue = SPICE_CONTAINEROF(queue_item, SpiceTimerQueue, link);
+
+ if (pthread_equal(self, queue->thread) != 0) {
+ return queue;
+ }
+ }
+
+ return NULL;
+}
+
+static SpiceTimerQueue *spice_timer_queue_find_with_lock(void)
+{
+ SpiceTimerQueue *queue;
+
+ pthread_mutex_lock(&queue_list_lock);
+ queue = spice_timer_queue_find();
+ pthread_mutex_unlock(&queue_list_lock);
+ return queue;
+}
+
+int spice_timer_queue_create(void)
+{
+ SpiceTimerQueue *queue;
+
+ pthread_mutex_lock(&queue_list_lock);
+ if (queue_count == 0) {
+ spice_timer_queue_init();
+ }
+
+ if (spice_timer_queue_find() != NULL) {
+ spice_printerr("timer queue was already created for the thread");
+ return FALSE;
+ }
+
+ queue = spice_new0(SpiceTimerQueue, 1);
+ queue->thread = pthread_self();
+ ring_init(&queue->timers);
+ ring_init(&queue->active_timers);
+
+ ring_add(&timer_queue_list, &queue->link);
+ queue_count++;
+
+ pthread_mutex_unlock(&queue_list_lock);
+
+ return TRUE;
+}
+
+void spice_timer_queue_destroy(void)
+{
+ RingItem *item;
+ SpiceTimerQueue *queue;
+
+ pthread_mutex_lock(&queue_list_lock);
+ queue = spice_timer_queue_find();
+
+ spice_assert(queue != NULL);
+
+ while ((item = ring_get_head(&queue->timers))) {
+ SpiceTimer *timer;
+
+ timer = SPICE_CONTAINEROF(item, SpiceTimer, link);
+ spice_timer_remove(timer);
+ }
+
+ ring_remove(&queue->link);
+ free(queue);
+ queue_count--;
+
+ pthread_mutex_unlock(&queue_list_lock);
+}
+
+SpiceTimer *spice_timer_queue_add(SpiceTimerFunc func, void *opaque)
+{
+ SpiceTimer *timer = spice_new0(SpiceTimer, 1);
+ SpiceTimerQueue *queue = spice_timer_queue_find_with_lock();
+
+ spice_assert(queue != NULL);
+
+ ring_item_init(&timer->link);
+ ring_item_init(&timer->active_link);
+
+ timer->opaque = opaque;
+ timer->func = func;
+ timer->queue = queue;
+
+ ring_add(&queue->timers, &timer->link);
+
+ return timer;
+}
+
+static void _spice_timer_set(SpiceTimer *timer, uint32_t ms, uint32_t now)
+{
+ RingItem *next_item;
+ SpiceTimerQueue *queue;
+
+ if (timer->is_active) {
+ spice_timer_cancel(timer);
+ }
+
+ queue = timer->queue;
+ timer->expiry_time = now + ms;
+ timer->ms = ms;
+
+ RING_FOREACH(next_item, &queue->active_timers) {
+ SpiceTimer *next_timer = SPICE_CONTAINEROF(next_item, SpiceTimer, active_link);
+
+ if (timer->expiry_time <= next_timer->expiry_time) {
+ break;
+ }
+ }
+
+ if (next_item) {
+ ring_add_before(&timer->active_link, next_item);
+ } else {
+ ring_add_before(&timer->active_link, &queue->active_timers);
+ }
+ timer->is_active = TRUE;
+}
+
+void spice_timer_set(SpiceTimer *timer, uint32_t ms)
+{
+ struct timespec now;
+
+ spice_assert(pthread_equal(timer->queue->thread, pthread_self()) != 0);
+
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ _spice_timer_set(timer, ms, now.tv_sec * 1000 + (now.tv_nsec / 1000 / 1000));
+}
+
+void spice_timer_cancel(SpiceTimer *timer)
+{
+ spice_assert(pthread_equal(timer->queue->thread, pthread_self()) != 0);
+
+ if (!ring_item_is_linked(&timer->active_link)) {
+ spice_assert(!timer->is_active);
+ return;
+ }
+
+ spice_assert(timer->is_active);
+ ring_remove(&timer->active_link);
+ timer->is_active = FALSE;
+}
+
+void spice_timer_remove(SpiceTimer *timer)
+{
+ spice_assert(timer->queue);
+ spice_assert(ring_item_is_linked(&timer->link));
+ spice_assert(pthread_equal(timer->queue->thread, pthread_self()) != 0);
+
+ if (timer->is_active) {
+ spice_assert(ring_item_is_linked(&timer->active_link));
+ ring_remove(&timer->active_link);
+ }
+ ring_remove(&timer->link);
+ free(timer);
+}
+
+unsigned int spice_timer_queue_get_timeout_ms(void)
+{
+ struct timespec now;
+ int now_ms;
+ RingItem *head;
+ SpiceTimer *head_timer;
+ SpiceTimerQueue *queue = spice_timer_queue_find_with_lock();
+
+ spice_assert(queue != NULL);
+
+ if (ring_is_empty(&queue->active_timers)) {
+ return -1;
+ }
+
+ head = ring_get_head(&queue->active_timers);
+ head_timer = SPICE_CONTAINEROF(head, SpiceTimer, active_link);
+
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ now_ms = (now.tv_sec * 1000) - (now.tv_nsec / 1000 / 1000);
+
+ return MAX(0, ((int)head_timer->expiry_time - now_ms));
+}
+
+
+void spice_timer_queue_cb(void)
+{
+ struct timespec now;
+ uint64_t now_ms;
+ RingItem *head;
+ SpiceTimerQueue *queue = spice_timer_queue_find_with_lock();
+
+ spice_assert(queue != NULL);
+
+ if (ring_is_empty(&queue->active_timers)) {
+ return;
+ }
+
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ now_ms = (now.tv_sec * 1000) + (now.tv_nsec / 1000 / 1000);
+
+ while ((head = ring_get_head(&queue->active_timers))) {
+ SpiceTimer *timer = SPICE_CONTAINEROF(head, SpiceTimer, active_link);
+
+ if (timer->expiry_time > now_ms) {
+ break;
+ } else {
+ timer->func(timer->opaque);
+ if (timer->is_active) {
+ _spice_timer_set(timer, timer->ms, now_ms);
+ }
+ }
+ }
+}
diff --git a/server/spice_timer_queue.h b/server/spice_timer_queue.h
new file mode 100644
index 0000000..a84f6cd
--- /dev/null
+++ b/server/spice_timer_queue.h
@@ -0,0 +1,43 @@
+/*
+ Copyright (C) 2013 Red Hat, Inc.
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _H_SPICE_TIMER_QUEUE
+#define _H_SPICE_TIMER_QUEUE
+
+#include <stdint.h>
+#include "spice.h"
+
+typedef struct SpiceTimerQueue SpiceTimerQueue;
+
+/* create/destroy a timer queue for the current thread.
+ * In order to execute the timers functions, spice_timer_queue_cb should be called
+ * periodically, according to spice_timer_queue_get_timeout_ms */
+int spice_timer_queue_create(void);
+void spice_timer_queue_destroy(void);
+
+SpiceTimer *spice_timer_queue_add(SpiceTimerFunc func, void *opaque);
+void spice_timer_set(SpiceTimer *timer, uint32_t ms);
+void spice_timer_cancel(SpiceTimer *timer);
+void spice_timer_remove(SpiceTimer *timer);
+
+/* returns the time left till the earliest timer in the queue expires.
+ * returns (unsigned)-1 if there are no active timers */
+unsigned int spice_timer_queue_get_timeout_ms(void);
+/* call the timeout callbacks of all the expired timers */
+void spice_timer_queue_cb(void);
+
+#endif
commit 622d7159c26876d9a579b0dadee52091a87220b9
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Wed Feb 20 21:01:22 2013 -0500
mjpeg_encoder: add stream warmup time, in which we avoid server and client drops
The stream starts after lossless frames were sent to the client,
and without rate control (except for pipe congestion). Thus, on the beginning
of the stream, we might observe frame drops on the client and server side which
are not necessarily related to mis-estimation of the bit rate, and we would
like to wait till the stream stabilizes.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index 41d234e..7328ea2 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -59,6 +59,15 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
*/
#define MJPEG_MAX_CLIENT_PLAYBACK_DELAY 5000 // 5 sec
+/*
+ * The stream starts after lossless frames were sent to the client,
+ * and without rate control (except for pipe congestion). Thus, on the beginning
+ * of the stream, we might observe frame drops on the client and server side which
+ * are not necessarily related to mis-estimation of the bit rate, and we would
+ * like to wait till the stream stabilizes.
+ */
+#define MJPEG_WARMUP_TIME 3000L // 3 sec
+
enum {
MJPEG_QUALITY_EVAL_TYPE_SET,
MJPEG_QUALITY_EVAL_TYPE_UPGRADE,
@@ -140,6 +149,7 @@ typedef struct MJpegEncoderRateControl {
uint64_t sum_recent_enc_size;
uint32_t num_recent_enc_frames;
+ uint64_t warmup_start_time;
} MJpegEncoderRateControl;
struct MJpegEncoder {
@@ -182,12 +192,16 @@ MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate
enc->rate_control_is_active = bit_rate_control;
enc->rate_control.byte_rate = starting_bit_rate / 8;
if (bit_rate_control) {
+ struct timespec time;
+
+ clock_gettime(CLOCK_MONOTONIC, &time);
enc->cbs = *cbs;
enc->cbs_opaque = opaque;
mjpeg_encoder_reset_quality(enc, MJPEG_QUALITY_SAMPLE_NUM / 2, 5, 0);
enc->rate_control.during_quality_eval = TRUE;
enc->rate_control.quality_eval_data.type = MJPEG_QUALITY_EVAL_TYPE_SET;
enc->rate_control.quality_eval_data.reason = MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE;
+ enc->rate_control.warmup_start_time = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
} else {
mjpeg_encoder_reset_quality(enc, MJPEG_LEGACY_STATIC_QUALITY_ID, MJPEG_MAX_FPS, 0);
}
@@ -904,6 +918,19 @@ static void mjpeg_encoder_decrease_bit_rate(MJpegEncoder *encoder)
rate_control->client_state.max_video_latency = 0;
rate_control->client_state.max_audio_latency = 0;
+ if (rate_control->warmup_start_time) {
+ struct timespec time;
+ uint64_t now;
+
+ clock_gettime(CLOCK_MONOTONIC, &time);
+ now = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
+ if (now - rate_control->warmup_start_time < MJPEG_WARMUP_TIME*1000*1000) {
+ spice_debug("during warmup. ignoring");
+ return;
+ } else {
+ rate_control->warmup_start_time = 0;
+ }
+ }
if (bit_rate_info->num_enc_frames > MJPEG_BIT_RATE_EVAL_MIN_NUM_FRAMES ||
bit_rate_info->num_enc_frames > rate_control->fps) {
commit ff1bde1d81d3f28c9079e8c274748253eb93b8d8
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Feb 18 09:25:43 2013 -0500
mjpeg_encoder: keep the average observed fps similar to the defined fps
The actual frames distribution does not necessarily fit the
condition "at least one frame every (1000/rate_contorl->fps)
milliseconds".
For keeping the average frame rate close to the defined fps, we
periodically measure the current average fps, and modify
rate_control->adjusted_fps accordingly. Then, we use
(1000/rate_control->adjusted_fps) as the interval between the
frames.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index ac92924..41d234e 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -51,6 +51,8 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
#define MJPEG_CLIENT_POSITIVE_REPORT_TIMEOUT 2000
#define MJPEG_CLIENT_POSITIVE_REPORT_STRICT_TIMEOUT 3000
+#define MJPEG_ADJUST_FPS_TIMEOUT 500
+
/*
* avoid interrupting the playback when there are temporary
* incidents of instability (with respect to server and client drops)
@@ -105,6 +107,7 @@ typedef struct MJpegEncoderBitRateInfo {
uint32_t num_enc_frames;
uint64_t sum_enc_size;
} MJpegEncoderBitRateInfo;
+
/*
* Adjusting the stream jpeg quality and frame rate (fps):
* When during_quality_eval=TRUE, we compress different frames with different
@@ -125,6 +128,10 @@ typedef struct MJpegEncoderRateControl {
uint64_t byte_rate;
int quality_id;
uint32_t fps;
+ double adjusted_fps;
+ uint64_t adjusted_fps_start_time;
+ uint64_t adjusted_fps_num_frames;
+
/* the encoded frame size which the quality and the fps evaluation was based upon */
uint64_t base_enc_size;
@@ -132,6 +139,7 @@ typedef struct MJpegEncoderRateControl {
uint64_t sum_recent_enc_size;
uint32_t num_recent_enc_frames;
+
} MJpegEncoderRateControl;
struct MJpegEncoder {
@@ -355,6 +363,7 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
uint64_t frame_enc_size)
{
MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ double fps_ratio;
rate_control->during_quality_eval = FALSE;
@@ -362,7 +371,6 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
rate_control->last_enc_size = 0;
}
-
if (rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE) {
memset(&rate_control->server_state, 0, sizeof(MJpegEncoderServerState));
}
@@ -371,8 +379,17 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
rate_control->quality_eval_data.max_quality_id = MJPEG_QUALITY_SAMPLE_NUM - 1;
rate_control->quality_eval_data.max_quality_fps = MJPEG_MAX_FPS;
+ if (rate_control->adjusted_fps) {
+ fps_ratio = rate_control->adjusted_fps / rate_control->fps;
+ } else {
+ fps_ratio = 1.5;
+ }
rate_control->fps = MAX(MJPEG_MIN_FPS, fps);
rate_control->fps = MIN(MJPEG_MAX_FPS, rate_control->fps);
+ rate_control->adjusted_fps = rate_control->fps*fps_ratio;
+ spice_debug("adjusted-fps-ratio=%.2f adjusted-fps=%.2f", fps_ratio, rate_control->adjusted_fps);
+ rate_control->adjusted_fps_start_time = 0;
+ rate_control->adjusted_fps_num_frames = 0;
rate_control->base_enc_size = frame_enc_size;
rate_control->sum_recent_enc_size = 0;
@@ -640,6 +657,54 @@ end:
}
}
+/*
+ * The actual frames distribution does not necessarily fit the condition "at least
+ * one frame every (1000/rate_contorl->fps) milliseconds".
+ * For keeping the average fps close to the defined fps, we periodically
+ * measure the current average fps, and modify rate_control->adjusted_fps accordingly.
+ * Then, we use (1000/rate_control->adjusted_fps) as the interval between frames.
+ */
+static void mjpeg_encoder_adjust_fps(MJpegEncoder *encoder, uint64_t now)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ uint64_t adjusted_fps_time_passed;
+
+ if (!encoder->rate_control_is_active) {
+ return;
+ }
+ adjusted_fps_time_passed = (now - rate_control->adjusted_fps_start_time) / 1000 / 1000;
+
+ if (!rate_control->during_quality_eval &&
+ adjusted_fps_time_passed > MJPEG_ADJUST_FPS_TIMEOUT &&
+ adjusted_fps_time_passed > 1000 / rate_control->adjusted_fps) {
+ double avg_fps;
+ double fps_ratio;
+
+ avg_fps = ((double)rate_control->adjusted_fps_num_frames*1000) /
+ adjusted_fps_time_passed;
+ spice_debug("#frames-adjust=%lu #adjust-time=%lu avg-fps=%.2f",
+ rate_control->adjusted_fps_num_frames, adjusted_fps_time_passed, avg_fps);
+ spice_debug("defined=%u old-adjusted=%.2f", rate_control->fps, rate_control->adjusted_fps);
+ fps_ratio = avg_fps / rate_control->fps;
+ if (avg_fps + 0.5 < rate_control->fps &&
+ encoder->cbs.get_source_fps(encoder->cbs_opaque) > avg_fps) {
+ double new_adjusted_fps = avg_fps ?
+ (rate_control->adjusted_fps/fps_ratio) :
+ rate_control->adjusted_fps * 2;
+
+ rate_control->adjusted_fps = MIN(rate_control->fps*2, new_adjusted_fps);
+ spice_debug("new-adjusted-fps=%.2f", rate_control->adjusted_fps);
+ } else if (rate_control->fps + 0.5 < avg_fps) {
+ double new_adjusted_fps = rate_control->adjusted_fps / fps_ratio;
+
+ rate_control->adjusted_fps = MAX(rate_control->fps, new_adjusted_fps);
+ spice_debug("new-adjusted-fps=%.2f", rate_control->adjusted_fps);
+ }
+ rate_control->adjusted_fps_start_time = now;
+ rate_control->adjusted_fps_num_frames = 0;
+ }
+}
+
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
int width, int height,
uint8_t **dest, size_t *dest_len,
@@ -655,9 +720,14 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
clock_gettime(CLOCK_MONOTONIC, &time);
now = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
+
+ if (!rate_control->adjusted_fps_start_time) {
+ rate_control->adjusted_fps_start_time = now;
+ }
+ mjpeg_encoder_adjust_fps(encoder, now);
interval = (now - rate_control->bit_rate_info.last_frame_time);
- if (interval < (1000*1000*1000) / rate_control->fps) {
+ if (interval < (1000*1000*1000) / rate_control->adjusted_fps) {
return MJPEG_ENCODER_FRAME_DROP;
}
@@ -782,6 +852,7 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder)
}
rate_control->sum_recent_enc_size += rate_control->last_enc_size;
rate_control->num_recent_enc_frames++;
+ rate_control->adjusted_fps_num_frames++;
}
rate_control->bit_rate_info.sum_enc_size += encoder->rate_control.last_enc_size;
rate_control->bit_rate_info.num_enc_frames++;
commit 6f883d0eb54ba8b0a1e7d45cd1ae77cd48178c08
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Sun Feb 17 22:48:05 2013 -0500
mjpeg_encoder: move the control over frame drops to mjpeg_encoder
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index 69cd20c..ac92924 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -651,9 +651,15 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
MJpegEncoderRateControl *rate_control = &encoder->rate_control;
struct timespec time;
uint64_t now;
+ uint64_t interval;
clock_gettime(CLOCK_MONOTONIC, &time);
now = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
+ interval = (now - rate_control->bit_rate_info.last_frame_time);
+
+ if (interval < (1000*1000*1000) / rate_control->fps) {
+ return MJPEG_ENCODER_FRAME_DROP;
+ }
mjpeg_encoder_adjust_params_to_bit_rate(encoder);
@@ -700,14 +706,14 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
break;
default:
spice_warning("unsupported format %d", format);
- return FALSE;
+ return MJPEG_ENCODER_FRAME_UNSUPPORTED;
}
if (encoder->pixel_converter != NULL) {
unsigned int stride = width * 3;
/* check for integer overflow */
if (stride < width) {
- return FALSE;
+ return MJPEG_ENCODER_FRAME_UNSUPPORTED;
}
if (encoder->row_size < stride) {
encoder->row = spice_realloc(encoder->row, stride);
@@ -725,7 +731,7 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
jpeg_set_quality(&encoder->cinfo, quality, TRUE);
jpeg_start_compress(&encoder->cinfo, encoder->first_frame);
- return TRUE;
+ return MJPEG_ENCODER_FRAME_ENCODE_START;
}
int mjpeg_encoder_encode_scanline(MJpegEncoder *encoder, uint8_t *src_pixels,
@@ -783,14 +789,6 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder)
return encoder->rate_control.last_enc_size;
}
-uint32_t mjpeg_encoder_get_fps(MJpegEncoder *encoder)
-{
- if (!encoder->rate_control_is_active) {
- spice_warning("bit rate control is not active");
- }
- return encoder->rate_control.fps;
-}
-
static void mjpeg_encoder_quality_eval_stop(MJpegEncoder *encoder)
{
MJpegEncoderRateControl *rate_control = &encoder->rate_control;
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index f9ae43c..0ee2e96 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -21,6 +21,12 @@
#include "red_common.h"
+enum {
+ MJPEG_ENCODER_FRAME_UNSUPPORTED = -1,
+ MJPEG_ENCODER_FRAME_DROP,
+ MJPEG_ENCODER_FRAME_ENCODE_START,
+};
+
typedef struct MJpegEncoder MJpegEncoder;
/*
@@ -44,8 +50,15 @@ void mjpeg_encoder_destroy(MJpegEncoder *encoder);
uint8_t mjpeg_encoder_get_bytes_per_pixel(MJpegEncoder *encoder);
/*
- * *dest must be either NULL or allocated by malloc, since it might be freed
+ * dest must be either NULL or allocated by malloc, since it might be freed
* during the encoding, if its size is too small.
+ *
+ * return:
+ * MJPEG_ENCODER_FRAME_UNSUPPORTED : frame cannot be encoded
+ * MJPEG_ENCODER_FRAME_DROP : frame should be dropped. This value can only be returned
+ * if mjpeg rate control is active.
+ * MJPEG_ENCODER_FRAME_ENCODE_START: frame encoding started. Continue with
+ * mjpeg_encoder_encode_scanline.
*/
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
int width, int height,
@@ -60,12 +73,6 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder);
*/
/*
- * The recommended output frame rate (per second) for the
- * current available bit rate.
- */
-uint32_t mjpeg_encoder_get_fps(MJpegEncoder *encoder);
-
-/*
* Data that should be periodically obtained from the client. The report contains:
* num_frames : the number of frames that reached the client during the time
* the report is referring to.
diff --git a/server/red_worker.c b/server/red_worker.c
index b453023..3d27f41 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -8352,6 +8352,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
RedWorker *worker = dcc->common.worker;
int n;
int width, height;
+ int ret;
if (!stream) {
spice_assert(drawable->sized_stream);
@@ -8389,13 +8390,24 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
}
outbuf_size = dcc->send_data.stream_outbuf_size;
- if (!mjpeg_encoder_start_frame(agent->mjpeg_encoder, image->u.bitmap.format,
- width, height,
- &dcc->send_data.stream_outbuf,
- &outbuf_size,
- drawable->red_drawable->mm_time)) {
+ ret = mjpeg_encoder_start_frame(agent->mjpeg_encoder, image->u.bitmap.format,
+ width, height,
+ &dcc->send_data.stream_outbuf,
+ &outbuf_size,
+ drawable->red_drawable->mm_time);
+ switch (ret) {
+ case MJPEG_ENCODER_FRAME_DROP:
+ spice_warning("mjpeg rate control is not supported yet");
+ return TRUE;
+ case MJPEG_ENCODER_FRAME_UNSUPPORTED:
+ return FALSE;
+ case MJPEG_ENCODER_FRAME_ENCODE_START:
+ break;
+ default:
+ spice_error("bad return value (%d) from mjpeg_encoder_start_frame", ret);
return FALSE;
}
+
if (!encode_frame(dcc, &drawable->red_drawable->u.copy.src_area,
&image->u.bitmap, stream)) {
return FALSE;
commit 44ce87b55a2f96fd83d1aeec3582cc945f0ec452
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Sun Feb 17 22:06:29 2013 -0500
mjpeg_encoder: update the client with estimations for the required playback latency
The required client playback latency is assessed based on the current
estimation of the bit rate, the network latency, and the encoding size
of the frames. When the playback delay that is reported by the client
seems too small, or when the stream parameters change, we send the
client an updated playback latency estimation.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index e67ceef..69cd20c 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -51,6 +51,12 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
#define MJPEG_CLIENT_POSITIVE_REPORT_TIMEOUT 2000
#define MJPEG_CLIENT_POSITIVE_REPORT_STRICT_TIMEOUT 3000
+/*
+ * avoid interrupting the playback when there are temporary
+ * incidents of instability (with respect to server and client drops)
+ */
+#define MJPEG_MAX_CLIENT_PLAYBACK_DELAY 5000 // 5 sec
+
enum {
MJPEG_QUALITY_EVAL_TYPE_SET,
MJPEG_QUALITY_EVAL_TYPE_UPGRADE,
@@ -151,6 +157,9 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
uint64_t frame_enc_size);
static uint32_t get_max_fps(uint64_t frame_size, uint64_t bytes_per_sec);
static void mjpeg_encoder_process_server_drops(MJpegEncoder *encoder);
+static uint32_t get_min_required_playback_delay(uint64_t frame_enc_size,
+ uint64_t byte_rate,
+ uint32_t latency);
MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate,
MJpegEncoderRateControlCbs *cbs, void *opaque)
@@ -512,6 +521,14 @@ complete_sample:
spice_debug("MJpeg quality sample end %p: quality %d fps %d",
encoder, mjpeg_quality_samples[rate_control->quality_id], rate_control->fps);
+ if (encoder->cbs.update_client_playback_delay) {
+ uint32_t latency = mjpeg_encoder_get_latency(encoder);
+ uint32_t min_delay = get_min_required_playback_delay(final_quality_enc_size,
+ rate_control->byte_rate,
+ latency);
+
+ encoder->cbs.update_client_playback_delay(encoder->cbs_opaque, min_delay);
+ }
}
static void mjpeg_encoder_quality_eval_set_upgrade(MJpegEncoder *encoder,
@@ -974,13 +991,15 @@ static uint32_t get_min_required_playback_delay(uint64_t frame_enc_size,
uint32_t latency)
{
uint32_t one_frame_time;
+ uint32_t min_delay;
if (!frame_enc_size || !byte_rate) {
return latency;
}
one_frame_time = (frame_enc_size*1000)/byte_rate;
- return one_frame_time*2 + latency;
+ min_delay = MIN(one_frame_time*2 + latency, MJPEG_MAX_CLIENT_PLAYBACK_DELAY);
+ return min_delay;
}
#define MJPEG_PLAYBACK_LATENCY_DECREASE_FACTOR 0.5
@@ -999,6 +1018,7 @@ void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
MJpegEncoderClientState *client_state = &rate_control->client_state;
uint64_t avg_enc_size = 0;
uint32_t min_playback_delay;
+ int is_video_delay_small = FALSE;
spice_debug("client report: #frames %u, #drops %d, duration %u video-delay %d audio-delay %u",
num_frames, num_drops,
@@ -1026,6 +1046,23 @@ void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
mjpeg_encoder_get_latency(encoder));
spice_debug("min-delay %u client-delay %d", min_playback_delay, end_frame_delay);
+ if (min_playback_delay > end_frame_delay) {
+ uint32_t src_fps = encoder->cbs.get_source_fps(encoder->cbs_opaque);
+ /*
+ * if the stream is at its highest rate, we can't estimate the "real"
+ * network bit rate and the min_playback_delay
+ */
+ if (rate_control->quality_id != MJPEG_QUALITY_SAMPLE_NUM - 1 ||
+ rate_control->fps < MIN(src_fps, MJPEG_MAX_FPS) || end_frame_delay < 0) {
+ is_video_delay_small = TRUE;
+ if (encoder->cbs.update_client_playback_delay) {
+ encoder->cbs.update_client_playback_delay(encoder->cbs_opaque,
+ min_playback_delay);
+ }
+ }
+ }
+
+
/*
* If the audio latency has decreased (since the start of the current
* sequence of positive reports), and the video latency is bigger, slow down
@@ -1045,25 +1082,12 @@ void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
mjpeg_encoder_handle_negative_client_stream_report(encoder,
end_frame_mm_time);
} else {
- int is_video_delay_small = FALSE;
double major_delay_decrease_thresh;
double medium_delay_decrease_thresh;
client_state->max_video_latency = MAX(end_frame_delay, client_state->max_video_latency);
client_state->max_audio_latency = MAX(audio_delay, client_state->max_audio_latency);
- if (min_playback_delay > end_frame_delay) {
- uint32_t src_fps = encoder->cbs.get_source_fps(encoder->cbs_opaque);
- /*
- * if the stream is at its highest rate, we can't estimate the "real"
- * network bit rate and the min_playback_delay
- */
- if (rate_control->quality_id != MJPEG_QUALITY_SAMPLE_NUM - 1 ||
- rate_control->fps < MIN(src_fps, MJPEG_MAX_FPS)) {
- is_video_delay_small = TRUE;
- }
- }
-
medium_delay_decrease_thresh = client_state->max_video_latency;
medium_delay_decrease_thresh *= MJPEG_PLAYBACK_LATENCY_DECREASE_FACTOR;
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index bc7f01c..f9ae43c 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -34,6 +34,7 @@ typedef struct MJpegEncoder MJpegEncoder;
typedef struct MJpegEncoderRateControlCbs {
uint32_t (*get_roundtrip_ms)(void *opaque);
uint32_t (*get_source_fps)(void *opaque);
+ void (*update_client_playback_delay)(void *opaque, uint32_t delay_ms);
} MJpegEncoderRateControlCbs;
MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate,
commit 3bbde4b3a67397c95d541913e2fe4210ed3d7f00
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Sun Feb 17 21:44:59 2013 -0500
mjpeg_encoder: modify stream bit rate based on server side pipe congestion
Downgrading stream bit rate when the input frame rate in the server
exceeds the output frame rate, and frames are being dropped from the
output pipe.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index fd0bae3..e67ceef 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -40,6 +40,9 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
#define MJPEG_BIT_RATE_EVAL_MIN_NUM_FRAMES 3
#define MJPEG_LOW_FPS_RATE_TH 3
+#define MJPEG_SERVER_STATUS_EVAL_FPS_INTERVAL 1
+#define MJPEG_SERVER_STATUS_DOWNGRADE_DROP_FACTOR_TH 0.1
+
/*
* acting on positive client reports only if enough frame mm time
* has passed since the last bit rate change and the report.
@@ -80,6 +83,11 @@ typedef struct MJpegEncoderClientState {
uint32_t max_audio_latency;
} MJpegEncoderClientState;
+typedef struct MJpegEncoderServerState {
+ uint32_t num_frames_encoded;
+ uint32_t num_frames_dropped;
+} MJpegEncoderServerState;
+
typedef struct MJpegEncoderBitRateInfo {
uint64_t change_start_time;
uint64_t last_frame_time;
@@ -106,6 +114,7 @@ typedef struct MJpegEncoderRateControl {
MJpegEncoderQualityEval quality_eval_data;
MJpegEncoderBitRateInfo bit_rate_info;
MJpegEncoderClientState client_state;
+ MJpegEncoderServerState server_state;
uint64_t byte_rate;
int quality_id;
@@ -141,6 +150,7 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
uint32_t fps,
uint64_t frame_enc_size);
static uint32_t get_max_fps(uint64_t frame_size, uint64_t bytes_per_sec);
+static void mjpeg_encoder_process_server_drops(MJpegEncoder *encoder);
MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate,
MJpegEncoderRateControlCbs *cbs, void *opaque)
@@ -343,6 +353,10 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
rate_control->last_enc_size = 0;
}
+
+ if (rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE) {
+ memset(&rate_control->server_state, 0, sizeof(MJpegEncoderServerState));
+ }
rate_control->quality_id = quality_id;
memset(&rate_control->quality_eval_data, 0, sizeof(MJpegEncoderQualityEval));
rate_control->quality_eval_data.max_quality_id = MJPEG_QUALITY_SAMPLE_NUM - 1;
@@ -532,7 +546,7 @@ static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
{
MJpegEncoderRateControl *rate_control;
MJpegEncoderQualityEval *quality_eval;
- uint64_t new_avg_enc_size;
+ uint64_t new_avg_enc_size = 0;
uint32_t new_fps;
uint32_t latency = 0;
uint32_t src_fps;
@@ -559,7 +573,7 @@ static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
if (rate_control->num_recent_enc_frames < MJPEG_AVERAGE_SIZE_WINDOW &&
rate_control->num_recent_enc_frames < rate_control->fps) {
- return;
+ goto end;
}
latency = mjpeg_encoder_get_latency(encoder);
@@ -600,10 +614,12 @@ static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
rate_control->quality_id,
rate_control->fps);
}
-
+end:
if (rate_control->during_quality_eval) {
quality_eval->encoded_size_by_quality[rate_control->quality_id] = new_avg_enc_size;
mjpeg_encoder_eval_quality(encoder);
+ } else {
+ mjpeg_encoder_process_server_drops(encoder);
}
}
@@ -612,10 +628,31 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
uint8_t **dest, size_t *dest_len,
uint32_t frame_mm_time)
{
- MJpegEncoderBitRateInfo *bit_rate_info;
uint32_t quality;
- mjpeg_encoder_adjust_params_to_bit_rate(encoder);
+ if (encoder->rate_control_is_active) {
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ struct timespec time;
+ uint64_t now;
+
+ clock_gettime(CLOCK_MONOTONIC, &time);
+ now = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
+
+ mjpeg_encoder_adjust_params_to_bit_rate(encoder);
+
+ if (!rate_control->during_quality_eval ||
+ rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE) {
+ MJpegEncoderBitRateInfo *bit_rate_info;
+
+ bit_rate_info = &encoder->rate_control.bit_rate_info;
+
+ if (!bit_rate_info->change_start_time) {
+ bit_rate_info->change_start_time = now;
+ bit_rate_info->change_start_mm_time = frame_mm_time;
+ }
+ bit_rate_info->last_frame_time = now;
+ }
+ }
encoder->cinfo.in_color_space = JCS_RGB;
encoder->cinfo.input_components = 3;
@@ -663,23 +700,6 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
spice_jpeg_mem_dest(&encoder->cinfo, dest, dest_len);
- if (!encoder->rate_control.during_quality_eval ||
- encoder->rate_control.quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE) {
- struct timespec time;
- uint64_t now;
-
- clock_gettime(CLOCK_MONOTONIC, &time);
- now = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
-
- bit_rate_info = &encoder->rate_control.bit_rate_info;
-
- if (!bit_rate_info->change_start_time) {
- bit_rate_info->change_start_time = now;
- bit_rate_info->change_start_mm_time = frame_mm_time;
- }
- bit_rate_info->last_frame_time = now;
- }
-
encoder->cinfo.image_width = width;
encoder->cinfo.image_height = height;
jpeg_set_defaults(&encoder->cinfo);
@@ -727,6 +747,7 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder)
encoder->first_frame = FALSE;
rate_control->last_enc_size = dest->pub.next_output_byte - dest->buffer;
+ rate_control->server_state.num_frames_encoded++;
if (!rate_control->during_quality_eval ||
rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE) {
@@ -1068,3 +1089,41 @@ void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
}
}
}
+
+void mjpeg_encoder_notify_server_frame_drop(MJpegEncoder *encoder)
+{
+ encoder->rate_control.server_state.num_frames_dropped++;
+ mjpeg_encoder_process_server_drops(encoder);
+}
+
+/*
+ * decrease the bit rate if the drop rate on the sever side exceeds a pre defined
+ * threshold.
+ */
+static void mjpeg_encoder_process_server_drops(MJpegEncoder *encoder)
+{
+ MJpegEncoderServerState *server_state = &encoder->rate_control.server_state;
+ uint32_t num_frames_total;
+ double drop_factor;
+ uint32_t fps;
+
+ fps = MIN(encoder->rate_control.fps, encoder->cbs.get_source_fps(encoder->cbs_opaque));
+ if (server_state->num_frames_encoded < fps * MJPEG_SERVER_STATUS_EVAL_FPS_INTERVAL) {
+ return;
+ }
+
+ num_frames_total = server_state->num_frames_dropped + server_state->num_frames_encoded;
+ drop_factor = ((double)server_state->num_frames_dropped) / num_frames_total;
+
+ spice_debug("#drops %u total %u fps %u src-fps %u",
+ server_state->num_frames_dropped,
+ num_frames_total,
+ encoder->rate_control.fps,
+ encoder->cbs.get_source_fps(encoder->cbs_opaque));
+
+ if (drop_factor > MJPEG_SERVER_STATUS_DOWNGRADE_DROP_FACTOR_TH) {
+ mjpeg_encoder_decrease_bit_rate(encoder);
+ }
+ server_state->num_frames_encoded = 0;
+ server_state->num_frames_dropped = 0;
+}
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index cc49edf..bc7f01c 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -84,4 +84,14 @@ void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
uint32_t end_frame_mm_time,
int32_t end_frame_delay,
uint32_t audio_delay);
+
+/*
+ * Notify the encoder each time a frame is dropped due to pipe
+ * congestion.
+ * We can deduce the client state by the frame dropping rate in the server.
+ * Monitoring the frame drops can help in fine tuning the playback parameters
+ * when the client reports are delayed.
+ */
+void mjpeg_encoder_notify_server_frame_drop(MJpegEncoder *encoder);
+
#endif
commit b490635130c87e418a3b35c0d7a1335d4377e975
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Fri Feb 15 15:40:20 2013 -0500
mjpeg_encoder: adjust the stream bit rate based on periodic client feedback
mjpeg_encoder can receive periodic reports about the playback status on
the client side. Then, mjpeg_encoder analyses the report and can
increase or decrease the stream bit rate, depending on the report.
When the bit rate is changed, the quality and frame rate of the stream
are re-evaluated.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index 28c7e69..fd0bae3 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -37,6 +37,17 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
#define MJPEG_AVERAGE_SIZE_WINDOW 3
+#define MJPEG_BIT_RATE_EVAL_MIN_NUM_FRAMES 3
+#define MJPEG_LOW_FPS_RATE_TH 3
+
+/*
+ * acting on positive client reports only if enough frame mm time
+ * has passed since the last bit rate change and the report.
+ * time
+ */
+#define MJPEG_CLIENT_POSITIVE_REPORT_TIMEOUT 2000
+#define MJPEG_CLIENT_POSITIVE_REPORT_STRICT_TIMEOUT 3000
+
enum {
MJPEG_QUALITY_EVAL_TYPE_SET,
MJPEG_QUALITY_EVAL_TYPE_UPGRADE,
@@ -64,6 +75,22 @@ typedef struct MJpegEncoderQualityEval {
int max_sampled_fps_quality_id;
} MJpegEncoderQualityEval;
+typedef struct MJpegEncoderClientState {
+ int max_video_latency;
+ uint32_t max_audio_latency;
+} MJpegEncoderClientState;
+
+typedef struct MJpegEncoderBitRateInfo {
+ uint64_t change_start_time;
+ uint64_t last_frame_time;
+ uint32_t change_start_mm_time;
+ int was_upgraded;
+
+ /* gathering data about the frames that
+ * were encoded since the last bit rate change*/
+ uint32_t num_enc_frames;
+ uint64_t sum_enc_size;
+} MJpegEncoderBitRateInfo;
/*
* Adjusting the stream jpeg quality and frame rate (fps):
* When during_quality_eval=TRUE, we compress different frames with different
@@ -77,6 +104,8 @@ typedef struct MJpegEncoderQualityEval {
typedef struct MJpegEncoderRateControl {
int during_quality_eval;
MJpegEncoderQualityEval quality_eval_data;
+ MJpegEncoderBitRateInfo bit_rate_info;
+ MJpegEncoderClientState client_state;
uint64_t byte_rate;
int quality_id;
@@ -580,8 +609,10 @@ static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
int width, int height,
- uint8_t **dest, size_t *dest_len)
+ uint8_t **dest, size_t *dest_len,
+ uint32_t frame_mm_time)
{
+ MJpegEncoderBitRateInfo *bit_rate_info;
uint32_t quality;
mjpeg_encoder_adjust_params_to_bit_rate(encoder);
@@ -632,6 +663,23 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
spice_jpeg_mem_dest(&encoder->cinfo, dest, dest_len);
+ if (!encoder->rate_control.during_quality_eval ||
+ encoder->rate_control.quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE) {
+ struct timespec time;
+ uint64_t now;
+
+ clock_gettime(CLOCK_MONOTONIC, &time);
+ now = ((uint64_t) time.tv_sec) * 1000000000 + time.tv_nsec;
+
+ bit_rate_info = &encoder->rate_control.bit_rate_info;
+
+ if (!bit_rate_info->change_start_time) {
+ bit_rate_info->change_start_time = now;
+ bit_rate_info->change_start_mm_time = frame_mm_time;
+ }
+ bit_rate_info->last_frame_time = now;
+ }
+
encoder->cinfo.image_width = width;
encoder->cinfo.image_height = height;
jpeg_set_defaults(&encoder->cinfo);
@@ -680,13 +728,19 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder)
encoder->first_frame = FALSE;
rate_control->last_enc_size = dest->pub.next_output_byte - dest->buffer;
- if (!rate_control->during_quality_eval) {
- if (rate_control->num_recent_enc_frames >= MJPEG_AVERAGE_SIZE_WINDOW) {
- rate_control->num_recent_enc_frames = 0;
- rate_control->sum_recent_enc_size = 0;
+ if (!rate_control->during_quality_eval ||
+ rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE) {
+
+ if (!rate_control->during_quality_eval) {
+ if (rate_control->num_recent_enc_frames >= MJPEG_AVERAGE_SIZE_WINDOW) {
+ rate_control->num_recent_enc_frames = 0;
+ rate_control->sum_recent_enc_size = 0;
+ }
+ rate_control->sum_recent_enc_size += rate_control->last_enc_size;
+ rate_control->num_recent_enc_frames++;
}
- rate_control->sum_recent_enc_size += rate_control->last_enc_size;
- rate_control->num_recent_enc_frames++;
+ rate_control->bit_rate_info.sum_enc_size += encoder->rate_control.last_enc_size;
+ rate_control->bit_rate_info.num_enc_frames++;
}
return encoder->rate_control.last_enc_size;
}
@@ -698,3 +752,319 @@ uint32_t mjpeg_encoder_get_fps(MJpegEncoder *encoder)
}
return encoder->rate_control.fps;
}
+
+static void mjpeg_encoder_quality_eval_stop(MJpegEncoder *encoder)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ uint32_t quality_id;
+ uint32_t fps;
+
+ if (!rate_control->during_quality_eval) {
+ return;
+ }
+ switch (rate_control->quality_eval_data.type) {
+ case MJPEG_QUALITY_EVAL_TYPE_UPGRADE:
+ quality_id = rate_control->quality_eval_data.min_quality_id;
+ fps = rate_control->quality_eval_data.min_quality_fps;
+ break;
+ case MJPEG_QUALITY_EVAL_TYPE_DOWNGRADE:
+ quality_id = rate_control->quality_eval_data.max_quality_id;
+ fps = rate_control->quality_eval_data.max_quality_fps;
+ break;
+ case MJPEG_QUALITY_EVAL_TYPE_SET:
+ quality_id = MJPEG_QUALITY_SAMPLE_NUM / 2;
+ fps = MJPEG_MAX_FPS / 2;
+ break;
+ default:
+ spice_warning("unexected");
+ return;
+ }
+ mjpeg_encoder_reset_quality(encoder, quality_id, fps, 0);
+ spice_debug("during quality evaluation: canceling."
+ "reset quality to %d fps %d",
+ mjpeg_quality_samples[rate_control->quality_id], rate_control->fps);
+}
+
+static void mjpeg_encoder_decrease_bit_rate(MJpegEncoder *encoder)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ MJpegEncoderBitRateInfo *bit_rate_info = &rate_control->bit_rate_info;
+ uint64_t measured_byte_rate;
+ uint32_t measured_fps;
+ uint64_t decrease_size;
+
+ mjpeg_encoder_quality_eval_stop(encoder);
+
+ rate_control->client_state.max_video_latency = 0;
+ rate_control->client_state.max_audio_latency = 0;
+
+ if (bit_rate_info->num_enc_frames > MJPEG_BIT_RATE_EVAL_MIN_NUM_FRAMES ||
+ bit_rate_info->num_enc_frames > rate_control->fps) {
+ double duration_sec;
+
+ duration_sec = (bit_rate_info->last_frame_time - bit_rate_info->change_start_time);
+ duration_sec /= (1000.0 * 1000.0 * 1000.0);
+ measured_byte_rate = bit_rate_info->sum_enc_size / duration_sec;
+ measured_fps = bit_rate_info->num_enc_frames / duration_sec;
+ decrease_size = bit_rate_info->sum_enc_size / bit_rate_info->num_enc_frames;
+ spice_debug("bit rate esitimation %.2f (Mbps) fps %u",
+ measured_byte_rate*8/1024.0/1024,
+ measured_fps);
+ } else {
+ measured_byte_rate = rate_control->byte_rate;
+ measured_fps = rate_control->fps;
+ decrease_size = measured_byte_rate/measured_fps;
+ spice_debug("bit rate not re-estimated %.2f (Mbps) fps %u",
+ measured_byte_rate*8/1024.0/1024,
+ measured_fps);
+ }
+
+ measured_byte_rate = MIN(rate_control->byte_rate, measured_byte_rate);
+
+ if (decrease_size >= measured_byte_rate) {
+ decrease_size = measured_byte_rate / 2;
+ }
+
+ rate_control->byte_rate = measured_byte_rate - decrease_size;
+ bit_rate_info->change_start_time = 0;
+ bit_rate_info->change_start_mm_time = 0;
+ bit_rate_info->last_frame_time = 0;
+ bit_rate_info->num_enc_frames = 0;
+ bit_rate_info->sum_enc_size = 0;
+ bit_rate_info->was_upgraded = FALSE;
+
+ spice_debug("decrease bit rate %.2f (Mbps)", rate_control->byte_rate * 8 / 1024.0/1024.0);
+ mjpeg_encoder_quality_eval_set_downgrade(encoder,
+ MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE,
+ rate_control->quality_id,
+ rate_control->fps);
+}
+
+static void mjpeg_encoder_handle_negative_client_stream_report(MJpegEncoder *encoder,
+ uint32_t report_end_frame_mm_time)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+
+ spice_debug(NULL);
+
+ if ((rate_control->bit_rate_info.change_start_mm_time > report_end_frame_mm_time ||
+ !rate_control->bit_rate_info.change_start_mm_time) &&
+ !rate_control->bit_rate_info.was_upgraded) {
+ spice_debug("ignoring, a downgrade has already occurred later to the report time");
+ return;
+ }
+
+ mjpeg_encoder_decrease_bit_rate(encoder);
+}
+
+static void mjpeg_encoder_increase_bit_rate(MJpegEncoder *encoder)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ MJpegEncoderBitRateInfo *bit_rate_info = &rate_control->bit_rate_info;
+ uint64_t measured_byte_rate;
+ uint32_t measured_fps;
+ uint64_t increase_size;
+
+
+ if (bit_rate_info->num_enc_frames > MJPEG_BIT_RATE_EVAL_MIN_NUM_FRAMES ||
+ bit_rate_info->num_enc_frames > rate_control->fps) {
+ uint64_t avg_frame_size;
+ double duration_sec;
+
+ duration_sec = (bit_rate_info->last_frame_time - bit_rate_info->change_start_time);
+ duration_sec /= (1000.0 * 1000.0 * 1000.0);
+ measured_byte_rate = bit_rate_info->sum_enc_size / duration_sec;
+ measured_fps = bit_rate_info->num_enc_frames / duration_sec;
+ avg_frame_size = bit_rate_info->sum_enc_size / bit_rate_info->num_enc_frames;
+ spice_debug("bit rate esitimation %.2f (Mbps) defined %.2f"
+ " fps %u avg-frame-size=%.2f (KB)",
+ measured_byte_rate*8/1024.0/1024,
+ rate_control->byte_rate*8/1024.0/1024,
+ measured_fps,
+ avg_frame_size/1024.0);
+ increase_size = avg_frame_size;
+ } else {
+ spice_debug("not enough samples for measuring the bit rate. no change");
+ return;
+ }
+
+
+ mjpeg_encoder_quality_eval_stop(encoder);
+
+ if (measured_byte_rate + increase_size < rate_control->byte_rate) {
+ spice_debug("measured byte rate is small: not upgrading, just re-evaluating");
+ } else {
+ rate_control->byte_rate = MIN(measured_byte_rate, rate_control->byte_rate) + increase_size;
+ }
+
+ bit_rate_info->change_start_time = 0;
+ bit_rate_info->change_start_mm_time = 0;
+ bit_rate_info->last_frame_time = 0;
+ bit_rate_info->num_enc_frames = 0;
+ bit_rate_info->sum_enc_size = 0;
+ bit_rate_info->was_upgraded = TRUE;
+
+ spice_debug("increase bit rate %.2f (Mbps)", rate_control->byte_rate * 8 / 1024.0/1024.0);
+ mjpeg_encoder_quality_eval_set_upgrade(encoder,
+ MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE,
+ rate_control->quality_id,
+ rate_control->fps);
+}
+static void mjpeg_encoder_handle_positive_client_stream_report(MJpegEncoder *encoder,
+ uint32_t report_start_frame_mm_time)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ MJpegEncoderBitRateInfo *bit_rate_info = &rate_control->bit_rate_info;
+ int stable_client_mm_time;
+ int timeout;
+
+ if (rate_control->during_quality_eval &&
+ rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE) {
+ spice_debug("during quality evaluation (rate change). ignoring report");
+ return;
+ }
+
+ if ((rate_control->fps > MJPEG_IMPROVE_QUALITY_FPS_STRICT_TH ||
+ rate_control->fps >= encoder->cbs.get_source_fps(encoder->cbs_opaque)) &&
+ rate_control->quality_id > MJPEG_QUALITY_SAMPLE_NUM / 2) {
+ timeout = MJPEG_CLIENT_POSITIVE_REPORT_STRICT_TIMEOUT;
+ } else {
+ timeout = MJPEG_CLIENT_POSITIVE_REPORT_TIMEOUT;
+ }
+
+ stable_client_mm_time = (int)report_start_frame_mm_time - bit_rate_info->change_start_mm_time;
+
+ if (!bit_rate_info->change_start_mm_time || stable_client_mm_time < timeout) {
+ /* assessing the stability of the current setting and only then
+ * respond to the report */
+ spice_debug("no drops, but not enough time has passed for assessing"
+ "the playback stability since the last bit rate change");
+ return;
+ }
+ mjpeg_encoder_increase_bit_rate(encoder);
+}
+
+/*
+ * the video playback jitter buffer should be at least (send_time*2 + net_latency) for
+ * preventing underflow
+ */
+static uint32_t get_min_required_playback_delay(uint64_t frame_enc_size,
+ uint64_t byte_rate,
+ uint32_t latency)
+{
+ uint32_t one_frame_time;
+
+ if (!frame_enc_size || !byte_rate) {
+ return latency;
+ }
+ one_frame_time = (frame_enc_size*1000)/byte_rate;
+
+ return one_frame_time*2 + latency;
+}
+
+#define MJPEG_PLAYBACK_LATENCY_DECREASE_FACTOR 0.5
+#define MJPEG_VIDEO_VS_AUDIO_LATENCY_FACTOR 1.25
+#define MJPEG_VIDEO_DELAY_TH -15
+
+void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
+ uint32_t num_frames,
+ uint32_t num_drops,
+ uint32_t start_frame_mm_time,
+ uint32_t end_frame_mm_time,
+ int32_t end_frame_delay,
+ uint32_t audio_delay)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+ MJpegEncoderClientState *client_state = &rate_control->client_state;
+ uint64_t avg_enc_size = 0;
+ uint32_t min_playback_delay;
+
+ spice_debug("client report: #frames %u, #drops %d, duration %u video-delay %d audio-delay %u",
+ num_frames, num_drops,
+ end_frame_mm_time - start_frame_mm_time,
+ end_frame_delay, audio_delay);
+
+ if (!encoder->rate_control_is_active) {
+ spice_debug("rate control was not activated: ignoring");
+ return;
+ }
+ if (rate_control->during_quality_eval) {
+ if (rate_control->quality_eval_data.type == MJPEG_QUALITY_EVAL_TYPE_DOWNGRADE &&
+ rate_control->quality_eval_data.reason == MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE) {
+ spice_debug("during rate downgrade evaluation");
+ return;
+ }
+ }
+
+ if (rate_control->num_recent_enc_frames) {
+ avg_enc_size = rate_control->sum_recent_enc_size /
+ rate_control->num_recent_enc_frames;
+ }
+ spice_debug("recent size avg %.2f (KB)", avg_enc_size / 1024.0);
+ min_playback_delay = get_min_required_playback_delay(avg_enc_size, rate_control->byte_rate,
+ mjpeg_encoder_get_latency(encoder));
+ spice_debug("min-delay %u client-delay %d", min_playback_delay, end_frame_delay);
+
+ /*
+ * If the audio latency has decreased (since the start of the current
+ * sequence of positive reports), and the video latency is bigger, slow down
+ * the video rate
+ */
+ if (end_frame_delay > 0 &&
+ audio_delay < MJPEG_PLAYBACK_LATENCY_DECREASE_FACTOR*client_state->max_audio_latency &&
+ end_frame_delay > MJPEG_VIDEO_VS_AUDIO_LATENCY_FACTOR*audio_delay) {
+ spice_debug("video_latency >> audio_latency && audio_latency << max (%u)",
+ client_state->max_audio_latency);
+ mjpeg_encoder_handle_negative_client_stream_report(encoder,
+ end_frame_mm_time);
+ return;
+ }
+
+ if (end_frame_delay < MJPEG_VIDEO_DELAY_TH) {
+ mjpeg_encoder_handle_negative_client_stream_report(encoder,
+ end_frame_mm_time);
+ } else {
+ int is_video_delay_small = FALSE;
+ double major_delay_decrease_thresh;
+ double medium_delay_decrease_thresh;
+
+ client_state->max_video_latency = MAX(end_frame_delay, client_state->max_video_latency);
+ client_state->max_audio_latency = MAX(audio_delay, client_state->max_audio_latency);
+
+ if (min_playback_delay > end_frame_delay) {
+ uint32_t src_fps = encoder->cbs.get_source_fps(encoder->cbs_opaque);
+ /*
+ * if the stream is at its highest rate, we can't estimate the "real"
+ * network bit rate and the min_playback_delay
+ */
+ if (rate_control->quality_id != MJPEG_QUALITY_SAMPLE_NUM - 1 ||
+ rate_control->fps < MIN(src_fps, MJPEG_MAX_FPS)) {
+ is_video_delay_small = TRUE;
+ }
+ }
+
+ medium_delay_decrease_thresh = client_state->max_video_latency;
+ medium_delay_decrease_thresh *= MJPEG_PLAYBACK_LATENCY_DECREASE_FACTOR;
+
+ major_delay_decrease_thresh = medium_delay_decrease_thresh;
+ major_delay_decrease_thresh *= MJPEG_PLAYBACK_LATENCY_DECREASE_FACTOR;
+ /*
+ * since the bit rate and the required latency are only evaluation based on the
+ * reports we got till now, we assume that the latency is too low only if it
+ * was higher during the time that passed since the last report that resulted
+ * in a bit rate decrement. If we find that the latency has decreased, it might
+ * suggest that the stream bit rate is too high.
+ */
+ if ((end_frame_delay < medium_delay_decrease_thresh &&
+ is_video_delay_small) || end_frame_delay < major_delay_decrease_thresh) {
+ spice_debug("downgrade due to short video delay (last=%u, past-max=%u",
+ end_frame_delay, client_state->max_video_latency);
+ mjpeg_encoder_handle_negative_client_stream_report(encoder,
+ end_frame_mm_time);
+ } else if (!num_drops) {
+ mjpeg_encoder_handle_positive_client_stream_report(encoder,
+ start_frame_mm_time);
+
+ }
+ }
+}
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index 902dcbe..cc49edf 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -48,7 +48,8 @@ uint8_t mjpeg_encoder_get_bytes_per_pixel(MJpegEncoder *encoder);
*/
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
int width, int height,
- uint8_t **dest, size_t *dest_len);
+ uint8_t **dest, size_t *dest_len,
+ uint32_t frame_mm_time);
int mjpeg_encoder_encode_scanline(MJpegEncoder *encoder, uint8_t *src_pixels,
size_t image_width);
size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder);
@@ -63,5 +64,24 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder);
*/
uint32_t mjpeg_encoder_get_fps(MJpegEncoder *encoder);
-
+/*
+ * Data that should be periodically obtained from the client. The report contains:
+ * num_frames : the number of frames that reached the client during the time
+ * the report is referring to.
+ * num_drops : the part of the above frames that was dropped by the client due to
+ * late arrival time.
+ * start_frame_mm_time: the mm_time of the first frame included in the report
+ * end_frame_mm_time : the mm_time of the last_frame included in the report
+ * end_frame_delay : (end_frame_mm_time - client_mm_time)
+ * audio delay : the latency of the audio playback.
+ * If there is no audio playback, set it to MAX_UINT.
+ *
+ */
+void mjpeg_encoder_client_stream_report(MJpegEncoder *encoder,
+ uint32_t num_frames,
+ uint32_t num_drops,
+ uint32_t start_frame_mm_time,
+ uint32_t end_frame_mm_time,
+ int32_t end_frame_delay,
+ uint32_t audio_delay);
#endif
diff --git a/server/red_worker.c b/server/red_worker.c
index 23d08a8..b453023 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -8392,7 +8392,8 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
if (!mjpeg_encoder_start_frame(agent->mjpeg_encoder, image->u.bitmap.format,
width, height,
&dcc->send_data.stream_outbuf,
- &outbuf_size)) {
+ &outbuf_size,
+ drawable->red_drawable->mm_time)) {
return FALSE;
}
if (!encode_frame(dcc, &drawable->red_drawable->u.copy.src_area,
commit 2025494af50d14ff92ef9a67787c9c46a4b23c33
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Fri Feb 15 09:15:51 2013 -0500
mjpeg_encoder: re-configure stream parameters when the frame's encoding size changes
If the encoding size seems to get smaller/bigger, re-evaluate the
stream quality and frame rate.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index 00b721d..28c7e69 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -35,7 +35,23 @@ static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40,
#define MJPEG_IMPROVE_QUALITY_FPS_STRICT_TH 10
#define MJPEG_IMPROVE_QUALITY_FPS_PERMISSIVE_TH 5
+#define MJPEG_AVERAGE_SIZE_WINDOW 3
+
+enum {
+ MJPEG_QUALITY_EVAL_TYPE_SET,
+ MJPEG_QUALITY_EVAL_TYPE_UPGRADE,
+ MJPEG_QUALITY_EVAL_TYPE_DOWNGRADE,
+};
+
+enum {
+ MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE,
+ MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE,
+};
+
typedef struct MJpegEncoderQualityEval {
+ int type;
+ int reason;
+
uint64_t encoded_size_by_quality[MJPEG_QUALITY_SAMPLE_NUM];
/* lower limit for the current evaluation round */
int min_quality_id;
@@ -52,7 +68,7 @@ typedef struct MJpegEncoderQualityEval {
* Adjusting the stream jpeg quality and frame rate (fps):
* When during_quality_eval=TRUE, we compress different frames with different
* jpeg quality. By considering (1) the resulting compression ratio, and (2) the available
- * bit rate, we evaulate the max frame frequency for the stream with the given quality,
+ * bit rate, we evaluate the max frame frequency for the stream with the given quality,
* and we choose the highest quality that will allow a reasonable frame rate.
* during_quality_eval is set for new streams and can also be set any time we want
* to re-evaluate the stream parameters (e.g., when the bit rate and/or
@@ -65,8 +81,13 @@ typedef struct MJpegEncoderRateControl {
uint64_t byte_rate;
int quality_id;
uint32_t fps;
+ /* the encoded frame size which the quality and the fps evaluation was based upon */
+ uint64_t base_enc_size;
uint64_t last_enc_size;
+
+ uint64_t sum_recent_enc_size;
+ uint32_t num_recent_enc_frames;
} MJpegEncoderRateControl;
struct MJpegEncoder {
@@ -86,7 +107,10 @@ struct MJpegEncoder {
void *cbs_opaque;
};
-static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder, int quality_id, uint32_t fps);
+static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
+ int quality_id,
+ uint32_t fps,
+ uint64_t frame_enc_size);
static uint32_t get_max_fps(uint64_t frame_size, uint64_t bytes_per_sec);
MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate,
@@ -104,10 +128,12 @@ MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate
if (bit_rate_control) {
enc->cbs = *cbs;
enc->cbs_opaque = opaque;
- mjpeg_encoder_reset_quality(enc, MJPEG_QUALITY_SAMPLE_NUM / 2, 5);
+ mjpeg_encoder_reset_quality(enc, MJPEG_QUALITY_SAMPLE_NUM / 2, 5, 0);
enc->rate_control.during_quality_eval = TRUE;
+ enc->rate_control.quality_eval_data.type = MJPEG_QUALITY_EVAL_TYPE_SET;
+ enc->rate_control.quality_eval_data.reason = MJPEG_QUALITY_EVAL_REASON_RATE_CHANGE;
} else {
- mjpeg_encoder_reset_quality(enc, MJPEG_LEGACY_STATIC_QUALITY_ID, MJPEG_MAX_FPS);
+ mjpeg_encoder_reset_quality(enc, MJPEG_LEGACY_STATIC_QUALITY_ID, MJPEG_MAX_FPS, 0);
}
enc->cinfo.err = jpeg_std_error(&enc->jerr);
@@ -275,7 +301,10 @@ static uint32_t get_max_fps(uint64_t frame_size, uint64_t bytes_per_sec)
return fps;
}
-static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder, int quality_id, uint32_t fps)
+static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder,
+ int quality_id,
+ uint32_t fps,
+ uint64_t frame_enc_size)
{
MJpegEncoderRateControl *rate_control = &encoder->rate_control;
@@ -284,12 +313,18 @@ static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder, int qualit
if (rate_control->quality_id != quality_id) {
rate_control->last_enc_size = 0;
}
+
rate_control->quality_id = quality_id;
memset(&rate_control->quality_eval_data, 0, sizeof(MJpegEncoderQualityEval));
rate_control->quality_eval_data.max_quality_id = MJPEG_QUALITY_SAMPLE_NUM - 1;
rate_control->quality_eval_data.max_quality_fps = MJPEG_MAX_FPS;
+
rate_control->fps = MAX(MJPEG_MIN_FPS, fps);
rate_control->fps = MIN(MJPEG_MAX_FPS, rate_control->fps);
+ rate_control->base_enc_size = frame_enc_size;
+
+ rate_control->sum_recent_enc_size = 0;
+ rate_control->num_recent_enc_frames = 0;
}
#define QUALITY_WAS_EVALUATED(encoder, quality) \
@@ -428,21 +463,57 @@ complete_sample:
if (final_quality_id == quality_eval->max_quality_id) {
final_fps = MIN(final_fps, quality_eval->max_quality_fps);
}
- mjpeg_encoder_reset_quality(encoder, final_quality_id, final_fps);
+ mjpeg_encoder_reset_quality(encoder, final_quality_id, final_fps, final_quality_enc_size);
+ rate_control->sum_recent_enc_size = final_quality_enc_size;
+ rate_control->num_recent_enc_frames = 1;
spice_debug("MJpeg quality sample end %p: quality %d fps %d",
encoder, mjpeg_quality_samples[rate_control->quality_id], rate_control->fps);
}
+static void mjpeg_encoder_quality_eval_set_upgrade(MJpegEncoder *encoder,
+ int reason,
+ uint32_t min_quality_id,
+ uint32_t min_quality_fps)
+{
+ MJpegEncoderQualityEval *quality_eval = &encoder->rate_control.quality_eval_data;
+
+ encoder->rate_control.during_quality_eval = TRUE;
+ quality_eval->type = MJPEG_QUALITY_EVAL_TYPE_UPGRADE;
+ quality_eval->reason = reason;
+ quality_eval->min_quality_id = min_quality_id;
+ quality_eval->min_quality_fps = min_quality_fps;
+}
+
+static void mjpeg_encoder_quality_eval_set_downgrade(MJpegEncoder *encoder,
+ int reason,
+ uint32_t max_quality_id,
+ uint32_t max_quality_fps)
+{
+ MJpegEncoderQualityEval *quality_eval = &encoder->rate_control.quality_eval_data;
+
+ encoder->rate_control.during_quality_eval = TRUE;
+ quality_eval->type = MJPEG_QUALITY_EVAL_TYPE_DOWNGRADE;
+ quality_eval->reason = reason;
+ quality_eval->max_quality_id = max_quality_id;
+ quality_eval->max_quality_fps = max_quality_fps;
+}
+
static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
{
MJpegEncoderRateControl *rate_control;
+ MJpegEncoderQualityEval *quality_eval;
+ uint64_t new_avg_enc_size;
+ uint32_t new_fps;
+ uint32_t latency = 0;
+ uint32_t src_fps;
if (!encoder->rate_control_is_active) {
return;
}
rate_control = &encoder->rate_control;
+ quality_eval = &rate_control->quality_eval_data;
if (!rate_control->last_enc_size) {
spice_debug("missing sample size");
@@ -450,9 +521,60 @@ static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
}
if (rate_control->during_quality_eval) {
- MJpegEncoderQualityEval *quality_eval = &rate_control->quality_eval_data;
quality_eval->encoded_size_by_quality[rate_control->quality_id] = rate_control->last_enc_size;
mjpeg_encoder_eval_quality(encoder);
+ return;
+ }
+
+ spice_assert(rate_control->num_recent_enc_frames);
+
+ if (rate_control->num_recent_enc_frames < MJPEG_AVERAGE_SIZE_WINDOW &&
+ rate_control->num_recent_enc_frames < rate_control->fps) {
+ return;
+ }
+
+ latency = mjpeg_encoder_get_latency(encoder);
+ new_avg_enc_size = rate_control->sum_recent_enc_size /
+ rate_control->num_recent_enc_frames;
+ new_fps = get_max_fps(new_avg_enc_size, rate_control->byte_rate);
+
+ spice_debug("cur-fps=%u new-fps=%u (new/old=%.2f) |"
+ "bit-rate=%.2f (Mbps) latency=%u (ms) quality=%d |"
+ " new-size-avg %lu , base-size %lu, (new/old=%.2f) ",
+ rate_control->fps, new_fps, ((double)new_fps)/rate_control->fps,
+ ((double)rate_control->byte_rate*8)/1024/1024,
+ latency,
+ mjpeg_quality_samples[rate_control->quality_id],
+ new_avg_enc_size, rate_control->base_enc_size,
+ rate_control->base_enc_size ?
+ ((double)new_avg_enc_size) / rate_control->base_enc_size :
+ 1);
+
+ src_fps = encoder->cbs.get_source_fps(encoder->cbs_opaque);
+
+ /*
+ * The ratio between the new_fps and the current fps reflects the changes
+ * in latency and frame size. When the change passes a threshold,
+ * we re-evaluate the quality and frame rate.
+ */
+ if (new_fps > rate_control->fps &&
+ (rate_control->fps < src_fps || rate_control->quality_id < MJPEG_QUALITY_SAMPLE_NUM - 1)) {
+ spice_debug("mjpeg %p FPS CHANGE >> : re-evaluating params", encoder);
+ mjpeg_encoder_quality_eval_set_upgrade(encoder, MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE,
+ rate_control->quality_id, /* fps has improved -->
+ don't allow stream quality
+ to deteriorate */
+ rate_control->fps);
+ } else if (new_fps < rate_control->fps && new_fps < src_fps) {
+ spice_debug("mjpeg %p FPS CHANGE << : re-evaluating params", encoder);
+ mjpeg_encoder_quality_eval_set_downgrade(encoder, MJPEG_QUALITY_EVAL_REASON_SIZE_CHANGE,
+ rate_control->quality_id,
+ rate_control->fps);
+ }
+
+ if (rate_control->during_quality_eval) {
+ quality_eval->encoded_size_by_quality[rate_control->quality_id] = new_avg_enc_size;
+ mjpeg_encoder_eval_quality(encoder);
}
}
@@ -551,12 +673,21 @@ int mjpeg_encoder_encode_scanline(MJpegEncoder *encoder, uint8_t *src_pixels,
size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder)
{
mem_destination_mgr *dest = (mem_destination_mgr *) encoder->cinfo.dest;
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
jpeg_finish_compress(&encoder->cinfo);
encoder->first_frame = FALSE;
- encoder->rate_control.last_enc_size = dest->pub.next_output_byte - dest->buffer;
+ rate_control->last_enc_size = dest->pub.next_output_byte - dest->buffer;
+ if (!rate_control->during_quality_eval) {
+ if (rate_control->num_recent_enc_frames >= MJPEG_AVERAGE_SIZE_WINDOW) {
+ rate_control->num_recent_enc_frames = 0;
+ rate_control->sum_recent_enc_size = 0;
+ }
+ rate_control->sum_recent_enc_size += rate_control->last_enc_size;
+ rate_control->num_recent_enc_frames++;
+ }
return encoder->rate_control.last_enc_size;
}
commit f68b539d70b914455a0b280260786ffc530988d4
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Thu Feb 14 17:23:51 2013 -0500
mjpeg_encoder: configure mjpeg quality and frame rate according to a given bit rate
Previously, the mjpeg quality was always 70. The frame rate was
tuned according to the frames' congestion in the pipe.
This patch sets the quality and frame rate according to
a given bit rate and the size of the first encoded frames.
The following patches will introduce an adaptive video streaming, in which
the bit rate, the quality, and the frame rate, change in response to
different parameters.
Patches that make red_worker adopt this feature will also follow.
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index b812ba0..00b721d 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -24,27 +24,92 @@
#include <jerror.h>
#include <jpeglib.h>
+#define MJPEG_MAX_FPS 25
+#define MJPEG_MIN_FPS 1
+
+#define MJPEG_QUALITY_SAMPLE_NUM 7
+static const int mjpeg_quality_samples[MJPEG_QUALITY_SAMPLE_NUM] = {20, 30, 40, 50, 60, 70, 80};
+
+#define MJPEG_LEGACY_STATIC_QUALITY_ID 5 // jpeg quality 70
+
+#define MJPEG_IMPROVE_QUALITY_FPS_STRICT_TH 10
+#define MJPEG_IMPROVE_QUALITY_FPS_PERMISSIVE_TH 5
+
+typedef struct MJpegEncoderQualityEval {
+ uint64_t encoded_size_by_quality[MJPEG_QUALITY_SAMPLE_NUM];
+ /* lower limit for the current evaluation round */
+ int min_quality_id;
+ int min_quality_fps; // min fps for the given quality
+ /* upper limit for the current evaluation round */
+ int max_quality_id;
+ int max_quality_fps; // max fps for the given quality
+ /* tracking the best sampled fps so far */
+ int max_sampled_fps;
+ int max_sampled_fps_quality_id;
+} MJpegEncoderQualityEval;
+
+/*
+ * Adjusting the stream jpeg quality and frame rate (fps):
+ * When during_quality_eval=TRUE, we compress different frames with different
+ * jpeg quality. By considering (1) the resulting compression ratio, and (2) the available
+ * bit rate, we evaulate the max frame frequency for the stream with the given quality,
+ * and we choose the highest quality that will allow a reasonable frame rate.
+ * during_quality_eval is set for new streams and can also be set any time we want
+ * to re-evaluate the stream parameters (e.g., when the bit rate and/or
+ * compressed frame size significantly change).
+ */
+typedef struct MJpegEncoderRateControl {
+ int during_quality_eval;
+ MJpegEncoderQualityEval quality_eval_data;
+
+ uint64_t byte_rate;
+ int quality_id;
+ uint32_t fps;
+
+ uint64_t last_enc_size;
+} MJpegEncoderRateControl;
+
struct MJpegEncoder {
uint8_t *row;
uint32_t row_size;
int first_frame;
- int quality;
struct jpeg_compress_struct cinfo;
struct jpeg_error_mgr jerr;
unsigned int bytes_per_pixel; /* bytes per pixel of the input buffer */
void (*pixel_converter)(uint8_t *src, uint8_t *dest);
+
+ int rate_control_is_active;
+ MJpegEncoderRateControl rate_control;
+ MJpegEncoderRateControlCbs cbs;
+ void *cbs_opaque;
};
-MJpegEncoder *mjpeg_encoder_new(void)
+static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder, int quality_id, uint32_t fps);
+static uint32_t get_max_fps(uint64_t frame_size, uint64_t bytes_per_sec);
+
+MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate,
+ MJpegEncoderRateControlCbs *cbs, void *opaque)
{
MJpegEncoder *enc;
+ spice_assert(!bit_rate_control || (cbs && cbs->get_roundtrip_ms && cbs->get_source_fps));
+
enc = spice_new0(MJpegEncoder, 1);
enc->first_frame = TRUE;
- enc->quality = 70;
+ enc->rate_control_is_active = bit_rate_control;
+ enc->rate_control.byte_rate = starting_bit_rate / 8;
+ if (bit_rate_control) {
+ enc->cbs = *cbs;
+ enc->cbs_opaque = opaque;
+ mjpeg_encoder_reset_quality(enc, MJPEG_QUALITY_SAMPLE_NUM / 2, 5);
+ enc->rate_control.during_quality_eval = TRUE;
+ } else {
+ mjpeg_encoder_reset_quality(enc, MJPEG_LEGACY_STATIC_QUALITY_ID, MJPEG_MAX_FPS);
+ }
+
enc->cinfo.err = jpeg_std_error(&enc->jerr);
jpeg_create_compress(&enc->cinfo);
@@ -191,10 +256,214 @@ spice_jpeg_mem_dest(j_compress_ptr cinfo,
}
/* end of code from libjpeg */
+static inline uint32_t mjpeg_encoder_get_latency(MJpegEncoder *encoder)
+{
+ return encoder->cbs.get_roundtrip_ms ?
+ encoder->cbs.get_roundtrip_ms(encoder->cbs_opaque) / 2 : 0;
+}
+
+static uint32_t get_max_fps(uint64_t frame_size, uint64_t bytes_per_sec)
+{
+ double fps;
+ double send_time_ms;
+
+ if (!bytes_per_sec) {
+ return 0;
+ }
+ send_time_ms = frame_size * 1000.0 / bytes_per_sec;
+ fps = send_time_ms ? 1000 / send_time_ms : MJPEG_MAX_FPS;
+ return fps;
+}
+
+static inline void mjpeg_encoder_reset_quality(MJpegEncoder *encoder, int quality_id, uint32_t fps)
+{
+ MJpegEncoderRateControl *rate_control = &encoder->rate_control;
+
+ rate_control->during_quality_eval = FALSE;
+
+ if (rate_control->quality_id != quality_id) {
+ rate_control->last_enc_size = 0;
+ }
+ rate_control->quality_id = quality_id;
+ memset(&rate_control->quality_eval_data, 0, sizeof(MJpegEncoderQualityEval));
+ rate_control->quality_eval_data.max_quality_id = MJPEG_QUALITY_SAMPLE_NUM - 1;
+ rate_control->quality_eval_data.max_quality_fps = MJPEG_MAX_FPS;
+ rate_control->fps = MAX(MJPEG_MIN_FPS, fps);
+ rate_control->fps = MIN(MJPEG_MAX_FPS, rate_control->fps);
+}
+
+#define QUALITY_WAS_EVALUATED(encoder, quality) \
+ ((encoder)->rate_control.quality_eval_data.encoded_size_by_quality[(quality)] != 0)
+
+/*
+ * Adjust the stream's jpeg quality and frame rate.
+ * We evaluate the compression ratio of different jpeg qualities;
+ * We compress successive frames with different qualities,
+ * and then we estimate the stream frame rate according to the currently
+ * evaluated jpeg quality and available bit rate.
+ *
+ * During quality evaluation, mjpeg_encoder_eval_quality is called before a new
+ * frame is encoded. mjpeg_encoder_eval_quality examines the encoding size of
+ * the previously encoded frame, and determines whether to continue evaluation
+ * (and chnages the quality for the frame that is going to be encoded),
+ * or stop evaluation (and sets the quality and frame rate for the stream).
+ * When qualities are scanned, we assume monotonicity of compression ratio
+ * as a function of jpeg quality. When we reach a quality with too small, or
+ * big enough compression ratio, we stop the evaluation and set the stream parameters.
+*/
+static inline void mjpeg_encoder_eval_quality(MJpegEncoder *encoder)
+{
+ MJpegEncoderRateControl *rate_control;
+ MJpegEncoderQualityEval *quality_eval;
+ uint32_t fps, src_fps;
+ uint64_t enc_size;
+ uint32_t final_quality_id;
+ uint32_t final_fps;
+ uint64_t final_quality_enc_size;
+
+ rate_control = &encoder->rate_control;
+ quality_eval = &rate_control->quality_eval_data;
+
+ spice_assert(rate_control->during_quality_eval);
+
+ /* retrieving the encoded size of the last encoded frame */
+ enc_size = quality_eval->encoded_size_by_quality[rate_control->quality_id];
+ if (enc_size == 0) {
+ spice_debug("size info missing");
+ return;
+ }
+
+ src_fps = encoder->cbs.get_source_fps(encoder->cbs_opaque);
+
+ fps = get_max_fps(enc_size, rate_control->byte_rate);
+ spice_debug("mjpeg %p: jpeg %d: %.2f (KB) fps %d src-fps %u",
+ encoder,
+ mjpeg_quality_samples[rate_control->quality_id],
+ enc_size / 1024.0,
+ fps,
+ src_fps);
+
+ if (fps > quality_eval->max_sampled_fps ||
+ ((fps == quality_eval->max_sampled_fps || fps >= src_fps) &&
+ rate_control->quality_id > quality_eval->max_sampled_fps_quality_id)) {
+ quality_eval->max_sampled_fps = fps;
+ quality_eval->max_sampled_fps_quality_id = rate_control->quality_id;
+ }
+
+ /*
+ * Choosing whether to evaluate another quality, or to complete evaluation
+ * and set the stream parameters according to one of the qualities that
+ * were already sampled.
+ */
+
+ if (rate_control->quality_id > MJPEG_QUALITY_SAMPLE_NUM / 2 &&
+ fps < MJPEG_IMPROVE_QUALITY_FPS_STRICT_TH &&
+ fps < src_fps) {
+ /*
+ * When the jpeg quality is bigger than the median quality, prefer a reasonable
+ * frame rate over improving the quality
+ */
+ spice_debug("fps < %d && (fps < src_fps), quality %d",
+ MJPEG_IMPROVE_QUALITY_FPS_STRICT_TH,
+ mjpeg_quality_samples[rate_control->quality_id]);
+ if (QUALITY_WAS_EVALUATED(encoder, rate_control->quality_id - 1)) {
+ /* the next worse quality was already evaluated and it passed the frame
+ * rate thresholds (we know that, because we continued evaluating a better
+ * quality) */
+ rate_control->quality_id--;
+ goto complete_sample;
+ } else {
+ /* evaluate the next worse quality */
+ rate_control->quality_id--;
+ }
+ } else if ((fps > MJPEG_IMPROVE_QUALITY_FPS_PERMISSIVE_TH &&
+ fps >= 0.66 * quality_eval->min_quality_fps) || fps >= src_fps) {
+ /* When the jpeg quality is worse than the median one (see first condition), we allow a less
+ strict threshold for fps, in order to improve the jpeg quality */
+ if (rate_control->quality_id + 1 == MJPEG_QUALITY_SAMPLE_NUM ||
+ rate_control->quality_id >= quality_eval->max_quality_id ||
+ QUALITY_WAS_EVALUATED(encoder, rate_control->quality_id + 1)) {
+ /* best quality has been reached, or the next (better) quality was
+ * already evaluated and didn't pass the fps thresholds */
+ goto complete_sample;
+ } else {
+ if (rate_control->quality_id == MJPEG_QUALITY_SAMPLE_NUM / 2 &&
+ fps < MJPEG_IMPROVE_QUALITY_FPS_STRICT_TH &&
+ fps < src_fps) {
+ goto complete_sample;
+ }
+ /* evaluate the next quality as well*/
+ rate_control->quality_id++;
+ }
+ } else { // very small frame rate, try to improve by downgrading the quality
+ if (rate_control->quality_id == 0 ||
+ rate_control->quality_id <= quality_eval->min_quality_id) {
+ goto complete_sample;
+ } else if (QUALITY_WAS_EVALUATED(encoder, rate_control->quality_id - 1)) {
+ rate_control->quality_id--;
+ goto complete_sample;
+ } else {
+ /* evaluate the next worse quality */
+ rate_control->quality_id--;
+ }
+ }
+ return;
+
+complete_sample:
+ if (quality_eval->max_sampled_fps != 0) {
+ /* covering a case were monotonicity was violated and we sampled
+ a better jepg quality, with better frame rate. */
+ final_quality_id = MAX(rate_control->quality_id,
+ quality_eval->max_sampled_fps_quality_id);
+ } else {
+ final_quality_id = rate_control->quality_id;
+ }
+ final_quality_enc_size = quality_eval->encoded_size_by_quality[final_quality_id];
+ final_fps = get_max_fps(final_quality_enc_size,
+ rate_control->byte_rate);
+
+ if (final_quality_id == quality_eval->min_quality_id) {
+ final_fps = MAX(final_fps, quality_eval->min_quality_fps);
+ }
+ if (final_quality_id == quality_eval->max_quality_id) {
+ final_fps = MIN(final_fps, quality_eval->max_quality_fps);
+ }
+ mjpeg_encoder_reset_quality(encoder, final_quality_id, final_fps);
+
+ spice_debug("MJpeg quality sample end %p: quality %d fps %d",
+ encoder, mjpeg_quality_samples[rate_control->quality_id], rate_control->fps);
+}
+
+static void mjpeg_encoder_adjust_params_to_bit_rate(MJpegEncoder *encoder)
+{
+ MJpegEncoderRateControl *rate_control;
+
+ if (!encoder->rate_control_is_active) {
+ return;
+ }
+
+ rate_control = &encoder->rate_control;
+
+ if (!rate_control->last_enc_size) {
+ spice_debug("missing sample size");
+ return;
+ }
+
+ if (rate_control->during_quality_eval) {
+ MJpegEncoderQualityEval *quality_eval = &rate_control->quality_eval_data;
+ quality_eval->encoded_size_by_quality[rate_control->quality_id] = rate_control->last_enc_size;
+ mjpeg_encoder_eval_quality(encoder);
+ }
+}
+
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
int width, int height,
uint8_t **dest, size_t *dest_len)
{
+ uint32_t quality;
+
+ mjpeg_encoder_adjust_params_to_bit_rate(encoder);
+
encoder->cinfo.in_color_space = JCS_RGB;
encoder->cinfo.input_components = 3;
encoder->pixel_converter = NULL;
@@ -245,7 +514,8 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
encoder->cinfo.image_height = height;
jpeg_set_defaults(&encoder->cinfo);
encoder->cinfo.dct_method = JDCT_IFAST;
- jpeg_set_quality(&encoder->cinfo, encoder->quality, TRUE);
+ quality = mjpeg_quality_samples[encoder->rate_control.quality_id];
+ jpeg_set_quality(&encoder->cinfo, quality, TRUE);
jpeg_start_compress(&encoder->cinfo, encoder->first_frame);
return TRUE;
@@ -271,6 +541,7 @@ int mjpeg_encoder_encode_scanline(MJpegEncoder *encoder, uint8_t *src_pixels,
}
if (scanlines_written == 0) { /* Not enough space */
jpeg_abort_compress(&encoder->cinfo);
+ encoder->rate_control.last_enc_size = 0;
return 0;
}
@@ -284,5 +555,15 @@ size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder)
jpeg_finish_compress(&encoder->cinfo);
encoder->first_frame = FALSE;
- return dest->pub.next_output_byte - dest->buffer;
+ encoder->rate_control.last_enc_size = dest->pub.next_output_byte - dest->buffer;
+
+ return encoder->rate_control.last_enc_size;
+}
+
+uint32_t mjpeg_encoder_get_fps(MJpegEncoder *encoder)
+{
+ if (!encoder->rate_control_is_active) {
+ spice_warning("bit rate control is not active");
+ }
+ return encoder->rate_control.fps;
}
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index b9a2ed7..902dcbe 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -23,7 +23,21 @@
typedef struct MJpegEncoder MJpegEncoder;
-MJpegEncoder *mjpeg_encoder_new(void);
+/*
+ * Callbacks required for controling and adjusting
+ * the stream bit rate:
+ * get_roundtrip_ms: roundtrip time in milliseconds
+ * get_source_fps: the input frame rate (#frames per second), i.e.,
+ * the rate of frames arriving from the guest to spice-server,
+ * before any drops.
+ */
+typedef struct MJpegEncoderRateControlCbs {
+ uint32_t (*get_roundtrip_ms)(void *opaque);
+ uint32_t (*get_source_fps)(void *opaque);
+} MJpegEncoderRateControlCbs;
+
+MJpegEncoder *mjpeg_encoder_new(int bit_rate_control, uint64_t starting_bit_rate,
+ MJpegEncoderRateControlCbs *cbs, void *opaque);
void mjpeg_encoder_destroy(MJpegEncoder *encoder);
uint8_t mjpeg_encoder_get_bytes_per_pixel(MJpegEncoder *encoder);
@@ -39,5 +53,15 @@ int mjpeg_encoder_encode_scanline(MJpegEncoder *encoder, uint8_t *src_pixels,
size_t image_width);
size_t mjpeg_encoder_end_frame(MJpegEncoder *encoder);
+/*
+ * bit rate control
+ */
+
+/*
+ * The recommended output frame rate (per second) for the
+ * current available bit rate.
+ */
+uint32_t mjpeg_encoder_get_fps(MJpegEncoder *encoder);
+
#endif
diff --git a/server/red_worker.c b/server/red_worker.c
index 4c7cca7..23d08a8 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -2887,7 +2887,7 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
agent->drops = 0;
agent->fps = MAX_FPS;
reset_rate(dcc, agent);
- agent->mjpeg_encoder = mjpeg_encoder_new();
+ agent->mjpeg_encoder = mjpeg_encoder_new(FALSE, 0, NULL, NULL);
red_channel_client_pipe_add(&dcc->common.base, &agent->create_item);
}
commit 41d740075879f31ddc2b5f8bb177b6523d9be3cb
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Mon Apr 16 09:10:49 2012 +0300
server/red_worker: streams: moving mjpeg_encoder from Stream to StreamAgent
The mjpeg_encoder should be client specific, and not shared between
different clients**, for the following reasons:
(1) Since we use abbreviated jpeg datastream for mjpeg, employing the same
mjpeg_encoder for different clients might cause errors when the
clients decode the jpeg data.
(2) The next patch introduces bit rate control to the mjpeg_encoder.
This feature depends on the bandwidth available, which is client
specific.
** at least till we change multi-clients not to re-encode the same
streams.
diff --git a/server/red_worker.c b/server/red_worker.c
index e6128e9..4c7cca7 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -427,7 +427,6 @@ struct Stream {
int width;
int height;
SpiceRect dest_area;
- MJpegEncoder *mjpeg_encoder;
int top_down;
Stream *next;
RingItem link;
@@ -448,6 +447,7 @@ typedef struct StreamAgent {
PipeItem destroy_item;
Stream *stream;
uint64_t last_send_time;
+ MJpegEncoder *mjpeg_encoder;
int frames;
int drops;
@@ -2484,9 +2484,6 @@ static void red_release_stream(RedWorker *worker, Stream *stream)
{
if (!--stream->refs) {
spice_assert(!ring_item_is_linked(&stream->link));
- if (stream->mjpeg_encoder) {
- mjpeg_encoder_destroy(stream->mjpeg_encoder);
- }
red_free_stream(worker, stream);
worker->stream_count--;
}
@@ -2587,6 +2584,7 @@ static void red_stop_stream(RedWorker *worker, Stream *stream)
spice_debug("stream %d", get_stream_id(worker, stream));
WORKER_FOREACH_DCC(worker, item, dcc) {
StreamAgent *stream_agent;
+
stream_agent = &dcc->stream_agents[get_stream_id(worker, stream)];
region_clear(&stream_agent->vis_region);
region_clear(&stream_agent->clip);
@@ -2889,6 +2887,7 @@ static void red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
agent->drops = 0;
agent->fps = MAX_FPS;
reset_rate(dcc, agent);
+ agent->mjpeg_encoder = mjpeg_encoder_new();
red_channel_client_pipe_add(&dcc->common.base, &agent->create_item);
}
@@ -2914,8 +2913,6 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
stream_width = src_rect->right - src_rect->left;
stream_height = src_rect->bottom - src_rect->top;
- stream->mjpeg_encoder = mjpeg_encoder_new();
-
ring_add(&worker->streams, &stream->link);
stream->current = drawable;
stream->last_time = drawable->creation_time;
@@ -2973,6 +2970,10 @@ static void red_display_destroy_streams_agents(DisplayChannelClient *dcc)
StreamAgent *agent = &dcc->stream_agents[i];
region_destroy(&agent->vis_region);
region_destroy(&agent->clip);
+ if (agent->mjpeg_encoder) {
+ mjpeg_encoder_destroy(agent->mjpeg_encoder);
+ agent->mjpeg_encoder = NULL;
+ }
}
}
@@ -8277,7 +8278,7 @@ static inline void display_begin_send_message(RedChannelClient *rcc)
red_channel_client_begin_send_message(rcc);
}
-static inline uint8_t *red_get_image_line(RedWorker *worker, SpiceChunks *chunks, size_t *offset,
+static inline uint8_t *red_get_image_line(SpiceChunks *chunks, size_t *offset,
int *chunk_nr, int stride)
{
uint8_t *ret;
@@ -8303,13 +8304,14 @@ static inline uint8_t *red_get_image_line(RedWorker *worker, SpiceChunks *chunks
return ret;
}
-static int encode_frame (RedWorker *worker, const SpiceRect *src,
- const SpiceBitmap *image, Stream *stream)
+static int encode_frame(DisplayChannelClient *dcc, const SpiceRect *src,
+ const SpiceBitmap *image, Stream *stream)
{
SpiceChunks *chunks;
uint32_t image_stride;
size_t offset;
int i, chunk;
+ StreamAgent *agent = &dcc->stream_agents[stream - dcc->common.worker->streams_buf];
chunks = image->data;
offset = 0;
@@ -8318,7 +8320,7 @@ static int encode_frame (RedWorker *worker, const SpiceRect *src,
const int skip_lines = stream->top_down ? src->top : image->y - (src->bottom - 0);
for (i = 0; i < skip_lines; i++) {
- red_get_image_line(worker, chunks, &offset, &chunk, image_stride);
+ red_get_image_line(chunks, &offset, &chunk, image_stride);
}
const unsigned int stream_height = src->bottom - src->top;
@@ -8326,14 +8328,14 @@ static int encode_frame (RedWorker *worker, const SpiceRect *src,
for (i = 0; i < stream_height; i++) {
uint8_t *src_line =
- (uint8_t *)red_get_image_line(worker, chunks, &offset, &chunk, image_stride);
+ (uint8_t *)red_get_image_line(chunks, &offset, &chunk, image_stride);
if (!src_line) {
return FALSE;
}
- src_line += src->left * mjpeg_encoder_get_bytes_per_pixel(stream->mjpeg_encoder);
- if (mjpeg_encoder_encode_scanline(stream->mjpeg_encoder, src_line, stream_width) == 0)
+ src_line += src->left * mjpeg_encoder_get_bytes_per_pixel(agent->mjpeg_encoder);
+ if (mjpeg_encoder_encode_scanline(agent->mjpeg_encoder, src_line, stream_width) == 0)
return FALSE;
}
@@ -8387,17 +8389,17 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
}
outbuf_size = dcc->send_data.stream_outbuf_size;
- if (!mjpeg_encoder_start_frame(stream->mjpeg_encoder, image->u.bitmap.format,
+ if (!mjpeg_encoder_start_frame(agent->mjpeg_encoder, image->u.bitmap.format,
width, height,
&dcc->send_data.stream_outbuf,
&outbuf_size)) {
return FALSE;
}
- if (!encode_frame(worker, &drawable->red_drawable->u.copy.src_area,
+ if (!encode_frame(dcc, &drawable->red_drawable->u.copy.src_area,
&image->u.bitmap, stream)) {
return FALSE;
}
- n = mjpeg_encoder_end_frame(stream->mjpeg_encoder);
+ n = mjpeg_encoder_end_frame(agent->mjpeg_encoder);
dcc->send_data.stream_outbuf_size = outbuf_size;
if (!drawable->sized_stream) {
@@ -8801,6 +8803,10 @@ static void red_display_marshall_stream_end(RedChannelClient *rcc,
red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DESTROY, NULL);
destroy.id = get_stream_id(dcc->common.worker, agent->stream);
+ if (agent->mjpeg_encoder) {
+ mjpeg_encoder_destroy(agent->mjpeg_encoder);
+ agent->mjpeg_encoder = NULL;
+ }
spice_marshall_msg_display_stream_destroy(base_marshaller, &destroy);
}
commit 317471fc0b54472876cd054afb531d96796cb54e
Author: Yonit Halperin <yhalperi at redhat.com>
Date: Thu Jan 24 13:24:20 2013 -0500
red_worker: stream agent - fix miscounting of frames
Frames counting was skipped when the previous frame was already
sent completely to the client.
diff --git a/server/red_worker.c b/server/red_worker.c
index 4842ad6..e6128e9 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -3118,22 +3118,30 @@ static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
if (pipe_item_is_linked(&dpi->dpi_pipe_item)) {
++agent->drops;
}
+ }
+
+ WORKER_FOREACH_DCC(worker, ring_item, dcc) {
+ double drop_factor;
+
+ agent = &dcc->stream_agents[index];
if (agent->frames / agent->fps < FPS_TEST_INTERVAL) {
agent->frames++;
- return;
+ continue;
}
- double drop_factor = ((double)agent->frames - (double)agent->drops) /
- (double)agent->frames;
-
+ drop_factor = ((double)agent->frames - (double)agent->drops) /
+ (double)agent->frames;
+ spice_debug("stream %d: #frames %u #drops %u", index, agent->frames, agent->drops);
if (drop_factor == 1) {
if (agent->fps < MAX_FPS) {
agent->fps++;
+ spice_debug("stream %d: fps++ %u", index, agent->fps);
}
} else if (drop_factor < 0.9) {
if (agent->fps > 1) {
agent->fps--;
+ spice_debug("stream %d: fps--%u", index, agent->fps);
}
}
agent->frames = 1;
More information about the Spice-commits
mailing list