[Spice-devel] [PATCH v2 spice 04/11] server: video streaming: add support for frames of different sizes
Yonit Halperin
yhalperi at redhat.com
Tue Apr 17 03:12:29 PDT 2012
When playing a youtube video on Windows guest, the driver sometimes sends
images which contain the video frames, but also other parts of the
screen (e.g., the youtube process bar). In order to prevent glitches, we send these
images as part of the stream, using SPICE_MSG_DISPLAY_STREAM_DATA_SIZED.
---
server/mjpeg_encoder.c | 15 ++---
server/mjpeg_encoder.h | 3 +-
server/red_worker.c | 183 +++++++++++++++++++++++++++++++++++++-----------
3 files changed, 150 insertions(+), 51 deletions(-)
diff --git a/server/mjpeg_encoder.c b/server/mjpeg_encoder.c
index 4692315..b3685f8 100644
--- a/server/mjpeg_encoder.c
+++ b/server/mjpeg_encoder.c
@@ -25,8 +25,6 @@
#include <jpeglib.h>
struct MJpegEncoder {
- int width;
- int height;
uint8_t *row;
int first_frame;
int quality;
@@ -38,15 +36,13 @@ struct MJpegEncoder {
void (*pixel_converter)(uint8_t *src, uint8_t *dest);
};
-MJpegEncoder *mjpeg_encoder_new(int width, int height)
+MJpegEncoder *mjpeg_encoder_new()
{
MJpegEncoder *enc;
enc = spice_new0(MJpegEncoder, 1);
enc->first_frame = TRUE;
- enc->width = width;
- enc->height = height;
enc->quality = 70;
enc->cinfo.err = jpeg_std_error(&enc->jerr);
jpeg_create_compress(&enc->cinfo);
@@ -200,6 +196,7 @@ spice_jpeg_mem_dest(j_compress_ptr cinfo,
/* end of code from libjpeg */
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
+ int width, int height,
uint8_t **dest, size_t *dest_len)
{
encoder->cinfo.in_color_space = JCS_RGB;
@@ -233,9 +230,9 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
}
if ((encoder->pixel_converter != NULL) && (encoder->row == NULL)) {
- unsigned int stride = encoder->width * 3;
+ unsigned int stride = width * 3;
/* check for integer overflow */
- if (stride < encoder->width) {
+ if (stride < width) {
return FALSE;
}
encoder->row = spice_malloc(stride);
@@ -243,8 +240,8 @@ int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
spice_jpeg_mem_dest(&encoder->cinfo, dest, dest_len);
- encoder->cinfo.image_width = encoder->width;
- encoder->cinfo.image_height = encoder->height;
+ encoder->cinfo.image_width = width;
+ encoder->cinfo.image_height = height;
jpeg_set_defaults(&encoder->cinfo);
encoder->cinfo.dct_method = JDCT_IFAST;
jpeg_set_quality(&encoder->cinfo, encoder->quality, TRUE);
diff --git a/server/mjpeg_encoder.h b/server/mjpeg_encoder.h
index c43827f..3a005b7 100644
--- a/server/mjpeg_encoder.h
+++ b/server/mjpeg_encoder.h
@@ -23,11 +23,12 @@
typedef struct MJpegEncoder MJpegEncoder;
-MJpegEncoder *mjpeg_encoder_new(int width, int height);
+MJpegEncoder *mjpeg_encoder_new();
void mjpeg_encoder_destroy(MJpegEncoder *encoder);
uint8_t mjpeg_encoder_get_bytes_per_pixel(MJpegEncoder *encoder);
int mjpeg_encoder_start_frame(MJpegEncoder *encoder, SpiceBitmapFmt format,
+ int width, int height,
uint8_t **dest, size_t *dest_len);
int mjpeg_encoder_encode_scanline(MJpegEncoder *encoder, uint8_t *src_pixels,
size_t image_width);
diff --git a/server/red_worker.c b/server/red_worker.c
index 34e6a04..c66e397 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -374,6 +374,12 @@ typedef struct ImageItem {
typedef struct Drawable Drawable;
+enum {
+ STREAM_FRAME_NONE,
+ STREAM_FRAME_NATIVE,
+ STREAM_FRAME_CONTAINER,
+};
+
typedef struct Stream Stream;
struct Stream {
uint8_t refs;
@@ -784,6 +790,7 @@ struct Drawable {
int gradual_frames_count;
int last_gradual_frame;
Stream *stream;
+ Stream *sized_stream;
int streamable;
BitmapGradualType copy_bitmap_graduality;
uint32_t group_id;
@@ -2773,7 +2780,7 @@ static void red_create_stream(RedWorker *worker, Drawable *drawable)
stream_width = src_rect->right - src_rect->left;
stream_height = src_rect->bottom - src_rect->top;
- stream->mjpeg_encoder = mjpeg_encoder_new(stream_width, stream_height);
+ stream->mjpeg_encoder = mjpeg_encoder_new();
ring_add(&worker->streams, &stream->link);
stream->current = drawable;
@@ -2850,34 +2857,63 @@ static inline int __red_is_next_stream_frame(RedWorker *worker,
const int other_src_height,
const SpiceRect *other_dest,
const red_time_t other_time,
- const Stream *stream)
+ const Stream *stream,
+ int container_candidate_allowed)
{
RedDrawable *red_drawable;
+ int is_frame_container = FALSE;
if (candidate->creation_time - other_time >
(stream ? RED_STREAM_CONTINUS_MAX_DELTA : RED_STREAM_DETACTION_MAX_DELTA)) {
- return FALSE;
+ return STREAM_FRAME_NONE;
}
red_drawable = candidate->red_drawable;
+ if (!container_candidate_allowed) {
+ SpiceRect* candidate_src;
- if (!rect_is_equal(&red_drawable->bbox, other_dest)) {
- return FALSE;
- }
+ if (!rect_is_equal(&red_drawable->bbox, other_dest)) {
+ return STREAM_FRAME_NONE;
+ }
- SpiceRect* candidate_src = &red_drawable->u.copy.src_area;
- if (candidate_src->right - candidate_src->left != other_src_width ||
- candidate_src->bottom - candidate_src->top != other_src_height) {
- return FALSE;
+ candidate_src = &red_drawable->u.copy.src_area;
+ if (candidate_src->right - candidate_src->left != other_src_width ||
+ candidate_src->bottom - candidate_src->top != other_src_height) {
+ return STREAM_FRAME_NONE;
+ }
+ } else {
+ if (rect_contains(&red_drawable->bbox, other_dest)) {
+ int candidate_area = rect_get_area(&red_drawable->bbox);
+ int other_area = rect_get_area(other_dest);
+
+ if (candidate_area > 1.3 * other_area) {
+ spice_debug("Too big candidate: (%d, %d) (%d, %d), new: (%d, %d) (%d, %d)",
+ other_dest->left, other_dest->top,
+ other_dest->right, other_dest->bottom,
+ red_drawable->bbox.left, red_drawable->bbox.top,
+ red_drawable->bbox.right, red_drawable->bbox.bottom);
+ return STREAM_FRAME_NONE;
+ }
+
+ if (candidate_area > other_area) {
+ is_frame_container = TRUE;
+ }
+ } else {
+ return STREAM_FRAME_NONE;
+ }
}
if (stream) {
SpiceBitmap *bitmap = &red_drawable->u.copy.src_bitmap->u.bitmap;
if (stream->top_down != !!(bitmap->flags & SPICE_BITMAP_FLAGS_TOP_DOWN)) {
- return FALSE;
+ return STREAM_FRAME_NONE;
}
}
- return TRUE;
+ if (is_frame_container) {
+ return STREAM_FRAME_CONTAINER;
+ } else {
+ return STREAM_FRAME_NATIVE;
+ }
}
static inline int red_is_next_stream_frame(RedWorker *worker, const Drawable *candidate,
@@ -2891,7 +2927,8 @@ static inline int red_is_next_stream_frame(RedWorker *worker, const Drawable *ca
return __red_is_next_stream_frame(worker, candidate, prev_src->right - prev_src->left,
prev_src->bottom - prev_src->top,
&prev->red_drawable->bbox, prev->creation_time,
- prev->stream);
+ prev->stream,
+ FALSE);
}
static void reset_rate(DisplayChannelClient *dcc, StreamAgent *stream_agent)
@@ -3033,20 +3070,31 @@ static inline void red_stream_maintenance(RedWorker *worker, Drawable *candidate
return;
}
- if (!red_is_next_stream_frame(worker, candidate, prev)) {
- return;
- }
-
if ((stream = prev->stream)) {
- pre_stream_item_swap(worker, stream);
- red_detach_stream(worker, stream);
- prev->streamable = FALSE; //prevent item trace
- red_attach_stream(worker, candidate, stream);
+ int is_next_frame = __red_is_next_stream_frame(worker,
+ candidate,
+ stream->width,
+ stream->height,
+ &stream->dest_area,
+ stream->last_time,
+ stream,
+ TRUE);
+ if (is_next_frame != STREAM_FRAME_NONE) {
+ pre_stream_item_swap(worker, stream);
+ red_detach_stream(worker, stream);
+ prev->streamable = FALSE; //prevent item trace
+ red_attach_stream(worker, candidate, stream);
+ if (is_next_frame == STREAM_FRAME_CONTAINER) {
+ candidate->sized_stream = stream;
+ }
+ }
} else {
- red_stream_add_frame(worker, candidate,
- prev->frames_count,
- prev->gradual_frames_count,
- prev->last_gradual_frame);
+ if (red_is_next_stream_frame(worker, candidate, prev) != STREAM_FRAME_NONE) {
+ red_stream_add_frame(worker, candidate,
+ prev->frames_count,
+ prev->gradual_frames_count,
+ prev->last_gradual_frame);
+ }
}
}
@@ -3174,14 +3222,24 @@ static inline void red_use_stream_trace(RedWorker *worker, Drawable *drawable)
while (item) {
Stream *stream = SPICE_CONTAINEROF(item, Stream, link);
- if (!stream->current && __red_is_next_stream_frame(worker,
- drawable,
- stream->width,
- stream->height,
- &stream->dest_area,
- stream->last_time,
- stream)) {
+ int is_next_frame = __red_is_next_stream_frame(worker,
+ drawable,
+ stream->width,
+ stream->height,
+ &stream->dest_area,
+ stream->last_time,
+ stream,
+ TRUE);
+ if (is_next_frame != STREAM_FRAME_NONE) {
+ if (stream->current) {
+ stream->current->streamable = FALSE; //prevent item trace
+ pre_stream_item_swap(worker, stream);
+ red_detach_stream(worker, stream);
+ }
red_attach_stream(worker, drawable, stream);
+ if (is_next_frame == STREAM_FRAME_CONTAINER) {
+ drawable->sized_stream = stream;
+ }
return;
}
item = ring_next(ring, item);
@@ -3191,7 +3249,8 @@ static inline void red_use_stream_trace(RedWorker *worker, Drawable *drawable)
trace_end = trace + NUM_TRACE_ITEMS;
for (; trace < trace_end; trace++) {
if (__red_is_next_stream_frame(worker, drawable, trace->width, trace->height,
- &trace->dest_area, trace->time, NULL)) {
+ &trace->dest_area, trace->time, NULL, FALSE) !=
+ STREAM_FRAME_NONE) {
if (red_stream_add_frame(worker, drawable,
trace->frames_count,
trace->gradual_frames_count,
@@ -8027,8 +8086,12 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
SpiceImage *image;
RedWorker *worker = dcc->common.worker;
int n;
+ int width, height;
- spice_assert(stream);
+ if (!stream) {
+ spice_assert(drawable->sized_stream);
+ stream = drawable->sized_stream;
+ }
spice_assert(drawable->red_drawable->type == QXL_DRAW_COPY);
worker = display_channel->common.worker;
@@ -8038,6 +8101,20 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
return FALSE;
}
+ if (drawable->sized_stream) {
+ if (red_channel_client_test_remote_cap(rcc, SPICE_DISPLAY_CAP_SIZED_STREAM)) {
+ SpiceRect *src_rect = &drawable->red_drawable->u.copy.src_area;
+
+ width = src_rect->right - src_rect->left;
+ height = src_rect->bottom - src_rect->top;
+ } else {
+ return FALSE;
+ }
+ } else {
+ width = stream->width;
+ height = stream->height;
+ }
+
StreamAgent *agent = &dcc->stream_agents[stream - worker->streams_buf];
uint64_t time_now = red_now();
size_t outbuf_size;
@@ -8048,6 +8125,7 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
outbuf_size = dcc->send_data.stream_outbuf_size;
if (!mjpeg_encoder_start_frame(stream->mjpeg_encoder, image->u.bitmap.format,
+ width, height,
&dcc->send_data.stream_outbuf,
&outbuf_size)) {
return FALSE;
@@ -8059,14 +8137,35 @@ static inline int red_marshall_stream_data(RedChannelClient *rcc,
n = mjpeg_encoder_end_frame(stream->mjpeg_encoder);
dcc->send_data.stream_outbuf_size = outbuf_size;
- red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DATA, NULL);
+ if (!drawable->sized_stream) {
+ SpiceMsgDisplayStreamData stream_data;
+
+ red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DATA, NULL);
- SpiceMsgDisplayStreamData stream_data;
+ stream_data.base.id = stream - worker->streams_buf;
+ stream_data.base.multi_media_time = drawable->red_drawable->mm_time;
+ stream_data.data_size = n;
- stream_data.base.id = stream - worker->streams_buf;
- stream_data.base.multi_media_time = drawable->red_drawable->mm_time;
- stream_data.data_size = n;
- spice_marshall_msg_display_stream_data(base_marshaller, &stream_data);
+ spice_marshall_msg_display_stream_data(base_marshaller, &stream_data);
+ } else {
+ SpiceMsgDisplayStreamDataSized stream_data;
+
+ red_channel_client_init_send_data(rcc, SPICE_MSG_DISPLAY_STREAM_DATA_SIZED, NULL);
+
+ stream_data.base.id = stream - worker->streams_buf;
+ stream_data.base.multi_media_time = drawable->red_drawable->mm_time;
+ stream_data.data_size = n;
+ stream_data.width = width;
+ stream_data.height = height;
+ stream_data.dest = drawable->red_drawable->bbox;
+
+ spice_debug("stream %d sized frame: dest %dx%d (%d,%d) (%d,%d))", stream_data.base.id,
+ stream_data.dest.right - stream_data.dest.left,
+ stream_data.dest.bottom - stream_data.dest.top,
+ stream_data.dest.left, stream_data.dest.top,
+ stream_data.dest.right, stream_data.dest.bottom);
+ spice_marshall_msg_display_stream_data_sized(base_marshaller, &stream_data);
+ }
spice_marshaller_add_ref(base_marshaller,
dcc->send_data.stream_outbuf, n);
agent->last_send_time = time_now;
@@ -8080,7 +8179,9 @@ static inline void marshall_qxl_drawable(RedChannelClient *rcc,
DisplayChannel *display_channel = SPICE_CONTAINEROF(rcc->channel, DisplayChannel, common.base);
spice_assert(display_channel && rcc);
- if (item->stream && red_marshall_stream_data(rcc, m, item)) {
+ /* allow sized frames to be streamed, even if they where detached from the stream, since
+ * unlike other frames, they are not dropped */
+ if ((item->stream || item->sized_stream) && red_marshall_stream_data(rcc, m, item)) {
return;
}
if (!display_channel->enable_jpeg)
--
1.7.7.6
More information about the Spice-devel
mailing list