[Spice-devel] [spice v8 12/24] server: Give up after a while if GStreamer cannot handle the video

[Francois Gouget]

This typically happens when sending very small frames (less than
16 pixels in one dimension) to the x264enc encoder.
This avoids repeatedly wasting time rebuilding the pipeline.

Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
---
 server/gstreamer-encoder.c | 46 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 9 deletions(-)

diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 44072f0..bf60773 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -74,6 +74,9 @@ typedef struct SpiceGstEncoder {
     SpiceFormatForGStreamer *format;
     SpiceBitmapFmt spice_format;
 
+    /* Number of consecutive frame encoding errors. */
+    uint32_t errors;
+
     /* ---------- GStreamer pipeline ---------- */
 
     /* Pointers to the GStreamer pipeline elements. If pipeline is NULL the
@@ -730,24 +733,27 @@ static void set_appsrc_caps(SpiceGstEncoder *encoder)
     g_object_set(G_OBJECT(encoder->appsrc), "caps", encoder->src_caps, NULL);
 }
 
-/* A helper for spice_gst_encoder_encode_frame() */
-static gboolean create_pipeline(SpiceGstEncoder *encoder)
+static const gchar* get_gst_codec_name(SpiceGstEncoder *encoder)
 {
-    const gchar* gstenc_name;
     switch (encoder->base.codec_type)
     {
     case SPICE_VIDEO_CODEC_TYPE_MJPEG:
-        gstenc_name = "avenc_mjpeg";
-        break;
+        return "avenc_mjpeg";
     case SPICE_VIDEO_CODEC_TYPE_VP8:
-        gstenc_name = "vp8enc";
-        break;
+        return "vp8enc";
     case SPICE_VIDEO_CODEC_TYPE_H264:
-        gstenc_name = "x264enc";
-        break;
+        return "x264enc";
     default:
         /* gstreamer_encoder_new() should have rejected this codec type */
         spice_warning("unsupported codec type %d", encoder->base.codec_type);
+        return NULL;
+    }
+}
+
+static gboolean create_pipeline(SpiceGstEncoder *encoder)
+{
+    const gchar* gstenc_name = get_gst_codec_name(encoder);
+    if (!gstenc_name) {
         return FALSE;
     }
 
@@ -1148,6 +1154,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
         encoder->format = map_format(bitmap->format);
         if (!encoder->format) {
             spice_debug("unable to map format type %d", bitmap->format);
+            encoder->errors = 4;
             return VIDEO_ENCODER_FRAME_UNSUPPORTED;
         }
         encoder->spice_format = bitmap->format;
@@ -1163,6 +1170,19 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
         } else if (encoder->pipeline) {
             reconfigure_pipeline(encoder);
         }
+        encoder->errors = 0;
+    } else if (encoder->errors >= 3) {
+        /* The pipeline keeps failing to handle the frames we send it, which is
+         * usually because they are too small (mouse pointer-sized).
+         * So give up until something changes.
+         */
+        if (encoder->errors == 3) {
+            spice_debug("%s cannot compress %dx%d:%dbpp frames",
+                        get_gst_codec_name(encoder), encoder->width,
+                        encoder->height, encoder->format->bpp);
+            encoder->errors++;
+        }
+        return VIDEO_ENCODER_FRAME_UNSUPPORTED;
     }
 
     if (rate_control_is_active(encoder) &&
@@ -1174,12 +1194,20 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
 
     if (!is_pipeline_configured(encoder) &&
         !configure_pipeline(encoder, bitmap)) {
+        encoder->errors++;
         return VIDEO_ENCODER_FRAME_UNSUPPORTED;
     }
 
     int rc = push_raw_frame(encoder, bitmap, src, top_down);
     if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
         rc = pull_compressed_buffer(encoder, video_buffer);
+        if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+            /* The input buffer will be stuck in the pipeline, preventing
+             * later ones from being processed. So reset the pipeline.
+             */
+            free_pipeline(encoder);
+            encoder->errors++;
+        }
 #ifdef DO_ZERO_COPY
         /* GStreamer should have released the source frame buffer by now */
         if (encoder->needs_bitmap) {
-- 
2.6.4