[Spice-devel] [spice v11 13/27] server: Adjust the frame rate based on the GStreamer encoding time

[Francois Gouget]

Signed-off-by: Francois Gouget <fgouget at codeweavers.com>
---

This patch limits the frame rate (i.e. drops frames) when the server has 
trouble keeping up with the encoding. This code seems to only be needed 
in SpiceDeferredFPS mode (so maybe it's in the SpiceDeferredFPS code 
that something should be modified). In any case this patch can be 
skipped without any impact on the rest of the series.

 server/gstreamer-encoder.c | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index 28589c3..d71c26c 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -48,6 +48,7 @@ typedef struct SpiceGstVideoBuffer {
 
 typedef struct {
     uint32_t mm_time;
+    uint64_t duration;
     uint32_t size;
 } SpiceGstFrameInformation;
 
@@ -128,6 +129,9 @@ typedef struct SpiceGstEncoder {
     /* The index of the oldest frame taken into account for the statistics. */
     uint32_t stat_first;
 
+    /* Used to compute the average frame encoding time. */
+    uint64_t stat_duration_sum;
+
     /* Used to compute the average frame size. */
     uint64_t stat_size_sum;
 
@@ -346,6 +350,14 @@ static uint64_t get_effective_bit_rate(SpiceGstEncoder *encoder)
     return elapsed ? encoder->stat_size_sum * 8 * MSEC_PER_SEC / elapsed : 0;
 }
 
+static uint64_t get_average_encoding_time(SpiceGstEncoder *encoder)
+{
+    uint32_t count = encoder->history_last +
+        (encoder->history_last < encoder->stat_first ? SPICE_GST_HISTORY_SIZE : 0) -
+        encoder->stat_first + 1;
+    return encoder->stat_duration_sum / count;
+}
+
 static uint64_t get_average_frame_size(SpiceGstEncoder *encoder)
 {
     uint32_t count = encoder->history_last +
@@ -415,19 +427,21 @@ static uint64_t get_period_bit_rate(SpiceGstEncoder *encoder, uint32_t from,
 }
 
 static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
-                      uint32_t size)
+                      uint64_t duration, uint32_t size)
 {
     /* Update the statistics */
     uint32_t count = encoder->history_last +
         (encoder->history_last < encoder->stat_first ? SPICE_GST_HISTORY_SIZE : 0) -
         encoder->stat_first + 1;
     if (count == SPICE_GST_FRAME_STATISTICS_COUNT) {
+        encoder->stat_duration_sum -= encoder->history[encoder->stat_first].duration;
         encoder->stat_size_sum -= encoder->history[encoder->stat_first].size;
         if (encoder->stat_size_max == encoder->history[encoder->stat_first].size) {
             encoder->stat_size_max = 0;
         }
         encoder->stat_first = (encoder->stat_first + 1) % SPICE_GST_HISTORY_SIZE;
     }
+    encoder->stat_duration_sum += duration;
     encoder->stat_size_sum += size;
     if (encoder->stat_size_max > 0 && size > encoder->stat_size_max) {
         encoder->stat_size_max = size;
@@ -439,6 +453,7 @@ static void add_frame(SpiceGstEncoder *encoder, uint32_t frame_mm_time,
         encoder->history_first = (encoder->history_first + 1) % SPICE_GST_HISTORY_SIZE;
     }
     encoder->history[encoder->history_last].mm_time = frame_mm_time;
+    encoder->history[encoder->history_last].duration = duration;
     encoder->history[encoder->history_last].size = size;
 }
 
@@ -473,15 +488,23 @@ static uint32_t get_min_playback_delay(SpiceGstEncoder *encoder)
 static void update_client_playback_delay(SpiceGstEncoder *encoder)
 {
     if (encoder->cbs.update_client_playback_delay) {
-        uint32_t min_delay = get_min_playback_delay(encoder);
+        uint32_t min_delay = get_min_playback_delay(encoder) + get_average_encoding_time(encoder) / NSEC_PER_MILLISEC;
         encoder->cbs.update_client_playback_delay(encoder->cbs.opaque, min_delay);
     }
 }
 
 static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
 {
+    uint64_t period_ns = NSEC_PER_SEC / get_source_fps(encoder);
+    uint64_t min_delay_ns = get_average_encoding_time(encoder);
+    if (min_delay_ns > period_ns) {
+        spice_warning("your system seems to be too slow to encode this %dx%d video in real time", encoder->width, encoder->height);
+    }
+
+    min_delay_ns = MIN(min_delay_ns, SPICE_GST_MAX_PERIOD);
     if (encoder->vbuffer_free >= 0) {
-        encoder->next_frame_mm_time = 0;
+        encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) +
+                                      min_delay_ns / NSEC_PER_MILLISEC;
         return;
     }
 
@@ -489,7 +512,6 @@ static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
      * Use nanoseconds to avoid precision loss.
      */
     uint64_t delay_ns = -encoder->vbuffer_free * 8 * NSEC_PER_SEC / encoder->bit_rate;
-    uint64_t period_ns = NSEC_PER_SEC / get_source_fps(encoder);
     uint32_t drops = (delay_ns + period_ns - 1) / period_ns; /* round up */
     spice_debug("drops=%u vbuffer %d/%d", drops, encoder->vbuffer_free,
                 encoder->vbuffer_size);
@@ -504,7 +526,8 @@ static void update_next_frame_mm_time(SpiceGstEncoder *encoder)
         }
         delay_ns = SPICE_GST_MAX_PERIOD;
     }
-    encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) + delay_ns / NSEC_PER_MILLISEC;
+    encoder->next_frame_mm_time = get_last_frame_mm_time(encoder) +
+                                  MAX(delay_ns, min_delay_ns) / NSEC_PER_MILLISEC;
 
     /* Drops mean a higher delay between encoded frames so update the
      * playback delay.
@@ -595,6 +618,7 @@ static void set_bit_rate(SpiceGstEncoder *encoder, uint64_t bit_rate)
      * situation anymore.
      */
     encoder->stat_first = encoder->history_last;
+    encoder->stat_duration_sum = encoder->history[encoder->history_last].duration;
     encoder->stat_size_sum = encoder->stat_size_max = encoder->history[encoder->history_last].size;
 
     if (bit_rate > encoder->video_bit_rate) {
@@ -654,7 +678,7 @@ static inline gboolean handle_server_drops(SpiceGstEncoder *encoder,
      * time during which the buffer was refilling. This implies dropping this
      * frame.
      */
-    add_frame(encoder, frame_mm_time, 0);
+    add_frame(encoder, frame_mm_time, 0, 0);
 
     if (encoder->server_drops >= get_source_fps(encoder)) {
         spice_debug("cut the bit rate");
@@ -1248,6 +1272,7 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
         return VIDEO_ENCODER_FRAME_UNSUPPORTED;
     }
 
+    uint64_t start = spice_get_monotonic_time_ns();
     int rc = push_raw_frame(encoder, bitmap, src, top_down, bitmap_opaque);
     if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
         rc = pull_compressed_buffer(encoder, outbuf);
@@ -1265,7 +1290,8 @@ static int spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
         return rc;
     }
     uint32_t last_mm_time = get_last_frame_mm_time(encoder);
-    add_frame(encoder, frame_mm_time, (*outbuf)->size);
+    add_frame(encoder, frame_mm_time, spice_get_monotonic_time_ns() - start,
+              (*outbuf)->size);
 
     int32_t refill = encoder->bit_rate * (frame_mm_time - last_mm_time) / MSEC_PER_SEC / 8;
     encoder->vbuffer_free = MIN(encoder->vbuffer_free + refill,
-- 
2.7.0