[Spice-devel] [RFC v2 3/5] display-channel: Share the drawable's copy of fd with the encoder

[Vivek Kasireddy]

Once the drawable gets a copy of the fd, the goal is to share it with
the encoder. This patch accomplishes this task and also registers
a callback to unblock the pipeline once the encoder is done using
the fd.

Additionally, this patch also does the following:
- Adds helpers to manage the async count for different situations
- Creates an object of type VideoEncoderDmabufData and populates
  it with relevant data such as fd, callback, etc.

Cc: Gerd Hoffmann <kraxel at redhat.com>
Cc: Marc-André Lureau <marcandre.lureau at redhat.com>
Cc: Dongwon Kim <dongwon.kim at intel.com>
Signed-off-by: Vivek Kasireddy <vivek.kasireddy at intel.com>
---
 server/dcc-send.cpp        | 30 ++++++++++++++++++++++++++++++
 server/display-channel.cpp | 14 ++++++++++++++
 server/display-channel.h   |  3 +++
 server/video-stream.cpp    |  6 ++++++
 4 files changed, 53 insertions(+)

diff --git a/server/dcc-send.cpp b/server/dcc-send.cpp
index 2c40a231..13406865 100644
--- a/server/dcc-send.cpp
+++ b/server/dcc-send.cpp
@@ -1637,6 +1637,13 @@ static void red_release_video_encoder_buffer(uint8_t *data, void *opaque)
     buffer->free(buffer);
 }
 
+static void red_mem_free_cb(void *opaque)
+{
+    auto display = static_cast<DisplayChannel *>(opaque);
+
+    display_channel_gl_draw_done(display);
+}
+
 static bool red_marshall_stream_data(DisplayChannelClient *dcc,
                                      SpiceMarshaller *base_marshaller,
                                      Drawable *drawable)
@@ -1667,10 +1674,22 @@ static bool red_marshall_stream_data(DisplayChannelClient *dcc,
     int stream_id = display_channel_get_video_stream_id(display, stream);
     VideoStreamAgent *agent = &dcc->priv->stream_agents[stream_id];
     VideoBuffer *outbuf;
+    VideoEncoderDmabufData *dmabuf_data = NULL;
     /* workaround for vga streams */
     frame_mm_time =  drawable->red_drawable->mm_time ?
                         drawable->red_drawable->mm_time :
                         reds_get_mm_time();
+
+    if (drawable->dmabuf_fd > 0) {
+        dmabuf_data = g_new0(VideoEncoderDmabufData, 1);
+        dmabuf_data->dmabuf_fd = drawable->dmabuf_fd;
+        dmabuf_data->opaque = display;
+        dmabuf_data->notify_mem_free = red_mem_free_cb;
+
+        display_channel_add_encode_async(display);
+    }
+    agent->video_encoder->dmabuf_data = dmabuf_data;
+
     ret = !agent->video_encoder ? VIDEO_ENCODER_FRAME_UNSUPPORTED :
           agent->video_encoder->encode_frame(agent->video_encoder,
                                              frame_mm_time,
@@ -1678,6 +1697,13 @@ static bool red_marshall_stream_data(DisplayChannelClient *dcc,
                                              &copy->src_area, stream->top_down,
                                              drawable->red_drawable.get(),
                                              &outbuf);
+    if (agent->video_encoder->dmabuf_data &&
+        ret != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+        display_channel_gl_draw_done(display);
+        g_free(agent->video_encoder->dmabuf_data);
+        agent->video_encoder->dmabuf_data = NULL;
+    }
+
     switch (ret) {
     case VIDEO_ENCODER_FRAME_DROP:
 #ifdef STREAM_STATS
@@ -2095,6 +2121,10 @@ static void marshall_qxl_drawable(DisplayChannelClient *dcc,
         marshall_lossy_qxl_drawable(dcc, m, dpi);
     else
         marshall_lossless_qxl_drawable(dcc, m, dpi);
+
+    if (item->dmabuf_fd > 0) {
+        display_channel_gl_draw_done(display);
+    }
 }
 
 static void marshall_stream_start(DisplayChannelClient *dcc,
diff --git a/server/display-channel.cpp b/server/display-channel.cpp
index 9534bbf2..349c1405 100644
--- a/server/display-channel.cpp
+++ b/server/display-channel.cpp
@@ -2343,6 +2343,20 @@ void display_channel_gl_draw_done(DisplayChannel *display)
     set_gl_draw_async_count(display, display->priv->gl_draw_async_count - 1);
 }
 
+void display_channel_set_encode_async(DisplayChannel *display, int num)
+{
+    set_gl_draw_async_count(display, num);
+}
+
+void display_channel_add_encode_async(DisplayChannel *display)
+{
+    int num = display->priv->gl_draw_async_count;
+
+    if (num == 0) {
+        display_channel_set_encode_async(display, num + 1);
+    }
+}
+
 int display_channel_get_video_stream_id(DisplayChannel *display, VideoStream *stream)
 {
     return static_cast<int>(stream - display->priv->streams_buf.data());
diff --git a/server/display-channel.h b/server/display-channel.h
index 276d015a..de1243d2 100644
--- a/server/display-channel.h
+++ b/server/display-channel.h
@@ -128,6 +128,9 @@ void                       display_channel_gl_scanout                (DisplayCha
 void                       display_channel_gl_draw                   (DisplayChannel *display,
                                                                       SpiceMsgDisplayGlDraw *draw);
 void                       display_channel_gl_draw_done              (DisplayChannel *display);
+void                       display_channel_set_encode_async          (DisplayChannel *display,
+                                                                      int num_clients);
+void                       display_channel_add_encode_async          (DisplayChannel *display);
 
 void display_channel_process_draw(DisplayChannel *display,
                                   red::shared_ptr<RedDrawable> &&red_drawable,
diff --git a/server/video-stream.cpp b/server/video-stream.cpp
index 056d0c31..28b42d91 100644
--- a/server/video-stream.cpp
+++ b/server/video-stream.cpp
@@ -371,6 +371,7 @@ static void display_channel_create_stream(DisplayChannel *display, Drawable *dra
     DisplayChannelClient *dcc;
     VideoStream *stream;
     SpiceRect* src_rect;
+    int num_clients = 0;
 
     spice_assert(!drawable->stream);
 
@@ -407,7 +408,12 @@ static void display_channel_create_stream(DisplayChannel *display, Drawable *dra
     display->priv->stream_count++;
     FOREACH_DCC(display, dcc) {
         dcc_create_stream(dcc, stream);
+        num_clients++;
     }
+    if (drawable->dmabuf_fd > 0) {
+        display_channel_set_encode_async(display, num_clients);
+    }
+
     spice_debug("stream %d %dx%d (%d, %d) (%d, %d) %u fps",
                 display_channel_get_video_stream_id(display, stream), stream->width,
                 stream->height, stream->dest_area.left, stream->dest_area.top,
-- 
2.37.2