[virglrenderer-devel] [PATCH] vrend: don't try to read back from the multisample textures/framebuffers

[Gert Wollny]

It seems that when the guest requests a multisample GLX context on the
host a multisample framebuffer is created an reading back from it fails
in vrend_transfer_send_readpixels because this is not supported. In the worst
case write_transfer_data evaluates strides that are to large for the buffer
given, and qemu crashes when the data is copied. For now bail out and
also skip over the subsequent vrend_transfer_send_getteximage that faces
the same problem.

TODO: Blit the multisample buffer to a non-multisample target and read from 
there. 

Signed-off-by: Gert Wollny <gert.wollny at collabora.com>
---
 src/vrend_renderer.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/vrend_renderer.c b/src/vrend_renderer.c
index d0e9495..9a22214 100644
--- a/src/vrend_renderer.c
+++ b/src/vrend_renderer.c
@@ -5699,6 +5699,7 @@ static int vrend_transfer_send_readpixels(struct vrend_context *ctx,
 {
    char *myptr = (char*)iov[0].iov_base + info->offset;
    int need_temp = 0;
+   int n_samples = 0;
    GLuint fb_id;
    char *data;
    bool actually_invert, separate_invert = false;
@@ -5752,6 +5753,15 @@ static int vrend_transfer_send_readpixels(struct vrend_context *ctx,
    } else
       glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, res->readback_fb_id);
 
+   glGetIntegerv(GL_SAMPLES, &n_samples);
+   if (n_samples > 1) {
+      /* TODO: blit the data to a non-multisample fbo here. */
+      fprintf(stderr, "VREND: Can't use glReadPixel on a multisample(%d) frame buffer\n", n_samples);
+      if (need_temp)
+         free(data);
+      return EOPNOTSUPP;
+   }
+
    if (actually_invert)
       y1 = h - info->box->y - info->box->height;
    else
@@ -5918,8 +5928,10 @@ static int vrend_renderer_transfer_send_iov(struct vrend_context *ctx,
          ret = vrend_transfer_send_readonly(ctx, res, iov, num_iovs, info);
       }
 
-      /* Can hit this on a non-error path as well. */
-      if (ret != 0) {
+      /* Can hit this on a non-error path as well.
+       * if send_readpixels bailed out because it didn't support the read operation
+       * don't try again. */
+      if (ret != 0 && ret != EOPNOTSUPP) {
          ret = vrend_transfer_send_getteximage(ctx, res, iov, num_iovs, info);
       }
       return ret;
-- 
2.17.1