[Mesa-dev] [PATCH] mesa: Validate the result of pipe_transfer_map in make_texture (v2)

[Marek Olšák]

From: Josh Pieper <jjp at pobox.com>

When using Freecad, I was getting intermittent segfaults inside of
mesa.  I traced it down to this path in st_cb_drawpixels.c where the
result of pipe_transfer_map wasn't being checked.  In my case, it was
returning NULL because nouveau_bo_new returned ENOENT.  I'm by no
means a mesa developer, but this patch solves the problem for me and
seems reasonable enough.

v2: Marek - also unmap the PBO and release the texture, and call
    the make_texture function sooner for less cleanup

Cc: 18.1 18.2 <mesa-stable at lists.freedesktop.org>
---
 src/mesa/state_tracker/st_cb_drawpixels.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/mesa/state_tracker/st_cb_drawpixels.c b/src/mesa/state_tracker/st_cb_drawpixels.c
index cb50b7104a0..4f08e751393 100644
--- a/src/mesa/state_tracker/st_cb_drawpixels.c
+++ b/src/mesa/state_tracker/st_cb_drawpixels.c
@@ -559,21 +559,25 @@ make_texture(struct st_context *st,
       GLubyte *dest;
       const GLbitfield imageTransferStateSave = ctx->_ImageTransferState;
 
       /* we'll do pixel transfer in a fragment shader */
       ctx->_ImageTransferState = 0x0;
 
       /* map texture transfer */
       dest = pipe_transfer_map(pipe, pt, 0, 0,
                                PIPE_TRANSFER_WRITE, 0, 0,
                                width, height, &transfer);
-
+      if (!dest) {
+         pipe_resource_reference(&pt, NULL);
+         _mesa_unmap_pbo_source(ctx, unpack);
+         return NULL;
+      }
 
       /* Put image into texture transfer.
        * Note that the image is actually going to be upside down in
        * the texture.  We deal with that with texcoords.
        */
       if ((format == GL_RGBA || format == GL_BGRA)
           && type == GL_UNSIGNED_BYTE) {
          /* Use a memcpy-based texstore to avoid software pixel swizzling.
           * We'll do the necessary swizzling with the pipe_sampler_view to
           * give much better performance.
@@ -1167,20 +1171,27 @@ st_DrawPixels(struct gl_context *ctx, GLint x, GLint y,
       write_depth = GL_TRUE;
 
    if (write_stencil &&
        !pipe->screen->get_param(pipe->screen, PIPE_CAP_SHADER_STENCIL_EXPORT)) {
       /* software fallback */
       draw_stencil_pixels(ctx, x, y, width, height, format, type,
                           unpack, pixels);
       return;
    }
 
+   /* Put glDrawPixels image into a texture */
+   pt = make_texture(st, width, height, format, type, unpack, pixels);
+   if (!pt) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glDrawPixels");
+      return;
+   }
+
    /*
     * Get vertex/fragment shaders
     */
    if (write_depth || write_stencil) {
       driver_fp = get_drawpix_z_stencil_program(st, write_depth,
                                                 write_stencil);
       driver_vp = make_passthrough_vertex_shader(st, GL_TRUE);
    }
    else {
       fpv = get_color_fp_variant(st);
@@ -1193,27 +1204,20 @@ st_DrawPixels(struct gl_context *ctx, GLint x, GLint y,
                                      st->pixel_xfer.pixelmap_sampler_view);
          num_sampler_view++;
       }
 
       /* compiling a new fragment shader variant added new state constants
        * into the constant buffer, we need to update them
        */
       st_upload_constants(st, &st->fp->Base);
    }
 
-   /* Put glDrawPixels image into a texture */
-   pt = make_texture(st, width, height, format, type, unpack, pixels);
-   if (!pt) {
-      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glDrawPixels");
-      return;
-   }
-
    /* create sampler view for the image */
    sv[0] = st_create_texture_sampler_view(st->pipe, pt);
    if (!sv[0]) {
       _mesa_error(ctx, GL_OUT_OF_MEMORY, "glDrawPixels");
       pipe_resource_reference(&pt, NULL);
       return;
    }
 
    /* Set up the sampler view's swizzle */
    setup_sampler_swizzle(sv[0], format, type);
-- 
2.17.1