[Mesa-dev] [PATCH] llvmpipe: fix overflow bug in total texture size computation

[Jose Fonseca]

Looks good to me. Thanks.

Jose

----- Original Message -----
> v2: use uint64_t for the total_size variable, per Jose.
> 
> Also add two earlier checks for exceeding the max texture size.
> For example a 1K^3 RGBA volume would overflow the lpr->image_stride
> variable.
> 
> Use simple algebra to avoid overflow in intermediate values.
> So instead of "x * y > z" use "x > z / y".
> 
> This should work if we happen to be on a platform that doesn't have
> 64-bit types.
> ---
>  src/gallium/drivers/llvmpipe/lp_texture.c |   18 ++++++++++++++++--
>  1 files changed, 16 insertions(+), 2 deletions(-)
> 
> diff --git a/src/gallium/drivers/llvmpipe/lp_texture.c
> b/src/gallium/drivers/llvmpipe/lp_texture.c
> index c0a612c..841df00 100644
> --- a/src/gallium/drivers/llvmpipe/lp_texture.c
> +++ b/src/gallium/drivers/llvmpipe/lp_texture.c
> @@ -113,7 +113,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen
> *screen,
>     unsigned width = pt->width0;
>     unsigned height = pt->height0;
>     unsigned depth = pt->depth0;
> -   size_t total_size = 0;
> +   uint64_t total_size = 0;
>  
>     assert(LP_MAX_TEXTURE_2D_LEVELS <= LP_MAX_TEXTURE_LEVELS);
>     assert(LP_MAX_TEXTURE_3D_LEVELS <= LP_MAX_TEXTURE_LEVELS);
> @@ -140,6 +140,12 @@ llvmpipe_texture_layout(struct llvmpipe_screen
> *screen,
>  
>           lpr->row_stride[level] = align(nblocksx * block_size, 16);
>  
> +         /* if row_stride * height > LP_MAX_TEXTURE_SIZE */
> +         if (lpr->row_stride[level] > LP_MAX_TEXTURE_SIZE /
> nblocksy) {
> +            /* image too large */
> +            goto fail;
> +         }
> +
>           lpr->img_stride[level] = lpr->row_stride[level] * nblocksy;
>        }
>  
> @@ -172,7 +178,15 @@ llvmpipe_texture_layout(struct llvmpipe_screen
> *screen,
>           }
>        }
>  
> -      total_size += lpr->num_slices_faces[level] *
> lpr->img_stride[level];
> +      /* if img_stride * num_slices_faces > LP_MAX_TEXTURE_SIZE */
> +      if (lpr->img_stride[level] >
> +          LP_MAX_TEXTURE_SIZE / lpr->num_slices_faces[level]) {
> +         /* volume too large */
> +         goto fail;
> +      }
> +
> +      total_size += (uint64_t) lpr->num_slices_faces[level]
> +                  * (uint64_t) lpr->img_stride[level];
>        if (total_size > LP_MAX_TEXTURE_SIZE) {
>           goto fail;
>        }
> --
> 1.7.3.4
> 
> _______________________________________________
> mesa-dev mailing list
> mesa-dev at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/mesa-dev
>