[Mesa-dev] [PATCH v2] mesa: fix type comparison errors in sub-texture error checking code

[Brian Paul]

On 04/18/2013 12:21 AM, Tapani Pälli wrote:
> patch fixes a crash that happens if glTexSubImage2D is called with a
> negative xoffset.
>
> NOTE: This is a candidate for stable branches.
>
> Signed-off-by: Tapani Pälli<tapani.palli at intel.com>
> ---
>
> v2: corrected xoffset cast to happen before sign + added more checks
>
>   src/mesa/main/teximage.c | 10 +++++-----
>   1 file changed, 5 insertions(+), 5 deletions(-)
>
> diff --git a/src/mesa/main/teximage.c b/src/mesa/main/teximage.c
> index 784b389..1a144e2 100644
> --- a/src/mesa/main/teximage.c
> +++ b/src/mesa/main/teximage.c
> @@ -1536,13 +1536,13 @@ error_check_subtexture_dimensions(struct gl_context *ctx,
>      }
>
>      /* check xoffset and width */
> -   if (xoffset<  -destImage->Border) {
> +   if (xoffset<  - (GLint) destImage->Border) {
>         _mesa_error(ctx, GL_INVALID_VALUE, "%s%dD(xoffset)",
>                     function, dims);
>         return GL_TRUE;
>      }
>
> -   if (xoffset + subWidth>  destImage->Width) {
> +   if (xoffset + subWidth>  (GLint) destImage->Width) {
>         _mesa_error(ctx, GL_INVALID_VALUE, "%s%dD(xoffset+width)",
>                     function, dims);
>         return GL_TRUE;
> @@ -1556,7 +1556,7 @@ error_check_subtexture_dimensions(struct gl_context *ctx,
>                        function, dims);
>            return GL_TRUE;
>         }
> -      if (yoffset + subHeight>  destImage->Height) {
> +      if (yoffset + subHeight>  (GLint) destImage->Height) {
>            _mesa_error(ctx, GL_INVALID_VALUE, "%s%dD(yoffset+height)",
>                        function, dims);
>            return GL_TRUE;
> @@ -1595,13 +1595,13 @@ error_check_subtexture_dimensions(struct gl_context *ctx,
>         }
>
>         /* size must be multiple of bw by bh or equal to whole texture size */
> -      if ((subWidth % bw != 0)&&  subWidth != destImage->Width) {
> +      if ((subWidth % bw != 0)&&  subWidth != (GLint) destImage->Width) {
>            _mesa_error(ctx, GL_INVALID_OPERATION,
>                        "%s%dD(width = %d)", function, dims, subWidth);
>            return GL_TRUE;
>         }
>
> -      if ((subHeight % bh != 0)&&  subHeight != destImage->Height) {
> +      if ((subHeight % bh != 0)&&  subHeight != (GLint) destImage->Height) {
>            _mesa_error(ctx, GL_INVALID_OPERATION,
>                        "%s%dD(height = %d)", function, dims, subHeight);
>            return GL_TRUE;

Reviewed-by: Brian Paul <brianp at vmware.com>