[Mesa-dev] [PATCH] mesa: Fix an issue with texture border in strip_texture_border()

[Anuj Phogat]

Border only applies to the width for a 1D texture array and for a 2D
texture array the it applies to the width and height but not to the
depth.  This was not handled correctly in strip_texture_border().

v2: height is also affected by border if target is GL_TEXTURE_3D

Note: This is a candidate for stable branches

Signed-off-by: Anuj Phogat <anuj.phogat at gmail.com>
---
A piglit test case for this patch will be posted on piglit mailing list.

 src/mesa/main/teximage.c |   57 +++++++++++++++++++++++++++++++++++++--------
 1 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/src/mesa/main/teximage.c b/src/mesa/main/teximage.c
index 5328ae2..b8ff67e 100644
--- a/src/mesa/main/teximage.c
+++ b/src/mesa/main/teximage.c
@@ -2426,7 +2426,7 @@ _mesa_choose_texture_format(struct gl_context *ctx,
  * \param unpackNew returns the new pixel unpack parameters
  */
 static void
-strip_texture_border(GLint *border,
+strip_texture_border(GLenum target, GLint *border,
                      GLint *width, GLint *height, GLint *depth,
                      const struct gl_pixelstore_attrib *unpack,
                      struct gl_pixelstore_attrib *unpackNew)
@@ -2442,17 +2442,54 @@ strip_texture_border(GLint *border,
       unpackNew->ImageHeight = *height;
 
    unpackNew->SkipPixels += *border;
-   if (height)
-      unpackNew->SkipRows += *border;
-   if (depth)
-      unpackNew->SkipImages += *border;
 
    assert(*width >= 3);
    *width = *width - 2 * *border;
-   if (height && *height >= 3)
-      *height = *height - 2 * *border;
-   if (depth && *depth >= 3)
-      *depth = *depth - 2 * *border;
+
+   switch(target) {
+   case GL_TEXTURE_1D:
+   case GL_TEXTURE_BUFFER:
+   case GL_PROXY_TEXTURE_1D:
+   case GL_TEXTURE_1D_ARRAY:
+   case GL_PROXY_TEXTURE_1D_ARRAY:
+      break;
+
+   case GL_TEXTURE_2D:
+   case GL_TEXTURE_RECTANGLE:
+   case GL_TEXTURE_CUBE_MAP_POSITIVE_X:
+   case GL_TEXTURE_CUBE_MAP_NEGATIVE_X:
+   case GL_TEXTURE_CUBE_MAP_POSITIVE_Y:
+   case GL_TEXTURE_CUBE_MAP_NEGATIVE_Y:
+   case GL_TEXTURE_CUBE_MAP_POSITIVE_Z:
+   case GL_TEXTURE_CUBE_MAP_NEGATIVE_Z:
+   case GL_TEXTURE_EXTERNAL_OES:
+   case GL_PROXY_TEXTURE_2D:
+   case GL_PROXY_TEXTURE_RECTANGLE:
+   case GL_PROXY_TEXTURE_CUBE_MAP:
+   case GL_TEXTURE_2D_ARRAY:
+   case GL_PROXY_TEXTURE_2D_ARRAY:
+      if (height)
+         unpackNew->SkipRows += *border;
+      if (height && *height >= 3)
+         *height = *height - 2 * *border;
+      break;
+
+   case GL_TEXTURE_3D:
+   case GL_PROXY_TEXTURE_3D:
+      if (height)
+         unpackNew->SkipRows += *border;
+      if (height && *height >= 3)
+         *height = *height - 2 * *border;
+      if (depth)
+         unpackNew->SkipImages += *border;
+      if (depth && *depth >= 3)
+         *depth = *depth - 2 * *border;
+     break;
+
+   default:
+      _mesa_problem(NULL, "invalid target 0x%x in strip_texture_border()",
+                    target);
+   }
    *border = 0;
 }
 
@@ -2538,7 +2575,7 @@ teximage(struct gl_context *ctx, GLuint dims,
        * rarely-tested software fallback rendering.
        */
       if (border && ctx->Const.StripTextureBorder) {
-	 strip_texture_border(&border, &width, &height, &depth, unpack,
+	 strip_texture_border(target, &border, &width, &height, &depth, unpack,
 			      &unpack_no_border);
 	 unpack = &unpack_no_border;
       }
-- 
1.7.7.6