[Mesa-dev] [PATCH] mesa: Fix an issue with texture border in strip_texture_border()

[Anuj Phogat]

Border only applies to the width for a 1D texture array and for 2D
texture array it applies to the width and height but not to the
depth.  This was not handled correctly in strip_texture_border().

v2: height is also affected by border if target is GL_TEXTURE_3D
v3: simplified the logic to strip texture dimensions:
    As the legal_teximage_target() is called earlier in teximage()
    function, we don't need to repeat that here. Also assuming that
    strip_texture_border() will not be used for proxy textures
    otherwise we have to include proxy targets as well.

Note: This is a candidate for stable branches

Signed-off-by: Anuj Phogat <anuj.phogat at gmail.com>
---
 src/mesa/main/teximage.c |   28 ++++++++++++++++++----------
 1 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/mesa/main/teximage.c b/src/mesa/main/teximage.c
index 5328ae2..1398329 100644
--- a/src/mesa/main/teximage.c
+++ b/src/mesa/main/teximage.c
@@ -2426,7 +2426,7 @@ _mesa_choose_texture_format(struct gl_context *ctx,
  * \param unpackNew returns the new pixel unpack parameters
  */
 static void
-strip_texture_border(GLint *border,
+strip_texture_border(GLenum target, GLint *border,
                      GLint *width, GLint *height, GLint *depth,
                      const struct gl_pixelstore_attrib *unpack,
                      struct gl_pixelstore_attrib *unpackNew)
@@ -2442,17 +2442,25 @@ strip_texture_border(GLint *border,
       unpackNew->ImageHeight = *height;
 
    unpackNew->SkipPixels += *border;
-   if (height)
-      unpackNew->SkipRows += *border;
-   if (depth)
-      unpackNew->SkipImages += *border;
 
    assert(*width >= 3);
    *width = *width - 2 * *border;
-   if (height && *height >= 3)
-      *height = *height - 2 * *border;
-   if (depth && *depth >= 3)
-      *depth = *depth - 2 * *border;
+
+   /* height is stripped for 2D, CubeMap and 3D texture targets */
+   if (target != GL_TEXTURE_1D &&
+       target != GL_TEXTURE_1D_ARRAY) {
+      if (height)
+         unpackNew->SkipRows += *border;
+      if (height && *height >= 3)
+         *height = *height - 2 * *border;
+   }
+   /* depth is stripped for 3D textures only */
+   if (target == GL_TEXTURE_3D) {
+      if (depth)
+         unpackNew->SkipImages += *border;
+      if (depth && *depth >= 3)
+         *depth = *depth - 2 * *border;
+   }
    *border = 0;
 }
 
@@ -2538,7 +2546,7 @@ teximage(struct gl_context *ctx, GLuint dims,
        * rarely-tested software fallback rendering.
        */
       if (border && ctx->Const.StripTextureBorder) {
-	 strip_texture_border(&border, &width, &height, &depth, unpack,
+	 strip_texture_border(target, &border, &width, &height, &depth, unpack,
 			      &unpack_no_border);
 	 unpack = &unpack_no_border;
       }
-- 
1.7.7.6