[Mesa-dev] [PATCH V2] mesa: Fix GL_LUMINANCE handling for textures in glGetTexImage

[Anuj Phogat]

We need to rebase colors (ex: set G=B=0) when getting GL_LUMINANCE
textures in following cases:
1. If the luminance texture is actually stored as rgba
2. If getting a luminance texture, but returning rgba
3. If getting an rgba texture, but returning luminance

A similar fix was pushed by Brian Paul for uncompressed textures
in commit: f5d0ced.
Fixes https://bugs.freedesktop.org/show_bug.cgi?id=47220

Observed no regressions in piglit and ogles2conform due to this fix.
This patch will cause failures in intel oglconform pxconv-gettex,
pxstore-gettex and pxtrans-gettex test cases. The cause of failures
is a bug in test cases. Expected luminance value is calculted
incorrectly in test cases: L = R+G+B.

V2: Set G = 0 when getting a RG texture but returning luminance.

Note: This is a candidate for stable branches.

Signed-off-by: Anuj Phogat <anuj.phogat at gmail.com>
Reviewed-by: Ian Romanick <idr at freedesktop.org>
---
 src/mesa/drivers/common/meta.c |   17 ++++++++++++++---
 src/mesa/main/texgetimage.c    |   38 +++++++++++++++++++++++++++-----------
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/src/mesa/drivers/common/meta.c b/src/mesa/drivers/common/meta.c
index 417dbd0..955259d 100644
--- a/src/mesa/drivers/common/meta.c
+++ b/src/mesa/drivers/common/meta.c
@@ -3685,6 +3685,7 @@ decompress_texture_image(struct gl_context *ctx,
    /* read pixels from renderbuffer */
    {
       GLenum baseTexFormat = texImage->_BaseFormat;
+      GLenum destBaseFormat = _mesa_base_tex_format(ctx, destFormat);
 
       /* The pixel transfer state will be set to default values at this point
        * (see MESA_META_PIXEL_TRANSFER) so pixel transfer ops are effectively
@@ -3693,9 +3694,19 @@ decompress_texture_image(struct gl_context *ctx,
        * returned as red and two-channel texture values are returned as
        * red/alpha.
        */
-      if (baseTexFormat == GL_LUMINANCE ||
-          baseTexFormat == GL_LUMINANCE_ALPHA ||
-          baseTexFormat == GL_INTENSITY) {
+      if ((baseTexFormat == GL_LUMINANCE ||
+           baseTexFormat == GL_LUMINANCE_ALPHA ||
+           baseTexFormat == GL_INTENSITY) ||
+          /* If we're reading back an RGB(A) texture (using glGetTexImage) as
+	   * luminance then we need to return L=tex(R).
+	   */
+          ((baseTexFormat == GL_RGBA ||
+            baseTexFormat == GL_RGB  ||
+            baseTexFormat == GL_RG) &&
+          (destBaseFormat == GL_LUMINANCE ||
+           destBaseFormat == GL_LUMINANCE_ALPHA ||
+           destBaseFormat == GL_LUMINANCE_INTEGER_EXT ||
+           destBaseFormat == GL_LUMINANCE_ALPHA_INTEGER_EXT))) {
          /* Green and blue must be zero */
          _mesa_PixelTransferf(GL_GREEN_SCALE, 0.0f);
          _mesa_PixelTransferf(GL_BLUE_SCALE, 0.0f);
diff --git a/src/mesa/main/texgetimage.c b/src/mesa/main/texgetimage.c
index 2ccdceb..69067df 100644
--- a/src/mesa/main/texgetimage.c
+++ b/src/mesa/main/texgetimage.c
@@ -229,6 +229,8 @@ get_tex_rgba_compressed(struct gl_context *ctx, GLuint dimensions,
    const gl_format texFormat =
       _mesa_get_srgb_format_linear(texImage->TexFormat);
    const GLenum baseFormat = _mesa_get_format_base_format(texFormat);
+   const GLenum destBaseFormat = _mesa_base_tex_format(ctx, format);
+   GLenum rebaseFormat = GL_NONE;
    const GLuint width = texImage->Width;
    const GLuint height = texImage->Height;
    const GLuint depth = texImage->Depth;
@@ -266,9 +268,31 @@ get_tex_rgba_compressed(struct gl_context *ctx, GLuint dimensions,
    }
 
    if (baseFormat == GL_LUMINANCE ||
+       baseFormat == GL_INTENSITY ||
        baseFormat == GL_LUMINANCE_ALPHA) {
+      /* If a luminance (or intensity) texture is read back as RGB(A), the
+       * returned value should be (L,0,0,1), not (L,L,L,1).  Set rebaseFormat
+       * here to get G=B=0.
+       */
+      rebaseFormat = texImage->_BaseFormat;
+   }
+   else if ((baseFormat == GL_RGBA ||
+             baseFormat == GL_RGB  ||
+             baseFormat == GL_RG) &&
+            (destBaseFormat == GL_LUMINANCE ||
+             destBaseFormat == GL_LUMINANCE_ALPHA ||
+             destBaseFormat == GL_LUMINANCE_INTEGER_EXT ||
+             destBaseFormat == GL_LUMINANCE_ALPHA_INTEGER_EXT)) {
+      /* If we're reading back an RGB(A) texture as luminance then we need
+       * to return L=tex(R).  Note, that's different from glReadPixels which
+       * returns L=R+G+B.
+       */
+      rebaseFormat = GL_LUMINANCE_ALPHA; /* this covers GL_LUMINANCE too */
+   }
+
+   if (rebaseFormat) {
       _mesa_rebase_rgba_float(width * height, (GLfloat (*)[4]) tempImage,
-                              baseFormat);
+                              rebaseFormat);
    }
 
    srcRow = tempImage;
@@ -332,7 +356,8 @@ get_tex_rgba_uncompressed(struct gl_context *ctx, GLuint dimensions,
       rebaseFormat = texImage->_BaseFormat;
    }
    else if ((texImage->_BaseFormat == GL_RGBA ||
-             texImage->_BaseFormat == GL_RGB) &&
+             texImage->_BaseFormat == GL_RGB ||
+             texImage->_BaseFormat == GL_RG) &&
             (destBaseFormat == GL_LUMINANCE ||
              destBaseFormat == GL_LUMINANCE_ALPHA ||
              destBaseFormat == GL_LUMINANCE_INTEGER_EXT ||
@@ -419,15 +444,6 @@ get_tex_rgba(struct gl_context *ctx, GLuint dimensions,
          transferOps |= IMAGE_CLAMP_BIT;
       }
    }
-   /* This applies to RGB, RGBA textures. if the format is either LUMINANCE
-    * or LUMINANCE ALPHA, luminance (L) is computed as L=R+G+B .we need to
-    * clamp the sum to [0,1].
-    */
-   else if ((format == GL_LUMINANCE ||
-            format == GL_LUMINANCE_ALPHA) &&
-            dataType == GL_UNSIGNED_NORMALIZED) {
-      transferOps |= IMAGE_CLAMP_BIT;
-   }
 
    if (_mesa_is_format_compressed(texImage->TexFormat)) {
       get_tex_rgba_compressed(ctx, dimensions, format, type,
-- 
1.7.7.6