[Mesa-dev] [PATCH 03/16] mesa: Clamps the stencil value masks to GLint when queried

[Eduardo Lima Mitev]

Stencil value masks values (ctx->Stencil.ValueMask[]) stores GLuint values
which are initialized with max unsigned integer (~0u). When these values
are queried by glGet* (GL_STENCIL_VALUE_MASK or GL_STENCIL_BACK_VALUE_MASK),
they are converted to a signed integer. Currently, these values overflow
and return incorrect result (-1).

This patch clamps these values to max int (0x7FFFFFFF) before storing.

Fixes 6 dEQP failing tests:
* dEQP-GLES3.functional.state_query.integers.stencil_value_mask_getfloat
* dEQP-GLES3.functional.state_query.integers.stencil_back_value_mask_getfloat
* dEQP-GLES3.functional.state_query.integers.stencil_value_mask_separate_getfloat
* dEQP-GLES3.functional.state_query.integers.stencil_value_mask_separate_both_getfloat
* dEQP-GLES3.functional.state_query.integers.stencil_back_value_mask_separate_getfloat
* dEQP-GLES3.functional.state_query.integers.stencil_back_value_mask_separate_both_getfloat
---
 src/mesa/main/get.c              | 11 ++++++++++-
 src/mesa/main/get_hash_params.py |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/mesa/main/get.c b/src/mesa/main/get.c
index 6091efc..4578a36 100644
--- a/src/mesa/main/get.c
+++ b/src/mesa/main/get.c
@@ -726,7 +726,16 @@ find_custom_value(struct gl_context *ctx, const struct value_desc *d, union valu
       v->value_int = _mesa_get_stencil_ref(ctx, 1);
       break;
    case GL_STENCIL_VALUE_MASK:
-      v->value_int = ctx->Stencil.ValueMask[ctx->Stencil.ActiveFace];
+      /* Since stencil value mask is a GLuint, it requires clamping
+       * before storing in a signed int to avoid overflow.
+       * Notice that Stencil.ValueMask values are initialized to ~0u,
+       * so without clamping it will return -1 when assigned to value_int.
+       */
+      v->value_int = ctx->Stencil.ValueMask[ctx->Stencil.ActiveFace] & 0x7FFFFFFF;
+      break;
+   case GL_STENCIL_BACK_VALUE_MASK:
+      /* Same as with GL_STENCIL_VALUE_MASK, value requires claming. */
+      v->value_int = ctx->Stencil.ValueMask[1] & 0x7FFFFFFF;
       break;
    case GL_STENCIL_WRITEMASK:
       v->value_int = ctx->Stencil.WriteMask[ctx->Stencil.ActiveFace];
diff --git a/src/mesa/main/get_hash_params.py b/src/mesa/main/get_hash_params.py
index 09a61ac..a3bf1cb 100644
--- a/src/mesa/main/get_hash_params.py
+++ b/src/mesa/main/get_hash_params.py
@@ -283,7 +283,7 @@ descriptor=[
 
 # OpenGL 2.0
   [ "STENCIL_BACK_FUNC", "CONTEXT_ENUM(Stencil.Function[1]), NO_EXTRA" ],
-  [ "STENCIL_BACK_VALUE_MASK", "CONTEXT_INT(Stencil.ValueMask[1]), NO_EXTRA" ],
+  [ "STENCIL_BACK_VALUE_MASK", "LOC_CUSTOM, TYPE_INT, NO_OFFSET, NO_EXTRA"],
   [ "STENCIL_BACK_WRITEMASK", "CONTEXT_INT(Stencil.WriteMask[1]), NO_EXTRA" ],
   [ "STENCIL_BACK_REF", "LOC_CUSTOM, TYPE_INT, NO_OFFSET, NO_EXTRA" ],
   [ "STENCIL_BACK_FAIL", "CONTEXT_ENUM(Stencil.FailFunc[1]), NO_EXTRA" ],
-- 
2.1.3