[Mesa-dev] [PATCH 3/3] mesa: Reorder some tests in update_array_format()

[Fredrik Höglund]

GL_INVALID_OPERATION should take precedence over GL_INVALID_VALUE
when there is a size error with a packed type.
---
 src/mesa/main/varray.c | 59 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 17 deletions(-)

diff --git a/src/mesa/main/varray.c b/src/mesa/main/varray.c
index 3571f7d..a062639 100644
--- a/src/mesa/main/varray.c
+++ b/src/mesa/main/varray.c
@@ -66,6 +66,9 @@
 #define UNSIGNED_INT_10F_11F_11F_REV_BIT  (1 << 14)
 #define ALL_TYPE_BITS                    ((1 << 15) - 1)
 
+#define PACKED_TYPES_MASK (UNSIGNED_INT_2_10_10_10_REV_BIT | \
+                           INT_2_10_10_10_REV_BIT | \
+                           UNSIGNED_INT_10F_11F_11F_REV_BIT)
 
 /** Convert GL datatype enum into a <type>_BIT value seen above */
 static GLbitfield
@@ -288,9 +291,16 @@ update_array_format(struct gl_context *ctx,
     * If sizeMax = BGRA_OR_4 it means that size = GL_BGRA is legal and
     * must be handled specially.
     */
-   if (ctx->Extensions.EXT_vertex_array_bgra &&
-       sizeMax == BGRA_OR_4 &&
-       size == GL_BGRA) {
+   if (size == GL_BGRA) {
+      if (sizeMax != BGRA_OR_4 || !ctx->Extensions.EXT_vertex_array_bgra) {
+         /* A size error with a packed type is an invalid operation */
+         const GLenum error = typeBit & PACKED_TYPES_MASK ?
+            GL_INVALID_OPERATION : GL_INVALID_VALUE;
+
+         _mesa_error(ctx, error, "%s(size = GL_BGRA)", func);
+         return false;
+      }
+
       /* Page 298 of the PDF of the OpenGL 4.3 (Core Profile) spec says:
        *
        * "An INVALID_OPERATION error is generated under any of the following
@@ -317,16 +327,36 @@ update_array_format(struct gl_context *ctx,
 
       format = GL_BGRA;
       size = 4;
-   }
-   else if (size < sizeMin || size > sizeMax || size > 4) {
-      _mesa_error(ctx, GL_INVALID_VALUE, "%s(size=%d)", func, size);
-      return false;
-   }
+   } else {
+      /* Page 335 of the PDF of the OpenGL 4.5 (Core Profile) spec says:
+       *
+       * "An INVALID_OPERATION error is generated under any of the following
+       *  conditions:
+       *
+       *    • type is INT_2_10_10_10_REV or UNSIGNED_INT_2_10_10_10_REV,
+       *      and size is neither 4 nor BGRA;
+       *    • type is UNSIGNED_INT_10F_11F_11F_REV and size is not 3;"
+       */
+      if (typeBit & (UNSIGNED_INT_2_10_10_10_REV_BIT |
+                     INT_2_10_10_10_REV_BIT) && size != 4) {
+         _mesa_error(ctx, GL_INVALID_OPERATION, "%s(size=%d)", func, size);
+         return false;
+      }
 
-   if (typeBit & (UNSIGNED_INT_2_10_10_10_REV_BIT |
-                  INT_2_10_10_10_REV_BIT) && size != 4) {
-      _mesa_error(ctx, GL_INVALID_OPERATION, "%s(size=%d)", func, size);
-      return false;
+      if (type == GL_UNSIGNED_INT_10F_11F_11F_REV && size != 3) {
+         _mesa_error(ctx, GL_INVALID_OPERATION, "%s(size=%d)", func, size);
+         return false;
+      }
+
+      /* Page 335 of the PDF of the OpenGL 4.5 (Core Profile) spec says:
+       *
+       * "An INVALID_VALUE error is generated if size is not one of the values
+       *  shown in table 10.2 for the corresponding command."
+       */
+      if (size < sizeMin || size > sizeMax || size > 4) {
+         _mesa_error(ctx, GL_INVALID_VALUE, "%s(size = GL_BGRA)", func);
+         return false;
+      }
    }
 
    /* The ARB_vertex_attrib_binding_spec says:
@@ -342,11 +372,6 @@ update_array_format(struct gl_context *ctx,
       return false;
    }
 
-   if (type == GL_UNSIGNED_INT_10F_11F_11F_REV && size != 3) {
-      _mesa_error(ctx, GL_INVALID_OPERATION, "%s(size=%d)", func, size);
-      return false;
-   }
-
    assert(size <= 4);
 
    elementSize = _mesa_bytes_per_vertex_attrib(size, type);
-- 
1.8.5.3