[Mesa-dev] [PATCH] linker: Reject shaders that use too many varyings

[Ian Romanick]

From: Ian Romanick <ian.d.romanick at intel.com>

Previously it was up to the driver or later code generator to reject
these shaders.  It turns out that nobody did this.

This will need changes to support geometry shaders.

NOTE: This is a candidate for the stable branches.

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=37743
---
 src/glsl/linker.cpp |   55 ++++++++++++++++++++++++++++++++++++++++++++++----
 1 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/src/glsl/linker.cpp b/src/glsl/linker.cpp
index 255edc6..eaa6412 100644
--- a/src/glsl/linker.cpp
+++ b/src/glsl/linker.cpp
@@ -1405,8 +1405,9 @@ demote_shader_inputs_and_outputs(gl_shader *sh, enum ir_variable_mode mode)
 }
 
 
-void
-assign_varying_locations(struct gl_shader_program *prog,
+bool
+assign_varying_locations(struct gl_context *ctx,
+			 struct gl_shader_program *prog,
 			 gl_shader *producer, gl_shader *consumer)
 {
    /* FINISHME: Set dynamically when geometry shader support is added. */
@@ -1462,6 +1463,9 @@ assign_varying_locations(struct gl_shader_program *prog,
       }
    }
 
+   unsigned float_components = 0;
+   unsigned varying_vectors = 0;
+
    foreach_list(node, consumer->ir) {
       ir_variable *const var = ((ir_instruction *) node)->as_variable();
 
@@ -1492,8 +1496,45 @@ assign_varying_locations(struct gl_shader_program *prog,
 	  * value is written by the previous stage.
 	  */
 	 var->mode = ir_var_auto;
+      } else {
+	 if (var->type->is_array()) {
+	    if (var->type->is_matrix()) {
+	       varying_vectors += var->type->array_size()
+		  * var->type->matrix_columns;
+	       float_components += var->type->array_size()
+		  * var->type->matrix_columns * 4;
+	    } else {
+	       varying_vectors += var->type->array_size();
+	       float_components += var->type->array_size() * 4;
+	    }
+	 } else if (var->type->is_matrix()) {
+	    varying_vectors += var->type->matrix_columns;
+	    float_components += var->type->matrix_columns * 4;
+	 } else {
+	    varying_vectors++;
+	    float_components += var->type->vector_elements;
+	 }
+      }
+   }
+
+   printf("varyings = %u, %u\n", varying_vectors, float_components);
+   if (ctx->API == API_OPENGLES2 || prog->Version == 100) {
+      if (varying_vectors > ctx->Const.MaxVarying) {
+	 linker_error_printf(prog, "shader uses too many varying vectors "
+			     "(%u > %u)\n",
+			     varying_vectors, ctx->Const.MaxVarying);
+	 return false;
+      }
+   } else {
+      if (float_components > ctx->Const.MaxVarying * 4) {
+	 linker_error_printf(prog, "shader uses too many varying components "
+			     "(%u > %u)\n",
+			     float_components, ctx->Const.MaxVarying * 4);
+	 return false;
       }
    }
+
+   return true;
 }
 
 
@@ -1666,9 +1707,13 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
       if (prog->_LinkedShaders[i] == NULL)
 	 continue;
 
-      assign_varying_locations(prog,
-			       prog->_LinkedShaders[prev],
-			       prog->_LinkedShaders[i]);
+      if (!assign_varying_locations(ctx, prog,
+				    prog->_LinkedShaders[prev],
+				    prog->_LinkedShaders[i])) {
+	 prog->LinkStatus = false;
+	 goto done;
+      }
+
       prev = i;
    }
 
-- 
1.7.5.2