Mesa (7.10): linker: Reject shaders that use too many varyings

[Ian Romanick]

Module: Mesa
Branch: 7.10
Commit: 85b965b462a27d537b7d9da6e26c43516a3e67fe
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=85b965b462a27d537b7d9da6e26c43516a3e67fe

Author: Ian Romanick <ian.d.romanick at intel.com>
Date:   Thu Jun  9 13:31:32 2011 -0700

linker: Reject shaders that use too many varyings

Previously it was up to the driver or later code generator to reject
these shaders.  It turns out that nobody did this.

This will need changes to support geometry shaders.

NOTE: This is a candidate for the stable branches.

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=37743
Reviewed-by: Kenneth Graunke <kenneth at whitecape.org>
(cherry picked from commit de77324d8f14951e4dc17f570e49451a0cd33121)

---

 src/glsl/linker.cpp |   41 ++++++++++++++++++++++++++++++++++++-----
 1 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/src/glsl/linker.cpp b/src/glsl/linker.cpp
index f8d8eae..03d0d79 100644
--- a/src/glsl/linker.cpp
+++ b/src/glsl/linker.cpp
@@ -1369,8 +1369,9 @@ demote_shader_inputs_and_outputs(gl_shader *sh, enum ir_variable_mode mode)
 }
 
 
-void
-assign_varying_locations(struct gl_shader_program *prog,
+bool
+assign_varying_locations(struct gl_context *ctx,
+			 struct gl_shader_program *prog,
 			 gl_shader *producer, gl_shader *consumer)
 {
    /* FINISHME: Set dynamically when geometry shader support is added. */
@@ -1426,6 +1427,8 @@ assign_varying_locations(struct gl_shader_program *prog,
       }
    }
 
+   unsigned varying_vectors = 0;
+
    foreach_list(node, consumer->ir) {
       ir_variable *const var = ((ir_instruction *) node)->as_variable();
 
@@ -1456,8 +1459,32 @@ assign_varying_locations(struct gl_shader_program *prog,
 	  * value is written by the previous stage.
 	  */
 	 var->mode = ir_var_auto;
+      } else {
+	 /* The packing rules are used for vertex shader inputs are also used
+	  * for fragment shader inputs.
+	  */
+	 varying_vectors += count_attribute_slots(var->type);
       }
    }
+
+   if (ctx->API == API_OPENGLES2 || prog->Version == 100) {
+      if (varying_vectors > ctx->Const.MaxVarying) {
+	 linker_error_printf(prog, "shader uses too many varying vectors "
+			     "(%u > %u)\n",
+			     varying_vectors, ctx->Const.MaxVarying);
+	 return false;
+      }
+   } else {
+      const unsigned float_components = varying_vectors * 4;
+      if (float_components > ctx->Const.MaxVarying * 4) {
+	 linker_error_printf(prog, "shader uses too many varying components "
+			     "(%u > %u)\n",
+			     float_components, ctx->Const.MaxVarying * 4);
+	 return false;
+      }
+   }
+
+   return true;
 }
 
 
@@ -1630,9 +1657,13 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
       if (prog->_LinkedShaders[i] == NULL)
 	 continue;
 
-      assign_varying_locations(prog,
-			       prog->_LinkedShaders[prev],
-			       prog->_LinkedShaders[i]);
+      if (!assign_varying_locations(ctx, prog,
+				    prog->_LinkedShaders[prev],
+				    prog->_LinkedShaders[i])) {
+	 prog->LinkStatus = false;
+	 goto done;
+      }
+
       prev = i;
    }