[Mesa-dev] [PATCH] swrast: fix 32-bit Z overflow bug

[Brian Paul]

When using a 32-bit Z buffer the float->uint conversion could overflow
and wrap around if the float value was too large.  Add a clamp.

Fixes piglit fbo-readpixels-depth-formats failure.
---
 src/mesa/swrast/s_tritemp.h |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mesa/swrast/s_tritemp.h b/src/mesa/swrast/s_tritemp.h
index 061759d..f870dde 100644
--- a/src/mesa/swrast/s_tritemp.h
+++ b/src/mesa/swrast/s_tritemp.h
@@ -624,8 +624,9 @@ static void NAME(struct gl_context *ctx, const SWvertex *v0,
                   }
                   else {
                      /* interpolate depth values w/out scaling */
-                     zLeft = (GLuint) (z0 + span.attrStepX[FRAG_ATTRIB_WPOS][2] * FixedToFloat(adjx)
-                                          + span.attrStepY[FRAG_ATTRIB_WPOS][2] * FixedToFloat(adjy));
+                     float z = (z0 + span.attrStepX[FRAG_ATTRIB_WPOS][2] * FixedToFloat(adjx)
+                                   + span.attrStepY[FRAG_ATTRIB_WPOS][2] * FixedToFloat(adjy));
+                     zLeft = (z > 4294967295.0) ? 0xffffffff : (GLuint) z;
                      fdzOuter = (GLint) (span.attrStepY[FRAG_ATTRIB_WPOS][2] +
                                          dxOuter * span.attrStepX[FRAG_ATTRIB_WPOS][2]);
                   }
-- 
1.7.3.4