[Mesa-dev] [PATCHv2 1/2] translate: deal with size overflows by casting to ptrdiff_t

[Ilia Mirkin]

This was discovered as a result of the draw-elements-base-vertex-neg
piglit test, which passes very negative offsets in, followed up by large
indices. The nouveau code correctly adjusts the pointer, but the
translate code needs to do the proper inverse correction. Similarly fix
up the SSE code to do a 64-bit multiply to compute the proper offset.

Signed-off-by: Ilia Mirkin <imirkin at alum.mit.edu>
Reviewed-by: Brian Paul <brianp at vmware.com>
---
 src/gallium/auxiliary/translate/translate_generic.c | 2 +-
 src/gallium/auxiliary/translate/translate_sse.c     | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/gallium/auxiliary/translate/translate_generic.c b/src/gallium/auxiliary/translate/translate_generic.c
index 5bf97db..5ffce32 100644
--- a/src/gallium/auxiliary/translate/translate_generic.c
+++ b/src/gallium/auxiliary/translate/translate_generic.c
@@ -638,7 +638,7 @@ static ALWAYS_INLINE void PIPE_CDECL generic_run_one( struct translate_generic *
          }
 
          src = tg->attrib[attr].input_ptr +
-               tg->attrib[attr].input_stride * index;
+               (ptrdiff_t)tg->attrib[attr].input_stride * index;
 
          copy_size = tg->attrib[attr].copy_size;
          if(likely(copy_size >= 0))
diff --git a/src/gallium/auxiliary/translate/translate_sse.c b/src/gallium/auxiliary/translate/translate_sse.c
index a78ea91..a72454a 100644
--- a/src/gallium/auxiliary/translate/translate_sse.c
+++ b/src/gallium/auxiliary/translate/translate_sse.c
@@ -1121,7 +1121,9 @@ static boolean init_inputs( struct translate_sse *p,
             x86_cmovcc(p->func, tmp_EAX, buf_max_index, cc_AE);
          }
 
-         x86_imul(p->func, tmp_EAX, buf_stride);
+         x86_mov(p->func, p->tmp2_EDX, buf_stride);
+         x64_rexw(p->func);
+         x86_imul(p->func, tmp_EAX, p->tmp2_EDX);
          x64_rexw(p->func);
          x86_add(p->func, tmp_EAX, buf_base_ptr);
 
@@ -1207,7 +1209,9 @@ static struct x86_reg get_buffer_ptr( struct translate_sse *p,
       x86_cmp(p->func, ptr, buf_max_index);
       x86_cmovcc(p->func, ptr, buf_max_index, cc_AE);
 
-      x86_imul(p->func, ptr, buf_stride);
+      x86_mov(p->func, p->tmp2_EDX, buf_stride);
+      x64_rexw(p->func);
+      x86_imul(p->func, ptr, p->tmp2_EDX);
       x64_rexw(p->func);
       x86_add(p->func, ptr, buf_base_ptr);
       return ptr;
-- 
1.8.3.2