[Cogl] [PATCH] indices: Fix cogl_get_rectangle_indices clipping bug

[Plamena Manolova]

cogl_get_rectangle_indices failed to generate enough
indices when the user requests more than 65535 of them
This is due to the function only using byte and usigned
int indices (it used short int even if the ammount of
indices requested required storing a value which exceeds
16 bits). This was fixed by using unsigned int indices
when appropriate.
---
 cogl/cogl-context-private.h |  2 ++
 cogl/cogl-context.c         |  4 ++++
 cogl/cogl-indices.c         | 53 ++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/cogl/cogl-context-private.h b/cogl/cogl-context-private.h
index cf41c88..56db02d 100644
--- a/cogl/cogl-context-private.h
+++ b/cogl/cogl-context-private.h
@@ -186,7 +186,9 @@ struct _CoglContext
      out of a vertex array of quads */
   CoglIndices      *rectangle_byte_indices;
   CoglIndices      *rectangle_short_indices;
+  CoglIndices      *rectangle_int_indices;
   int               rectangle_short_indices_len;
+  int               rectangle_int_indices_len;
 
   CoglPipeline     *texture_download_pipeline;
   CoglPipeline     *blit_texture_pipeline;
diff --git a/cogl/cogl-context.c b/cogl/cogl-context.c
index e4714be..0e391fb 100644
--- a/cogl/cogl-context.c
+++ b/cogl/cogl-context.c
@@ -335,6 +335,8 @@ cogl_context_new (CoglDisplay *display,
   context->rectangle_byte_indices = NULL;
   context->rectangle_short_indices = NULL;
   context->rectangle_short_indices_len = 0;
+  context->rectangle_int_indices = NULL;
+  context->rectangle_int_indices_len = 0;
 
   context->texture_download_pipeline = NULL;
   context->blit_texture_pipeline = NULL;
@@ -459,6 +461,8 @@ _cogl_context_free (CoglContext *context)
     cogl_object_unref (context->rectangle_byte_indices);
   if (context->rectangle_short_indices)
     cogl_object_unref (context->rectangle_short_indices);
+  if (context->rectangle_int_indices)
+    cogl_object_unref (context->rectangle_int_indices);
 
   if (context->default_pipeline)
     cogl_object_unref (context->default_pipeline);
diff --git a/cogl/cogl-indices.c b/cogl/cogl-indices.c
index 80041d1..56eaa00 100644
--- a/cogl/cogl-indices.c
+++ b/cogl/cogl-indices.c
@@ -181,6 +181,7 @@ _cogl_indices_immutable_unref (CoglIndices *indices)
 CoglIndices *
 cogl_get_rectangle_indices (CoglContext *ctx, int n_rectangles)
 {
+  CoglIndices *rectangle_indices = NULL;
   int n_indices = n_rectangles * 6;
 
   /* Check if the largest index required will fit in a byte array... */
@@ -213,9 +214,9 @@ cogl_get_rectangle_indices (CoglContext *ctx, int n_rectangles)
           g_free (byte_array);
         }
 
-      return ctx->rectangle_byte_indices;
+      rectangle_indices = ctx->rectangle_byte_indices;
     }
-  else
+  else if (n_indices <= 65535 / 4 * 6)
     {
       if (ctx->rectangle_short_indices_len < n_indices)
         {
@@ -257,7 +258,53 @@ cogl_get_rectangle_indices (CoglContext *ctx, int n_rectangles)
           g_free (short_array);
         }
 
-      return ctx->rectangle_short_indices;
+      rectangle_indices = ctx->rectangle_short_indices;
+    }
+  else if (n_indices < 4294967295 / 4 * 6 &&
+           cogl_has_feature (COGL_FEATURE_ID_UNSIGNED_INT_INDICES))
+    {
+      if (ctx->rectangle_int_indices_len < n_indices)
+        {
+          uint32_t *int_array;
+          uint32_t *p;
+          int i, vert_num = 0;
+
+          if (ctx->rectangle_int_indices != NULL)
+            cogl_object_unref (ctx->rectangle_int_indices);
+          if (ctx->rectangle_int_indices_len == 0)
+            ctx->rectangle_int_indices_len = 1024;
+          while (ctx->rectangle_int_indices_len < n_indices)
+            ctx->rectangle_int_indices_len *= 2;
+
+          p = int_array = g_malloc ((ctx->rectangle_int_indices_len
+                                     + 5) / 6 * 6
+                                    * sizeof (uint32_t));
+
+          for (i = 0; i < ctx->rectangle_int_indices_len; i += 6)
+            {
+              *(p++) = vert_num + 0;
+              *(p++) = vert_num + 1;
+              *(p++) = vert_num + 2;
+              *(p++) = vert_num + 0;
+              *(p++) = vert_num + 2;
+              *(p++) = vert_num + 3;
+              vert_num += 4;
+            }
+
+            ctx->rectangle_int_indices
+              = cogl_indices_new (ctx,
+                                  COGL_INDICES_TYPE_UNSIGNED_INT,
+                                  int_array,
+                                  ctx->rectangle_int_indices_len);
+
+            g_free (int_array);
+          }
+
+      rectangle_indices = ctx->rectangle_int_indices;
     }
+  else
+    g_warn_if_reached ();
+
+  return rectangle_indices;
 }
 
-- 
1.8.2.1

---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47

This e-mail and any attachments may contain confidential material for
the sole use of the intended recipient(s). Any review or distribution
by others is strictly prohibited. If you are not the intended
recipient, please contact the sender and delete all copies.