[Pixman] [PATCH] In pixman_image_create_bits() allow images larger than 2GB

[Søren Sandmann]

From: Søren Sandmann Pedersen <ssp at redhat.com>

There is no reason for pixman_image_create_bits() to check that the
image size fits in int32_t. The correct check is against size_t since
that is what the argument to calloc() is.

This patch fixes this by adding a new
_pixman_multiply_overflows_size() and using it in create_bits(). Also
prepend an underscore to the names of other similar functions since
they are internal to pixman.
---
 pixman/pixman-bits-image.c |   14 +++++++-------
 pixman/pixman-private.h    |    7 +++++--
 pixman/pixman-utils.c      |   12 ++++++++----
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/pixman/pixman-bits-image.c b/pixman/pixman-bits-image.c
index 4e9ed14..d8abeac 100644
--- a/pixman/pixman-bits-image.c
+++ b/pixman/pixman-bits-image.c
@@ -1488,12 +1488,12 @@ _pixman_bits_image_dest_iter_init (pixman_image_t *image, pixman_iter_t *iter)
 
 static uint32_t *
 create_bits (pixman_format_code_t format,
-             int                  width,
-             int                  height,
-             int *                rowstride_bytes)
+             ssize_t              width,
+             ssize_t              height,
+             int *		  rowstride_bytes)
 {
     int stride;
-    int buf_size;
+    size_t buf_size;
     int bpp;
 
     /* what follows is a long-winded way, avoiding any possibility of integer
@@ -1502,11 +1502,11 @@ create_bits (pixman_format_code_t format,
      */
 
     bpp = PIXMAN_FORMAT_BPP (format);
-    if (pixman_multiply_overflows_int (width, bpp))
+    if (_pixman_multiply_overflows_int (width, bpp))
 	return NULL;
 
     stride = width * bpp;
-    if (pixman_addition_overflows_int (stride, 0x1f))
+    if (_pixman_addition_overflows_int (stride, 0x1f))
 	return NULL;
 
     stride += 0x1f;
@@ -1514,7 +1514,7 @@ create_bits (pixman_format_code_t format,
 
     stride *= sizeof (uint32_t);
 
-    if (pixman_multiply_overflows_int (height, stride))
+    if (_pixman_multiply_overflows_size (height, stride))
 	return NULL;
 
     buf_size = height * stride;
diff --git a/pixman/pixman-private.h b/pixman/pixman-private.h
index 6a3935e..a25897d 100644
--- a/pixman/pixman-private.h
+++ b/pixman/pixman-private.h
@@ -691,10 +691,13 @@ void *
 pixman_malloc_abc (unsigned int a, unsigned int b, unsigned int c);
 
 pixman_bool_t
-pixman_multiply_overflows_int (unsigned int a, unsigned int b);
+_pixman_multiply_overflows_size (size_t a, size_t b);
 
 pixman_bool_t
-pixman_addition_overflows_int (unsigned int a, unsigned int b);
+_pixman_multiply_overflows_int (unsigned int a, unsigned int b);
+
+pixman_bool_t
+_pixman_addition_overflows_int (unsigned int a, unsigned int b);
 
 /* Compositing utilities */
 void
diff --git a/pixman/pixman-utils.c b/pixman/pixman-utils.c
index cb4e621..49e3488 100644
--- a/pixman/pixman-utils.c
+++ b/pixman/pixman-utils.c
@@ -31,15 +31,19 @@
 #include "pixman-private.h"
 
 pixman_bool_t
-pixman_multiply_overflows_int (unsigned int a,
-                               unsigned int b)
+_pixman_multiply_overflows_size (size_t a, size_t b)
+{
+    return a >= SIZE_MAX / b;
+}
+
+pixman_bool_t
+_pixman_multiply_overflows_int (unsigned int a, unsigned int b)
 {
     return a >= INT32_MAX / b;
 }
 
 pixman_bool_t
-pixman_addition_overflows_int (unsigned int a,
-                               unsigned int b)
+_pixman_addition_overflows_int (unsigned int a, unsigned int b)
 {
     return a > INT32_MAX - b;
 }
-- 
1.7.4