[Pixman] [PATCH 3/5] Fix a bunch of signed overflow issues

[Søren Sandmann]

From: Søren Sandmann Pedersen <ssp at redhat.com>

In pixman-fast-path.c: (1 << 31) - 1 causes a signed overflow, so
change to (1U << n) - 1.

In pixman-image.c: The check for whether m10 == -m01 will overflow
when -m01 == INT_MIN. Instead just check whether the variables are 1
and -1.

In pixman-utils.c: When the depth of the topmost channel is 0, we can
end up shifting by 32.

In blitters-test.c: Replicating the mask would end up shifting more
than 32.

In region-contains-test.c: Computing the average of two large integers
could overflow. Instead add half the difference between them to the
first integer.

In stress-test.c: Masking the value in fake_reader() would sometimes
shift by 32. Instead just use the most significant bits instead of
the least significant.

All these issues were found by the IOC tool:

    http://embed.cs.utah.edu/ioc/
---
 pixman/pixman-fast-path.c   |    4 ++--
 pixman/pixman-image.c       |   13 ++++++-------
 pixman/pixman-utils.c       |   35 ++++++++++++++++++++++++++---------
 test/blitters-test.c        |    2 +-
 test/region-contains-test.c |    4 ++--
 test/stress-test.c          |    3 ++-
 6 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/pixman/pixman-fast-path.c b/pixman/pixman-fast-path.c
index 038dcf7..eac8dea 100644
--- a/pixman/pixman-fast-path.c
+++ b/pixman/pixman-fast-path.c
@@ -1969,9 +1969,9 @@ static const pixman_fast_path_t c_fast_paths[] =
 };
 
 #ifdef WORDS_BIGENDIAN
-#define A1_FILL_MASK(n, offs) (((1 << (n)) - 1) << (32 - (offs) - (n)))
+#define A1_FILL_MASK(n, offs) (((1U << (n)) - 1) << (32 - (offs) - (n)))
 #else
-#define A1_FILL_MASK(n, offs) (((1 << (n)) - 1) << (offs))
+#define A1_FILL_MASK(n, offs) (((1U << (n)) - 1) << (offs))
 #endif
 
 static force_inline void
diff --git a/pixman/pixman-image.c b/pixman/pixman-image.c
index 913853c..06eb08f 100644
--- a/pixman/pixman-image.c
+++ b/pixman/pixman-image.c
@@ -299,13 +299,12 @@ compute_image_info (pixman_image_t *image)
 	             image->common.transform->matrix[1][1] == 0)
 	    {
 		pixman_fixed_t m01 = image->common.transform->matrix[0][1];
-		if (m01 == -image->common.transform->matrix[1][0])
-		{
-			if (m01 == -pixman_fixed_1)
-			    flags |= FAST_PATH_ROTATE_90_TRANSFORM;
-			else if (m01 == pixman_fixed_1)
-			    flags |= FAST_PATH_ROTATE_270_TRANSFORM;
-		}
+		pixman_fixed_t m10 = image->common.transform->matrix[1][0];
+
+		if (m01 == -1 && m10 == 1)
+		    flags |= FAST_PATH_ROTATE_90_TRANSFORM;
+		else if (m01 == 1 && m10 == -1)
+		    flags |= FAST_PATH_ROTATE_270_TRANSFORM;
 	    }
 	}
 
diff --git a/pixman/pixman-utils.c b/pixman/pixman-utils.c
index d2af51a..2ec2594 100644
--- a/pixman/pixman-utils.c
+++ b/pixman/pixman-utils.c
@@ -220,16 +220,33 @@ pixman_expand (uint64_t *           dst,
     for (i = width - 1; i >= 0; i--)
     {
 	const uint32_t pixel = src[i];
-	const uint8_t a = (pixel >> a_shift) & a_mask,
-	              r = (pixel >> r_shift) & r_mask,
-	              g = (pixel >> g_shift) & g_mask,
-	              b = (pixel >> b_shift) & b_mask;
-	const uint64_t
-	    a16 = a_size ? unorm_to_unorm (a, a_size, 16) : 0xffff,
-	    r16 = unorm_to_unorm (r, r_size, 16),
-	    g16 = unorm_to_unorm (g, g_size, 16),
-	    b16 = unorm_to_unorm (b, b_size, 16);
+	uint8_t a, r, g, b;
+	uint64_t a16, r16, g16, b16;
+
+	if (a_size)
+	{
+	    a = (pixel >> a_shift) & a_mask;
+	    a16 = unorm_to_unorm (a, a_size, 16);
+	}
+	else
+	{
+	    a16 = 0xffff;
+	}
 
+	if (r_size)
+	{
+	    r = (pixel >> r_shift) & r_mask;
+	    g = (pixel >> g_shift) & g_mask;
+	    b = (pixel >> b_shift) & b_mask;
+	    r16 = unorm_to_unorm (r, r_size, 16);
+	    g16 = unorm_to_unorm (g, g_size, 16);
+	    b16 = unorm_to_unorm (b, b_size, 16);
+	}
+	else
+	{
+	    r16 = g16 = b16 = 0;
+	}
+	
 	dst[i] = a16 << 48 | r16 << 32 | g16 << 16 | b16;
     }
 }
diff --git a/test/blitters-test.c b/test/blitters-test.c
index 63162e6..fd62c67 100644
--- a/test/blitters-test.c
+++ b/test/blitters-test.c
@@ -103,7 +103,7 @@ free_random_image (uint32_t initcrc,
 		mask <<= (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt));
 	    }
 
-	    for (i = 0; i < 32; i++)
+	    for (i = 0; i * PIXMAN_FORMAT_BPP (fmt) < 32; i++)
 		mask |= mask << (i * PIXMAN_FORMAT_BPP (fmt));
 
 	    for (i = 0; i < stride * height / 4; i++)
diff --git a/test/region-contains-test.c b/test/region-contains-test.c
index 2372686..9524e28 100644
--- a/test/region-contains-test.c
+++ b/test/region-contains-test.c
@@ -73,7 +73,7 @@ random_coord (pixman_region32_t *region, pixman_bool_t x)
     case 3:
 	return begin;
     default:
-	return (begin + end) / 2;
+	return (end - begin) / 2 + begin;
     }
     return 0;
 }
@@ -163,7 +163,7 @@ main (int argc, const char *argv[])
 {
     return fuzzer_test_main ("region_contains",
 			     1000000,
-			     0xD7C297CC,
+			     0xD2BF8C73,
 			     test_region_contains_rectangle,
 			     argc, argv);
 }
diff --git a/test/stress-test.c b/test/stress-test.c
index 08bf1d4..3174621 100644
--- a/test/stress-test.c
+++ b/test/stress-test.c
@@ -166,7 +166,8 @@ fake_reader (const void *src, int size)
     uint32_t r = lcg_rand_u32 ();
 
     assert (size == 1 || size == 2 || size == 4);
-    return r & ((1 << (size * 8)) - 1);
+
+    return r >> (32 - (size * 8));
 }
 
 static void
-- 
1.6.0.6