[gst-cvs] gst-plugins-good: pngdec: Avoid possible overflow in calculations

[Jan Schmidt]

Module: gst-plugins-good
Branch: master
Commit: d9544bcc44adcef769cbdf7f6453e140058a3adc
URL:    http://cgit.freedesktop.org/gstreamer/gst-plugins-good/commit/?id=d9544bcc44adcef769cbdf7f6453e140058a3adc

Author: Jan Schmidt <thaytan at noraisin.net>
Date:   Wed May 27 17:06:34 2009 +0100

pngdec: Avoid possible overflow in calculations

A malformed (or simply huge) PNG file can lead to integer overflow in
calculating the size of the output buffer, leading to crashes or buffer
overflows later. Fixes SA35205 security advisory.

---

 ext/libpng/gstpngdec.c |   20 ++++++++++++++++----
 1 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/ext/libpng/gstpngdec.c b/ext/libpng/gstpngdec.c
index 524b468..dde459d 100644
--- a/ext/libpng/gstpngdec.c
+++ b/ext/libpng/gstpngdec.c
@@ -201,7 +201,14 @@ user_info_callback (png_structp png_ptr, png_infop info)
 
   /* Allocate output buffer */
   pngdec->rowbytes = png_get_rowbytes (pngdec->png, pngdec->info);
-  buffer_size = pngdec->height * GST_ROUND_UP_4 (pngdec->rowbytes);
+  if (pngdec->rowbytes > (G_MAXUINT32 - 3)
+      || pngdec->height > G_MAXUINT32 / pngdec->rowbytes) {
+    ret = GST_FLOW_ERROR;
+    goto beach;
+  }
+  pngdec->rowbytes = GST_ROUND_UP_4 (pngdec->rowbytes);
+  buffer_size = pngdec->height * pngdec->rowbytes;
+
   ret =
       gst_pad_alloc_buffer_and_set_caps (pngdec->srcpad, GST_BUFFER_OFFSET_NONE,
       buffer_size, GST_PAD_CAPS (pngdec->srcpad), &buffer);
@@ -228,7 +235,7 @@ user_endrow_callback (png_structp png_ptr, png_bytep new_row,
   /* If buffer_out doesn't exist, it means buffer_alloc failed, which 
    * will already have set the return code */
   if (GST_IS_BUFFER (pngdec->buffer_out)) {
-    size_t offset = row_num * GST_ROUND_UP_4 (pngdec->rowbytes);
+    size_t offset = row_num * pngdec->rowbytes;
 
     GST_LOG ("got row %u, copying in buffer %p at offset %" G_GSIZE_FORMAT,
         (guint) row_num, pngdec->buffer_out, offset);
@@ -496,7 +503,12 @@ gst_pngdec_task (GstPad * pad)
 
   /* Allocate output buffer */
   rowbytes = png_get_rowbytes (pngdec->png, pngdec->info);
-  buffer_size = pngdec->height * GST_ROUND_UP_4 (rowbytes);
+  if (rowbytes > (G_MAXUINT32 - 3) || pngdec->height > G_MAXUINT32 / rowbytes) {
+    ret = GST_FLOW_ERROR;
+    goto pause;
+  }
+  rowbytes = GST_ROUND_UP_4 (rowbytes);
+  buffer_size = pngdec->height * rowbytes;
   ret =
       gst_pad_alloc_buffer_and_set_caps (pngdec->srcpad, GST_BUFFER_OFFSET_NONE,
       buffer_size, GST_PAD_CAPS (pngdec->srcpad), &buffer);
@@ -509,7 +521,7 @@ gst_pngdec_task (GstPad * pad)
 
   for (i = 0; i < pngdec->height; i++) {
     rows[i] = inp;
-    inp += GST_ROUND_UP_4 (rowbytes);
+    inp += rowbytes;
   }
 
   /* Read the actual picture */