Mesa (master): util/u_format: don' t crash in util_format_translate if we can't do translation

[Roland Scheidegger]

Module: Mesa
Branch: master
Commit: 612a1d5be1e8e21bceb28ebd77c3b25b72576394
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=612a1d5be1e8e21bceb28ebd77c3b25b72576394

Author: Roland Scheidegger <sroland at vmware.com>
Date:   Thu Feb 27 17:35:08 2014 +0100

util/u_format: don't crash in util_format_translate if we can't do translation

Some formats can't be handled - in particular cannot handle ints/uints formats,
which lack the pack_rgba_float/unpack_rgba_float functions. Instead of trying
to call these (and crash) return an error (I'm not sure yet if we should try
to translate such formats too here might not make much sense).

v2: suggested by Jose, use separate checks for pack/unpack of rgba_8unorm and
rgba_float functions (right now if one exists the other should as well).

Reviewed-by: Jose Fonseca <jfonseca at vmware.com>

---

 src/gallium/auxiliary/util/u_format.c |   21 ++++++++++++++++-----
 src/gallium/auxiliary/util/u_format.h |    2 +-
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/gallium/auxiliary/util/u_format.c b/src/gallium/auxiliary/util/u_format.c
index 6b602bf..056f82f 100644
--- a/src/gallium/auxiliary/util/u_format.c
+++ b/src/gallium/auxiliary/util/u_format.c
@@ -527,7 +527,7 @@ util_format_fits_8unorm(const struct util_format_description *format_desc)
 }
 
 
-void
+boolean
 util_format_translate(enum pipe_format dst_format,
                       void *dst, unsigned dst_stride,
                       unsigned dst_x, unsigned dst_y,
@@ -555,7 +555,7 @@ util_format_translate(enum pipe_format dst_format,
       util_copy_rect(dst, dst_format, dst_stride,  dst_x, dst_y,
                      width, height, src, (int)src_stride,
                      src_x, src_y);
-      return;
+      return TRUE;
    }
 
    assert(dst_x % dst_format_desc->block.width == 0);
@@ -621,7 +621,7 @@ util_format_translate(enum pipe_format dst_format,
 
       FREE(tmp_z);
 
-      return;
+      return TRUE;
    }
 
    if (util_format_fits_8unorm(src_format_desc) ||
@@ -629,10 +629,15 @@ util_format_translate(enum pipe_format dst_format,
       unsigned tmp_stride;
       uint8_t *tmp_row;
 
+      if (!src_format_desc->unpack_rgba_8unorm ||
+          !dst_format_desc->pack_rgba_8unorm) {
+         return FALSE;
+      }
+
       tmp_stride = MAX2(width, x_step) * 4 * sizeof *tmp_row;
       tmp_row = MALLOC(y_step * tmp_stride);
       if (!tmp_row)
-         return;
+         return FALSE;
 
       while (height >= y_step) {
          src_format_desc->unpack_rgba_8unorm(tmp_row, tmp_stride, src_row, src_stride, width, y_step);
@@ -654,10 +659,15 @@ util_format_translate(enum pipe_format dst_format,
       unsigned tmp_stride;
       float *tmp_row;
 
+      if (!src_format_desc->unpack_rgba_float ||
+          !dst_format_desc->pack_rgba_float) {
+         return FALSE;
+      }
+
       tmp_stride = MAX2(width, x_step) * 4 * sizeof *tmp_row;
       tmp_row = MALLOC(y_step * tmp_stride);
       if (!tmp_row)
-         return;
+         return FALSE;
 
       while (height >= y_step) {
          src_format_desc->unpack_rgba_float(tmp_row, tmp_stride, src_row, src_stride, width, y_step);
@@ -675,6 +685,7 @@ util_format_translate(enum pipe_format dst_format,
 
       FREE(tmp_row);
    }
+   return TRUE;
 }
 
 void util_format_compose_swizzles(const unsigned char swz1[4],
diff --git a/src/gallium/auxiliary/util/u_format.h b/src/gallium/auxiliary/util/u_format.h
index 5f86e2d..e36a9e2 100644
--- a/src/gallium/auxiliary/util/u_format.h
+++ b/src/gallium/auxiliary/util/u_format.h
@@ -1190,7 +1190,7 @@ util_format_write_4i(enum pipe_format format,
 boolean
 util_format_fits_8unorm(const struct util_format_description *format_desc);
 
-void
+boolean
 util_format_translate(enum pipe_format dst_format,
                       void *dst, unsigned dst_stride,
                       unsigned dst_x, unsigned dst_y,