[Mesa-dev] [PATCH] util/u_format: don't crash in util_format_translate if we can't do translation

[sroland at vmware.com]

From: Roland Scheidegger <sroland at vmware.com>

Some formats can't be handled - in particular cannot handle ints/uints formats,
which lack the pack_rgba_float/unpack_rgba_float functions. Instead of trying
to call these (and crash) return an error (I'm not sure yet if we should try
to translate such formats too here might not make much sense).
---
 src/gallium/auxiliary/util/u_format.c |   16 +++++++++++-----
 src/gallium/auxiliary/util/u_format.h |    2 +-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/gallium/auxiliary/util/u_format.c b/src/gallium/auxiliary/util/u_format.c
index 6b602bf..92a0c54 100644
--- a/src/gallium/auxiliary/util/u_format.c
+++ b/src/gallium/auxiliary/util/u_format.c
@@ -527,7 +527,7 @@ util_format_fits_8unorm(const struct util_format_description *format_desc)
 }
 
 
-void
+boolean
 util_format_translate(enum pipe_format dst_format,
                       void *dst, unsigned dst_stride,
                       unsigned dst_x, unsigned dst_y,
@@ -555,7 +555,7 @@ util_format_translate(enum pipe_format dst_format,
       util_copy_rect(dst, dst_format, dst_stride,  dst_x, dst_y,
                      width, height, src, (int)src_stride,
                      src_x, src_y);
-      return;
+      return TRUE;
    }
 
    assert(dst_x % dst_format_desc->block.width == 0);
@@ -621,7 +621,12 @@ util_format_translate(enum pipe_format dst_format,
 
       FREE(tmp_z);
 
-      return;
+      return TRUE;
+   }
+
+   if (!src_format_desc->unpack_rgba_float ||
+       !dst_format_desc->pack_rgba_float) {
+      return FALSE;
    }
 
    if (util_format_fits_8unorm(src_format_desc) ||
@@ -632,7 +637,7 @@ util_format_translate(enum pipe_format dst_format,
       tmp_stride = MAX2(width, x_step) * 4 * sizeof *tmp_row;
       tmp_row = MALLOC(y_step * tmp_stride);
       if (!tmp_row)
-         return;
+         return FALSE;
 
       while (height >= y_step) {
          src_format_desc->unpack_rgba_8unorm(tmp_row, tmp_stride, src_row, src_stride, width, y_step);
@@ -657,7 +662,7 @@ util_format_translate(enum pipe_format dst_format,
       tmp_stride = MAX2(width, x_step) * 4 * sizeof *tmp_row;
       tmp_row = MALLOC(y_step * tmp_stride);
       if (!tmp_row)
-         return;
+         return FALSE;
 
       while (height >= y_step) {
          src_format_desc->unpack_rgba_float(tmp_row, tmp_stride, src_row, src_stride, width, y_step);
@@ -675,6 +680,7 @@ util_format_translate(enum pipe_format dst_format,
 
       FREE(tmp_row);
    }
+   return TRUE;
 }
 
 void util_format_compose_swizzles(const unsigned char swz1[4],
diff --git a/src/gallium/auxiliary/util/u_format.h b/src/gallium/auxiliary/util/u_format.h
index 5f86e2d..e36a9e2 100644
--- a/src/gallium/auxiliary/util/u_format.h
+++ b/src/gallium/auxiliary/util/u_format.h
@@ -1190,7 +1190,7 @@ util_format_write_4i(enum pipe_format format,
 boolean
 util_format_fits_8unorm(const struct util_format_description *format_desc);
 
-void
+boolean
 util_format_translate(enum pipe_format dst_format,
                       void *dst, unsigned dst_stride,
                       unsigned dst_x, unsigned dst_y,
-- 
1.7.9.5