[Mesa-dev] [PATCH] util: Fix 64-bit depth/stencil packs for big-endian

[Richard Sandiford]

The pack64 routines were hardwired for little-endian layout.
On big-endian, loading a PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
as a 64-bit integer puts the depth float in the high 32 bits
and the stencil in the low 8 bits, like for the associated
MESA_FORMAT_*.

Signed-off-by: Richard Sandiford <rsandifo at linux.vnet.ibm.com>
---
 src/gallium/auxiliary/util/u_pack_color.h | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/gallium/auxiliary/util/u_pack_color.h b/src/gallium/auxiliary/util/u_pack_color.h
index e0c9018..f5e17f5 100644
--- a/src/gallium/auxiliary/util/u_pack_color.h
+++ b/src/gallium/auxiliary/util/u_pack_color.h
@@ -467,7 +467,11 @@ util_pack64_mask_z(enum pipe_format format, uint32_t z)
 {
    switch (format) {
    case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
+#ifdef PIPE_ARCH_BIG_ENDIAN
+      return (uint64_t)z << 32;
+#else
       return z;
+#endif
    default:
       return util_pack_mask_z(format, z);
    }
@@ -505,7 +509,11 @@ util_pack64_mask_z_stencil(enum pipe_format format, uint32_t z, uint8_t s)
    switch (format) {
    case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
       packed = util_pack64_mask_z(format, z);
+#ifdef PIPE_ARCH_BIG_ENDIAN
+      packed |= s;
+#else
       packed |= (uint64_t)s << 32ull;
+#endif
       return packed;
    default:
       return util_pack_mask_z_stencil(format, z, s);
@@ -569,7 +577,11 @@ util_pack64_z(enum pipe_format format, double z)
    switch (format) {
    case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
       fui.f = (float)z;
+#ifdef PIPE_ARCH_BIG_ENDIAN
+      return (uint64_t)fui.ui << 32;
+#else
       return fui.ui;
+#endif
    default:
       return util_pack_z(format, z);
    }
@@ -611,7 +623,11 @@ util_pack64_z_stencil(enum pipe_format format, double z, uint8_t s)
    switch (format) {
    case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
       packed = util_pack64_z(format, z);
+#ifdef PIPE_ARCH_BIG_ENDIAN
+      packed |= s;
+#else
       packed |= (uint64_t)s << 32ull;
+#endif
       break;
    default:
       return util_pack_z_stencil(format, z, s);
-- 
1.8.3.1