Mesa (gallium-llvmpipe): llvmpipe: Get conversions working correctly for all integer types.
Jose Fonseca
jrfonseca at kemper.freedesktop.org
Sat Aug 22 11:40:44 UTC 2009
Module: Mesa
Branch: gallium-llvmpipe
Commit: a878b936c0129f1b8aa53b7cd3bab009532a5d95
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=a878b936c0129f1b8aa53b7cd3bab009532a5d95
Author: José Fonseca <jfonseca at vmware.com>
Date: Sat Aug 22 12:37:12 2009 +0100
llvmpipe: Get conversions working correctly for all integer types.
---
src/gallium/drivers/llvmpipe/lp_bld_const.c | 96 ++++++++
src/gallium/drivers/llvmpipe/lp_bld_const.h | 11 +
src/gallium/drivers/llvmpipe/lp_bld_conv.c | 339 +++++++++++++++++++++------
src/gallium/drivers/llvmpipe/lp_bld_conv.h | 12 +
src/gallium/drivers/llvmpipe/lp_test_conv.c | 14 +-
5 files changed, 389 insertions(+), 83 deletions(-)
diff --git a/src/gallium/drivers/llvmpipe/lp_bld_const.c b/src/gallium/drivers/llvmpipe/lp_bld_const.c
index 991cf24..a205c33 100644
--- a/src/gallium/drivers/llvmpipe/lp_bld_const.c
+++ b/src/gallium/drivers/llvmpipe/lp_bld_const.c
@@ -41,6 +41,31 @@
#include "lp_bld_const.h"
+unsigned
+lp_mantissa(union lp_type type)
+{
+ assert(type.floating);
+
+ if(type.floating) {
+ switch(type.width) {
+ case 32:
+ return 23;
+ case 64:
+ return 53;
+ default:
+ assert(0);
+ return 0;
+ }
+ }
+ else {
+ if(type.sign)
+ return type.width - 1;
+ else
+ return type.width;
+ }
+}
+
+
/**
* Shift of the unity.
*
@@ -93,6 +118,77 @@ lp_const_scale(union lp_type type)
}
+/**
+ * Minimum value representable by the type.
+ */
+double
+lp_const_min(union lp_type type)
+{
+ unsigned bits;
+
+ if(!type.sign)
+ return 0.0;
+
+ if(type.norm)
+ return -1.0;
+
+ if (type.floating) {
+ switch(type.width) {
+ case 32:
+ return -FLT_MAX;
+ case 64:
+ return -DBL_MAX;
+ default:
+ assert(0);
+ return 0.0;
+ }
+ }
+
+ if(type.fixed)
+ /* FIXME: consider the fractional bits? */
+ bits = type.width / 2 - 1;
+ else
+ bits = type.width - 1;
+
+ return (double)-((long long)1 << bits);
+}
+
+
+/**
+ * Maximum value representable by the type.
+ */
+double
+lp_const_max(union lp_type type)
+{
+ unsigned bits;
+
+ if(type.norm)
+ return 1.0;
+
+ if (type.floating) {
+ switch(type.width) {
+ case 32:
+ return FLT_MAX;
+ case 64:
+ return DBL_MAX;
+ default:
+ assert(0);
+ return 0.0;
+ }
+ }
+
+ if(type.fixed)
+ bits = type.width / 2;
+ else
+ bits = type.width;
+
+ if(type.sign)
+ bits -= 1;
+
+ return (double)(((unsigned long long)1 << bits) - 1);
+}
+
+
double
lp_const_eps(union lp_type type)
{
diff --git a/src/gallium/drivers/llvmpipe/lp_bld_const.h b/src/gallium/drivers/llvmpipe/lp_bld_const.h
index fb0e70a..18692aa 100644
--- a/src/gallium/drivers/llvmpipe/lp_bld_const.h
+++ b/src/gallium/drivers/llvmpipe/lp_bld_const.h
@@ -46,6 +46,10 @@ union lp_type type;
unsigned
+lp_mantissa(union lp_type type);
+
+
+unsigned
lp_const_shift(union lp_type type);
@@ -56,6 +60,13 @@ lp_const_offset(union lp_type type);
double
lp_const_scale(union lp_type type);
+double
+lp_const_min(union lp_type type);
+
+
+double
+lp_const_max(union lp_type type);
+
double
lp_const_eps(union lp_type type);
diff --git a/src/gallium/drivers/llvmpipe/lp_bld_conv.c b/src/gallium/drivers/llvmpipe/lp_bld_conv.c
index 520052c..477b28c 100644
--- a/src/gallium/drivers/llvmpipe/lp_bld_conv.c
+++ b/src/gallium/drivers/llvmpipe/lp_bld_conv.c
@@ -72,17 +72,7 @@ lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
assert(src_type.floating);
- switch(src_type.width) {
- case 32:
- mantissa = 23;
- break;
- case 64:
- mantissa = 53;
- break;
- default:
- assert(0);
- return LLVMGetUndef(int_vec_type);
- }
+ mantissa = lp_mantissa(src_type);
/* We cannot carry more bits than the mantissa */
n = MIN2(mantissa, dst_width);
@@ -99,9 +89,6 @@ lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
res = LLVMBuildAdd(builder, res, lp_build_const_uni(src_type, bias), "");
res = LLVMBuildBitCast(builder, res, int_vec_type, "");
- if(dst_width < src_type.width)
- res = LLVMBuildAnd(builder, res, lp_build_int_const_uni(src_type, mask), "");
-
if(dst_width > n) {
int shift = dst_width - n;
res = LLVMBuildShl(builder, res, lp_build_int_const_uni(src_type, shift), "");
@@ -123,6 +110,60 @@ lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
}
#endif
}
+ else
+ res = LLVMBuildAnd(builder, res, lp_build_int_const_uni(src_type, mask), "");
+
+ return res;
+}
+
+
+/**
+ * Inverse of lp_build_clamped_float_to_unsigned_norm.
+ */
+LLVMValueRef
+lp_build_unsigned_norm_to_float(LLVMBuilderRef builder,
+ unsigned src_width,
+ union lp_type dst_type,
+ LLVMValueRef src)
+{
+ LLVMTypeRef vec_type = lp_build_vec_type(dst_type);
+ LLVMTypeRef int_vec_type = lp_build_int_vec_type(dst_type);
+ LLVMValueRef bias_;
+ LLVMValueRef res;
+ unsigned mantissa;
+ unsigned n;
+ unsigned long long ubound;
+ unsigned long long mask;
+ double scale;
+ double bias;
+
+ mantissa = lp_mantissa(dst_type);
+
+ /* We cannot carry more bits than the mantissa */
+ n = MIN2(mantissa, src_width);
+
+ ubound = ((unsigned long long)1 << n);
+ mask = ubound - 1;
+ scale = (double)ubound/mask;
+ bias = (double)((unsigned long long)1 << (mantissa - n));
+
+ res = src;
+
+ if(src_width > mantissa) {
+ int shift = src_width - mantissa;
+ res = LLVMBuildLShr(builder, res, lp_build_int_const_uni(dst_type, shift), "");
+ }
+
+ bias_ = lp_build_const_uni(dst_type, bias);
+
+ res = LLVMBuildOr(builder,
+ res,
+ LLVMBuildBitCast(builder, bias_, int_vec_type, ""), "");
+
+ res = LLVMBuildBitCast(builder, res, vec_type, "");
+
+ res = LLVMBuildSub(builder, res, bias_, "");
+ res = LLVMBuildMul(builder, res, lp_build_const_uni(dst_type, scale), "");
return res;
}
@@ -132,7 +173,7 @@ lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
* Build shuffle vectors that match PUNPCKLxx and PUNPCKHxx instructions.
*/
static LLVMValueRef
-lp_build_const_expand_shuffle(unsigned n, unsigned lo_hi)
+lp_build_const_unpack_shuffle(unsigned n, unsigned lo_hi)
{
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
unsigned i, j;
@@ -151,6 +192,26 @@ lp_build_const_expand_shuffle(unsigned n, unsigned lo_hi)
}
+/**
+ * Build shuffle vectors that match PACKxx instructions.
+ */
+static LLVMValueRef
+lp_build_const_pack_shuffle(unsigned n)
+{
+ LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
+ unsigned i;
+
+ assert(n <= LP_MAX_VECTOR_LENGTH);
+
+ /* TODO: cache results in a static table */
+
+ for(i = 0; i < n; ++i)
+ elems[i] = LLVMConstInt(LLVMInt32Type(), 2*i, 0);
+
+ return LLVMConstVector(elems, n);
+}
+
+
static void
lp_build_expand(LLVMBuilderRef builder,
union lp_type src_type,
@@ -186,8 +247,8 @@ lp_build_expand(LLVMBuilderRef builder,
LLVMValueRef hi;
zero = lp_build_zero(src_type);
- shuffle_lo = lp_build_const_expand_shuffle(src_type.length, 0);
- shuffle_hi = lp_build_const_expand_shuffle(src_type.length, 1);
+ shuffle_lo = lp_build_const_unpack_shuffle(src_type.length, 0);
+ shuffle_hi = lp_build_const_unpack_shuffle(src_type.length, 1);
/* PUNPCKLBW, PUNPCKHBW */
lo = LLVMBuildShuffleVector(builder, dst[i], zero, shuffle_lo, "");
@@ -206,6 +267,85 @@ lp_build_expand(LLVMBuilderRef builder,
}
+/**
+ * Non-interleaved pack.
+ *
+ * lo = __ l0 __ l1 __ l2 __.. __ ln
+ * hi = __ h0 __ h1 __ h2 __.. __ hn
+ * res = l0 l1 l2 .. ln h0 h1 h2 .. hn
+ */
+static LLVMValueRef
+lp_build_pack2(LLVMBuilderRef builder,
+ union lp_type src_type,
+ union lp_type dst_type,
+ LLVMValueRef lo,
+ LLVMValueRef hi)
+{
+ LLVMTypeRef src_vec_type = lp_build_vec_type(src_type);
+ LLVMTypeRef dst_vec_type = lp_build_vec_type(dst_type);
+ LLVMValueRef shuffle;
+ LLVMValueRef res;
+
+ /* Register width must remain constant */
+ assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
+
+ /* We must not loose or gain channels. Only precision */
+ assert(src_type.length * 2 == dst_type.length);
+
+ assert(!src_type.floating);
+ assert(!dst_type.floating);
+
+#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
+ if(src_type.width * src_type.length == 128) {
+ /* All X86 non-interleaved pack instructions all take signed inputs and
+ * saturate them, so saturate beforehand. */
+ if(!src_type.sign) {
+ struct lp_build_context bld;
+ unsigned dst_bits = dst_type.sign ? dst_type.width - 1 : dst_type.width;
+ LLVMValueRef dst_max = lp_build_int_const_uni(src_type, ((unsigned long long)1 << dst_bits) - 1);
+ lp_build_context_init(&bld, builder, src_type);
+ lo = lp_build_min(&bld, lo, dst_max);
+ hi = lp_build_min(&bld, hi, dst_max);
+ }
+
+ switch(src_type.width) {
+ case 32:
+ if(dst_type.sign)
+ res = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packssdw.128", src_vec_type, lo, hi);
+ else
+ /* PACKUSDW is the only instrinsic with a consistent signature */
+ return lp_build_intrinsic_binary(builder, "llvm.x86.sse41.packusdw", dst_vec_type, lo, hi);
+ break;
+
+ case 16:
+ if(dst_type.sign)
+ res = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packsswb.128", src_vec_type, lo, hi);
+ else
+ res = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packuswb.128", src_vec_type, lo, hi);
+ break;
+
+ default:
+ assert(0);
+ return LLVMGetUndef(dst_vec_type);
+ break;
+ }
+
+ res = LLVMBuildBitCast(builder, res, dst_vec_type, "");
+ return res;
+ }
+#endif
+
+ lo = LLVMBuildBitCast(builder, lo, dst_vec_type, "");
+ hi = LLVMBuildBitCast(builder, hi, dst_vec_type, "");
+
+ shuffle = lp_build_const_pack_shuffle(dst_type.length);
+
+ res = LLVMBuildShuffleVector(builder, lo, hi, shuffle, "");
+
+ return res;
+}
+
+
static LLVMValueRef
lp_build_trunc(LLVMBuilderRef builder,
union lp_type src_type,
@@ -225,46 +365,21 @@ lp_build_trunc(LLVMBuilderRef builder,
tmp[i] = src[i];
while(src_type.width > dst_type.width) {
- LLVMTypeRef tmp_vec_type = lp_build_vec_type(src_type);
union lp_type new_type = src_type;
- LLVMTypeRef new_vec_type;
new_type.width /= 2;
new_type.length *= 2;
- new_vec_type = lp_build_vec_type(new_type);
- for(i = 0; i < num_srcs/2; ++i) {
- LLVMValueRef lo = tmp[2*i + 0];
- LLVMValueRef hi = tmp[2*i + 1];
- LLVMValueRef packed = NULL;
+ /* Take in consideration the sign changes only in the last step */
+ if(new_type.width == dst_type.width)
+ new_type.sign = dst_type.sign;
- if(src_type.width == 32) {
-#if 0
- if(dst_type.sign)
- packed = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packssdw.128", tmp_vec_type, lo, hi);
- else {
- /* XXX: PACKUSDW intrinsic is actually the only one with a consistent signature */
- packed = lp_build_intrinsic_binary(builder, "llvm.x86.sse41.packusdw", new_vec_type, lo, hi);
- }
-#else
- packed = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packssdw.128", tmp_vec_type, lo, hi);
-#endif
- }
- else if(src_type.width == 16) {
- if(dst_type.sign)
- packed = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packsswb.128", tmp_vec_type, lo, hi);
- else
- packed = lp_build_intrinsic_binary(builder, "llvm.x86.sse2.packuswb.128", tmp_vec_type, lo, hi);
- }
- else
- assert(0);
+ num_srcs /= 2;
- tmp[i] = LLVMBuildBitCast(builder, packed, new_vec_type, "");
- }
+ for(i = 0; i < num_srcs; ++i)
+ tmp[i] = lp_build_pack2(builder, src_type, new_type, tmp[2*i + 0], tmp[2*i + 1]);
src_type = new_type;
-
- num_srcs /= 2;
}
assert(num_srcs == 1);
@@ -312,17 +427,33 @@ lp_build_conv(LLVMBuilderRef builder,
* Clamp if necessary
*/
- if(tmp_type.sign != dst_type.sign || tmp_type.norm != dst_type.norm) {
+ if(src_type.value != dst_type.value) {
struct lp_build_context bld;
+ double src_min = lp_const_min(src_type);
+ double dst_min = lp_const_min(dst_type);
+ double src_max = lp_const_max(src_type);
+ double dst_max = lp_const_max(dst_type);
+ LLVMValueRef thres;
+
lp_build_context_init(&bld, builder, tmp_type);
- if(tmp_type.sign && !dst_type.sign)
+ if(src_min < dst_min) {
+ if(dst_min == 0.0)
+ thres = bld.zero;
+ else
+ thres = lp_build_const_uni(src_type, dst_min);
for(i = 0; i < num_tmps; ++i)
- tmp[i] = lp_build_max(&bld, tmp[i], bld.zero);
+ tmp[i] = lp_build_max(&bld, tmp[i], thres);
+ }
- if(!tmp_type.norm && dst_type.norm)
+ if(src_max > dst_max) {
+ if(dst_max == 1.0)
+ thres = bld.one;
+ else
+ thres = lp_build_const_uni(src_type, dst_max);
for(i = 0; i < num_tmps; ++i)
- tmp[i] = lp_build_min(&bld, tmp[i], bld.one);
+ tmp[i] = lp_build_min(&bld, tmp[i], thres);
+ }
}
/*
@@ -376,7 +507,7 @@ lp_build_conv(LLVMBuilderRef builder,
if(src_shift > dst_shift) {
LLVMValueRef shift = lp_build_int_const_uni(tmp_type, src_shift - dst_shift);
for(i = 0; i < num_tmps; ++i)
- if(dst_type.sign)
+ if(src_type.sign)
tmp[i] = LLVMBuildAShr(builder, tmp[i], shift, "");
else
tmp[i] = LLVMBuildLShr(builder, tmp[i], shift, "");
@@ -417,30 +548,41 @@ lp_build_conv(LLVMBuilderRef builder,
/* Nothing to do */
}
else if(!src_type.floating && dst_type.floating) {
- double src_scale = lp_const_scale(src_type);
- LLVMTypeRef tmp_vec_type;
-
- /* Use an equally sized integer for intermediate computations */
- tmp_type.floating = TRUE;
- tmp_type.sign = TRUE;
- tmp_vec_type = lp_build_vec_type(tmp_type);
- for(i = 0; i < num_tmps; ++i) {
- #if 0
- if(dst_type.sign)
+ if(!src_type.fixed && !src_type.sign && src_type.norm) {
+ for(i = 0; i < num_tmps; ++i) {
+ tmp[i] = lp_build_unsigned_norm_to_float(builder,
+ src_type.width,
+ dst_type,
+ tmp[i]);
+ }
+ tmp_type.floating = TRUE;
+ }
+ else {
+ double src_scale = lp_const_scale(src_type);
+ LLVMTypeRef tmp_vec_type;
+
+ /* Use an equally sized integer for intermediate computations */
+ tmp_type.floating = TRUE;
+ tmp_type.sign = TRUE;
+ tmp_vec_type = lp_build_vec_type(tmp_type);
+ for(i = 0; i < num_tmps; ++i) {
+#if 0
+ if(dst_type.sign)
+ tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
+ else
+ tmp[i] = LLVMBuildUIToFP(builder, tmp[i], tmp_vec_type, "");
+#else
+ /* FIXME: there is no SSE counterpart for LLVMBuildUIToFP */
tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
- else
- tmp[i] = LLVMBuildUIToFP(builder, tmp[i], tmp_vec_type, "");
- #else
- /* FIXME: there is no SSE counterpart for LLVMBuildUIToFP */
- tmp[i] = LLVMBuildSIToFP(builder, tmp[i], tmp_vec_type, "");
- #endif
- }
+#endif
+ }
- if (src_scale != 1.0) {
- LLVMValueRef scale = lp_build_const_uni(tmp_type, 1.0/src_scale);
- for(i = 0; i < num_tmps; ++i)
- tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
- }
+ if (src_scale != 1.0) {
+ LLVMValueRef scale = lp_build_const_uni(tmp_type, 1.0/src_scale);
+ for(i = 0; i < num_tmps; ++i)
+ tmp[i] = LLVMBuildMul(builder, tmp[i], scale, "");
+ }
+ }
}
else {
unsigned src_shift = lp_const_shift(src_type);
@@ -457,3 +599,48 @@ lp_build_conv(LLVMBuilderRef builder,
for(i = 0; i < num_dsts; ++i)
dst[i] = tmp[i];
}
+
+
+/**
+ * Convenience wrapper around lp_build_conv for bit masks.
+ */
+void
+lp_build_conv_mask(LLVMBuilderRef builder,
+ union lp_type src_type,
+ union lp_type dst_type,
+ const LLVMValueRef *src, unsigned num_srcs,
+ LLVMValueRef *dst, unsigned num_dsts)
+{
+ /* Register width must remain constant */
+ assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
+
+ /* We must not loose or gain channels. Only precision */
+ assert(src_type.length * num_srcs == dst_type.length * num_dsts);
+
+ src_type.floating = FALSE;
+ src_type.fixed = FALSE;
+ src_type.sign = FALSE;
+ src_type.norm = TRUE;
+
+ dst_type.floating = FALSE;
+ dst_type.fixed = FALSE;
+ dst_type.sign = FALSE;
+ dst_type.norm = TRUE;
+
+ /*
+ * Truncate or expand bit width
+ */
+
+ if(src_type.width > dst_type.width) {
+ assert(num_dsts == 1);
+ dst[0] = lp_build_trunc(builder, src_type, dst_type, src, num_srcs);
+ }
+ else if(src_type.width < dst_type.width) {
+ assert(num_srcs == 1);
+ lp_build_expand(builder, src_type, dst_type, src[0], dst, num_dsts);
+ }
+ else {
+ assert(num_srcs == num_dsts);
+ memcpy(dst, src, num_dsts * sizeof *dst);
+ }
+}
diff --git a/src/gallium/drivers/llvmpipe/lp_bld_conv.h b/src/gallium/drivers/llvmpipe/lp_bld_conv.h
index b1e3da7..05c1ef2 100644
--- a/src/gallium/drivers/llvmpipe/lp_bld_conv.h
+++ b/src/gallium/drivers/llvmpipe/lp_bld_conv.h
@@ -49,6 +49,12 @@ lp_build_clamped_float_to_unsigned_norm(LLVMBuilderRef builder,
unsigned dst_width,
LLVMValueRef src);
+LLVMValueRef
+lp_build_unsigned_norm_to_float(LLVMBuilderRef builder,
+ unsigned src_width,
+ union lp_type dst_type,
+ LLVMValueRef src);
+
void
lp_build_conv(LLVMBuilderRef builder,
@@ -57,5 +63,11 @@ lp_build_conv(LLVMBuilderRef builder,
const LLVMValueRef *srcs, unsigned num_srcs,
LLVMValueRef *dsts, unsigned num_dsts);
+void
+lp_build_conv_mask(LLVMBuilderRef builder,
+ union lp_type src_type,
+ union lp_type dst_type,
+ const LLVMValueRef *src, unsigned num_srcs,
+ LLVMValueRef *dst, unsigned num_dsts);
#endif /* !LP_BLD_CONV_H */
diff --git a/src/gallium/drivers/llvmpipe/lp_test_conv.c b/src/gallium/drivers/llvmpipe/lp_test_conv.c
index e8b0b56..e648983 100644
--- a/src/gallium/drivers/llvmpipe/lp_test_conv.c
+++ b/src/gallium/drivers/llvmpipe/lp_test_conv.c
@@ -351,7 +351,13 @@ const union lp_type conv_types[] = {
{{ TRUE, FALSE, FALSE, TRUE, 32, 4 }},
{{ TRUE, FALSE, FALSE, FALSE, 32, 4 }},
-#if 0
+ /* TODO: test fixed formats too */
+
+ {{ FALSE, FALSE, TRUE, TRUE, 16, 8 }},
+ {{ FALSE, FALSE, TRUE, FALSE, 16, 8 }},
+ {{ FALSE, FALSE, FALSE, TRUE, 16, 8 }},
+ {{ FALSE, FALSE, FALSE, FALSE, 16, 8 }},
+
{{ FALSE, FALSE, TRUE, TRUE, 32, 4 }},
{{ FALSE, FALSE, TRUE, FALSE, 32, 4 }},
{{ FALSE, FALSE, FALSE, TRUE, 32, 4 }},
@@ -366,12 +372,6 @@ const union lp_type conv_types[] = {
{{ FALSE, FALSE, TRUE, FALSE, 8, 16 }},
{{ FALSE, FALSE, FALSE, TRUE, 8, 16 }},
{{ FALSE, FALSE, FALSE, FALSE, 8, 16 }},
-#else
- {{ FALSE, FALSE, TRUE, FALSE, 32, 4 }},
- {{ FALSE, FALSE, TRUE, FALSE, 16, 8 }},
- {{ FALSE, FALSE, FALSE, FALSE, 8, 16 }},
- {{ FALSE, FALSE, FALSE, TRUE, 8, 16 }},
-#endif
};
More information about the mesa-commit
mailing list