Mesa (master): util/format: Add some NEON intrinsics-based u_format_unpack.
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Mon Apr 19 18:24:14 UTC 2021
Module: Mesa
Branch: master
Commit: 80923e8d58cc6bdcceb8e1b2910737fc76fdc0d3
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=80923e8d58cc6bdcceb8e1b2910737fc76fdc0d3
Author: Eric Anholt <eric at anholt.net>
Date: Mon Apr 5 09:33:21 2021 -0700
util/format: Add some NEON intrinsics-based u_format_unpack.
In looking at the profile of dEQP, GLES3 was spending 5-10% of its time in
ReadPixels, and almost all of that is b8g8r8a8_unorm8. It's really slow
because we're getting about 47MB/s by doing uncached reads 32 bits at a
time in the code-generated unpack. If we use NEON to generate larger bus
transactions, we can speed things up to 136MB/s. In comparison, raw
ldr/str read/writes with no byte swapping can hit a max of 216MB/sec.
Reviewed-by: Jesse Natalie <jenatali at microsoft.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10014>
---
src/util/Android.mk | 2 +
src/util/format/meson.build | 1 +
src/util/format/u_format.c | 28 ++++++++++++
src/util/format/u_format.h | 9 ++++
src/util/format/u_format_table.py | 6 ++-
src/util/format/u_format_unpack_neon.c | 79 ++++++++++++++++++++++++++++++++++
6 files changed, 123 insertions(+), 2 deletions(-)
diff --git a/src/util/Android.mk b/src/util/Android.mk
index 829699db669..3866610cf81 100644
--- a/src/util/Android.mk
+++ b/src/util/Android.mk
@@ -50,6 +50,8 @@ LOCAL_C_INCLUDES := \
$(intermediates)/util/format \
$(intermediates)
+LOCAL_CFLAGS := -DNO_FORMAT_ASM
+
# If Android version >=8 MESA should static link libexpat else should dynamic link
ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 27; echo $$?), 0)
LOCAL_STATIC_LIBRARIES := \
diff --git a/src/util/format/meson.build b/src/util/format/meson.build
index e6438629562..b51608f5275 100644
--- a/src/util/format/meson.build
+++ b/src/util/format/meson.build
@@ -28,6 +28,7 @@ files_mesa_format = [
'u_format_rgtc.c',
'u_format_s3tc.c',
'u_format_tests.c',
+ 'u_format_unpack_neon.c',
'u_format_yuv.c',
'u_format_zs.c',
]
diff --git a/src/util/format/u_format.c b/src/util/format/u_format.c
index 1c3ca57fc76..59bb7d287da 100644
--- a/src/util/format/u_format.c
+++ b/src/util/format/u_format.c
@@ -34,6 +34,7 @@
#include "util/format/u_format.h"
#include "util/format/u_format_s3tc.h"
+#include "util/u_cpu_detect.h"
#include "util/u_math.h"
#include "pipe/p_defines.h"
@@ -1130,3 +1131,30 @@ util_format_rgb_to_bgr(enum pipe_format format)
return PIPE_FORMAT_NONE;
}
}
+
+static const struct util_format_unpack_description *util_format_unpack_table[PIPE_FORMAT_COUNT];
+
+static void
+util_format_unpack_table_init(void)
+{
+ for (enum pipe_format format = PIPE_FORMAT_NONE; format < PIPE_FORMAT_COUNT; format++) {
+#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM
+ const struct util_format_unpack_description *unpack = util_format_unpack_description_neon(format);
+ if (unpack) {
+ util_format_unpack_table[format] = unpack;
+ continue;
+ }
+#endif
+
+ util_format_unpack_table[format] = util_format_unpack_description_generic(format);
+ }
+}
+
+const struct util_format_unpack_description *
+util_format_unpack_description(enum pipe_format format)
+{
+ static once_flag flag = ONCE_FLAG_INIT;
+ call_once(&flag, util_format_unpack_table_init);
+
+ return util_format_unpack_table[format];
+}
diff --git a/src/util/format/u_format.h b/src/util/format/u_format.h
index a1b5ec1ecbc..f7b29b407a4 100644
--- a/src/util/format/u_format.h
+++ b/src/util/format/u_format.h
@@ -415,8 +415,17 @@ util_format_description(enum pipe_format format) ATTRIBUTE_CONST;
const struct util_format_pack_description *
util_format_pack_description(enum pipe_format format) ATTRIBUTE_CONST;
+/* Lookup with CPU detection for choosing optimized paths. */
const struct util_format_unpack_description *
util_format_unpack_description(enum pipe_format format) ATTRIBUTE_CONST;
+
+/* Codegenned table of CPU-agnostic unpack code. */
+const struct util_format_unpack_description *
+util_format_unpack_description_generic(enum pipe_format format) ATTRIBUTE_CONST;
+
+const struct util_format_unpack_description *
+util_format_unpack_description_neon(enum pipe_format format) ATTRIBUTE_CONST;
+
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif
diff --git a/src/util/format/u_format_table.py b/src/util/format/u_format_table.py
index b9176956cc3..c76ca583a5c 100644
--- a/src/util/format/u_format_table.py
+++ b/src/util/format/u_format_table.py
@@ -166,8 +166,11 @@ def write_format_table(formats):
print(" },")
def generate_table_getter(type):
+ suffix = ""
+ if type == "unpack_":
+ suffix = "_generic"
print("const struct util_format_%sdescription *" % type)
- print("util_format_%sdescription(enum pipe_format format)" % type)
+ print("util_format_%sdescription%s(enum pipe_format format)" % (type, suffix))
print("{")
print(" if (format >= ARRAY_SIZE(util_format_%sdescriptions))" % (type))
print(" return NULL;")
@@ -242,7 +245,6 @@ def write_format_table(formats):
print("};")
print()
generate_table_getter("pack_")
-
print('static const struct util_format_unpack_description')
print('util_format_unpack_descriptions[] = {')
for format in formats:
diff --git a/src/util/format/u_format_unpack_neon.c b/src/util/format/u_format_unpack_neon.c
new file mode 100644
index 00000000000..7456d7aaa88
--- /dev/null
+++ b/src/util/format/u_format_unpack_neon.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright © 2021 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <u_format.h>
+
+#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM
+
+/* armhf builds default to vfp, not neon, and refuses to compile neon intrinsics
+ * unless you tell it "no really".
+ */
+#ifdef PIPE_ARCH_ARM
+#pragma GCC target ("fpu=neon")
+#endif
+
+#include <arm_neon.h>
+#include "u_format_pack.h"
+#include "util/u_cpu_detect.h"
+
+static void
+util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon(uint8_t *restrict dst, const uint8_t *restrict src, unsigned width)
+{
+ while (width >= 16) {
+ uint8x16x4_t load = vld4q_u8(src);
+ uint8x16x4_t swap = { .val = { load.val[2], load.val[1], load.val[0], load.val[3] } };
+ vst4q_u8(dst, swap);
+ width -= 16;
+ dst += 16 * 4;
+ src += 16 * 4;
+ }
+ if (width)
+ util_format_b8g8r8a8_unorm_unpack_rgba_8unorm(dst, src, width);
+}
+
+static const struct util_format_unpack_description util_format_unpack_descriptions_neon[] = {
+ [PIPE_FORMAT_B8G8R8A8_UNORM] = {
+ .unpack_rgba_8unorm = &util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon,
+ .unpack_rgba = &util_format_b8g8r8a8_unorm_unpack_rgba_float,
+ },
+};
+
+const struct util_format_unpack_description *
+util_format_unpack_description_neon(enum pipe_format format)
+{
+ /* CPU detect for NEON support. On arm64, it's implied. */
+#ifdef PIPE_ARCH_ARM
+ if (!util_get_cpu_caps()->has_neon)
+ return NULL;
+#endif
+
+ if (format >= ARRAY_SIZE(util_format_unpack_descriptions_neon))
+ return NULL;
+
+ if (!util_format_unpack_descriptions_neon[format].unpack_rgba)
+ return NULL;
+
+ return &util_format_unpack_descriptions_neon[format];
+}
+
+#endif /* PIPE_ARCH_AARCH64 | PIPE_ARCH_ARM */
More information about the mesa-commit
mailing list