[Intel-gfx] [PATCH] drm/i915: Use SSE4.1 movntqda to accelerate reads from WC memory

[Chris Wilson]

This patch provides the infrastructure for performing a 16-byte aligned
read from WC (or even uncached) memory using non-temporal instructions
introduced with sse4.1. Using movntdqa we can bypass the CPU caches and
read directly from memory and ignoring the page attributes set on the
CPU PTE i.e. negating the impact of an otherwise UC access, copying
using movntqda from WC is almost as fast as reading from WB memory,
modulo the possibility of both hitting the CPU cache or leaving the data
in the CPU cache for the next consumer.

This will be used in later patches to accelerate accessing WC memory.

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Akash Goel <akash.goel at intel.com>
Cc: Damien Lespiau <damien.lespiau at intel.com>
Cc: Mika Kuoppala <mika.kuoppala at intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin at intel.com>
---
 drivers/gpu/drm/i915/Makefile      |  6 ++++
 drivers/gpu/drm/i915/i915_drv.c    |  2 ++
 drivers/gpu/drm/i915/i915_drv.h    |  3 ++
 drivers/gpu/drm/i915/i915_memcpy.c | 64 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 75 insertions(+)
 create mode 100644 drivers/gpu/drm/i915/i915_memcpy.c

diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 75318ebb8d25..15ecd7383d85 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -4,11 +4,17 @@
 
 subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror
 
+ifeq ($(call as-instr,movntdqa (%eax)$(comma)%xmm0,y),y)
+	subdir-ccflags-y += -DCONFIG_AS_MOVNTDQA
+	subdir-ccflags-y += -mpreferred-stack-boundary=4
+endif
+
 # Please keep these build lists sorted!
 
 # core driver code
 i915-y := i915_drv.o \
 	  i915_irq.o \
+	  i915_memcpy.o \
 	  i915_params.o \
 	  i915_pci.o \
           i915_suspend.o \
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index c5b7b8e0678a..16612542496a 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -848,6 +848,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	mutex_init(&dev_priv->wm.wm_mutex);
 	mutex_init(&dev_priv->pps_mutex);
 
+	i915_memcpy_init_early(dev_priv);
+
 	ret = i915_workqueues_init(dev_priv);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 27d9b2c374b3..fe58ee78de98 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -4070,4 +4070,7 @@ static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
 	return false;
 }
 
+void i915_memcpy_init_early(struct drm_i915_private *dev_priv);
+void i915_memcpy_from_wc(void *dst, const void *src, unsigned long len);
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c
new file mode 100644
index 000000000000..09c24164f2b6
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_memcpy.c
@@ -0,0 +1,64 @@
+#include "i915_drv.h"
+
+DEFINE_STATIC_KEY_FALSE(has_movntqa);
+
+#ifdef CONFIG_AS_MOVNTDQA
+static void __movntqda(void *dst, const void *src, unsigned long len)
+{
+	len >>= 4;
+	while (len >= 4) {
+		__asm__ __volatile__(
+		"movntdqa (%0), %%xmm0\n"
+		"movntdqa 8(%0), %%xmm1\n"
+		"movntdqa 16(%0), %%xmm2\n"
+		"movntdqa 24(%0), %%xmm3\n"
+		"movaps %%xmm0, (%1)\n"
+		"movaps %%xmm1, 8(%1)\n"
+		"movaps %%xmm2, 16(%1)\n"
+		"movaps %%xmm3, 24(%1)\n"
+		: : "r" (src), "r" (dst) : "memory");
+		src += 64;
+		dst += 64;
+		len -= 4;
+	}
+	while (len--) {
+		__asm__ __volatile__(
+		"movntdqa (%0), %%xmm0\n"
+		"movaps %%xmm0, (%1)\n"
+		: : "r" (src), "r" (dst) : "memory");
+		src += 16;
+		dst += 16;
+	}
+}
+#endif
+
+/**
+ * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
+ * @dst: destination pointer
+ * @src: source pointer
+ * @len: how many bytes to copy
+ *
+ * i915_memcpy_from_wc copies @len bytes from @src to @dst using
+ * non-temporal instructions where available. Note that all arguments
+ * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
+ * of 16.
+ */
+void i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
+{
+	GEM_BUG_ON((unsigned long)dst & 15);
+	GEM_BUG_ON((unsigned long)src & 15);
+	GEM_BUG_ON((unsigned long)len & 15);
+
+#ifdef CONFIG_AS_MOVNTDQA
+	if (static_branch_likely(&has_movntqa))
+		 __movntqda(dst, src, len);
+	else
+#endif
+		memcpy(dst, src, len);
+}
+
+void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
+{
+	if (static_cpu_has(X86_FEATURE_XMM4_1))
+		static_branch_enable(&has_movntqa);
+}
-- 
2.8.1