[Intel-gfx] [PATCH 6/6] drm/i915: obey wbinvd threshold in more places

[Ben Widawsky]

Signed-off-by: Ben Widawsky <ben at bwidawsk.net>
---
 drivers/gpu/drm/i915/i915_drv.h     |  4 ++++
 drivers/gpu/drm/i915/i915_gem.c     | 32 ++++++++++++++++++++++++++++----
 drivers/gpu/drm/i915/i915_gem_gtt.c | 13 ++++++++++---
 3 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 5d2f62d..dfecdfd 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2818,6 +2818,10 @@ static inline bool cpu_cache_is_coherent(struct drm_device *dev,
 {
 	return HAS_LLC(dev) || level != I915_CACHE_NONE;
 }
+static inline bool i915_gem_obj_should_clflush(struct drm_i915_gem_object *obj)
+{
+	return obj->base.size >= to_i915(obj->base.dev)->wbinvd_threshold;
+}
 int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_init(struct drm_device *dev);
 int i915_gem_init_rings(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 4d5a69d..59be709 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -204,6 +204,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 	char *vaddr = obj->phys_handle->vaddr;
 	struct sg_table *st;
 	struct scatterlist *sg;
+	const bool do_wbinvd = i915_gem_obj_should_clflush(obj);
 	int i;
 
 	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
@@ -219,12 +220,15 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 
 		src = kmap_atomic(page);
 		memcpy(vaddr, src, PAGE_SIZE);
-		drm_clflush_virt_range(vaddr, PAGE_SIZE);
+		if (!do_wbinvd)
+			drm_clflush_virt_range(vaddr, PAGE_SIZE);
 		kunmap_atomic(src);
 
 		page_cache_release(page);
 		vaddr += PAGE_SIZE;
 	}
+	if (do_wbinvd)
+		wbinvd();
 
 	i915_gem_chipset_flush(obj->base.dev);
 
@@ -252,6 +256,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 static void
 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 {
+	const bool do_wbinvd = i915_gem_obj_should_clflush(obj);
 	int ret;
 
 	BUG_ON(obj->madv == __I915_MADV_PURGED);
@@ -282,7 +287,8 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 				continue;
 
 			dst = kmap_atomic(page);
-			drm_clflush_virt_range(vaddr, PAGE_SIZE);
+			if (!do_wbinvd)
+				drm_clflush_virt_range(vaddr, PAGE_SIZE);
 			memcpy(dst, vaddr, PAGE_SIZE);
 			kunmap_atomic(dst);
 
@@ -295,6 +301,9 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 		obj->dirty = 0;
 	}
 
+	if (do_wbinvd && !ret)
+		wbinvd();
+
 	sg_free_table(obj->pages);
 	kfree(obj->pages);
 
@@ -396,7 +405,10 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 			return -EFAULT;
 	}
 
-	drm_clflush_virt_range(vaddr, args->size);
+	if (args->size >= to_i915(obj->base.dev)->wbinvd_threshold)
+		wbinvd();
+	else
+		drm_clflush_virt_range(vaddr, args->size);
 	i915_gem_chipset_flush(dev);
 	return 0;
 }
@@ -647,6 +659,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 	int prefaulted = 0;
 	int needs_clflush = 0;
+	bool do_wbinvd = false;
 	struct sg_page_iter sg_iter;
 
 	user_data = to_user_ptr(args->data_ptr);
@@ -658,6 +671,9 @@ i915_gem_shmem_pread(struct drm_device *dev,
 	if (ret)
 		return ret;
 
+	if (needs_clflush && i915_gem_obj_should_clflush(obj))
+		do_wbinvd = true;
+
 	offset = args->offset;
 
 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
@@ -714,6 +730,9 @@ next_page:
 	}
 
 out:
+	if (do_wbinvd && !ret)
+		wbinvd();
+
 	i915_gem_object_unpin_pages(obj);
 
 	return ret;
@@ -4061,7 +4080,12 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 
 	/* Flush the CPU cache if it's still invalid. */
 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
-		i915_gem_clflush_object(obj, false);
+		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+		if (is_cpu_flush_required(obj) &&
+		    obj->base.size >= dev_priv->wbinvd_threshold)
+			wbinvd();
+		else
+			i915_gem_clflush_object(obj, false);
 
 		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
 	}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 746f77f..13cc493 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -371,6 +371,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 	struct sg_page_iter sg_iter;
+	const bool needs_flush = !HAS_LLC(ppgtt->base.dev);
+	bool do_wbinvd = needs_flush &&
+		pages->nents * PAGE_SIZE >= to_i915(vm->dev)->wbinvd_threshold;
 
 	pt_vaddr = NULL;
 
@@ -385,7 +388,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
 					cache_level, true);
 		if (++pte == GEN8_PTES_PER_PAGE) {
-			if (!HAS_LLC(ppgtt->base.dev))
+			if (needs_flush && !do_wbinvd)
 				drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 			kunmap_atomic(pt_vaddr);
 			pt_vaddr = NULL;
@@ -401,6 +404,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 			drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 		kunmap_atomic(pt_vaddr);
 	}
+
+	if (do_wbinvd)
+		wbinvd();
 }
 
 static void gen8_free_page_tables(struct page **pt_pages)
@@ -660,11 +666,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 			pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 						      I915_CACHE_LLC);
 		}
-		if (!HAS_LLC(ppgtt->base.dev))
-			drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
 		kunmap_atomic(pd_vaddr);
 	}
 
+	if (!HAS_LLC(ppgtt->base.dev))
+		wbinvd();
+
 	ppgtt->switch_mm = gen8_mm_switch;
 	ppgtt->base.clear_range = gen8_ppgtt_clear_range;
 	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
-- 
2.3.0