[PATCH 06/17] drm/i915/gtt: Serialise writes to the GGTT

[Chris Wilson]

After we modify the contents of the GSM (i.e. the Global GTT), we have
to tell the GPU to invalidate its TLB cache. We have to wait for that
invalidation cycle to complete before issuing new commands that may try
to access the new writes into the GSM or else the GPU may race against
the TLB flush and see stale data.

We already employed this serialisation for Broxton, where it was of
critical importance. However, with the advent of mm switching on gen7
where we first write into the GGTT while the GPU is trying to read from
it, we start to observe that the race is commonplace.

References: 0ef34ad6222a ("drm/i915: Serialize GTT/Aperture accesses on BXT")
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen at linux.intel.com>
Cc: Mika Kuoppala <mika.kuoppala at linux.intel.com>
Cc: Matthew Auld <matthew.william.auld at gmail.com>
---
 drivers/gpu/drm/i915/i915_gem_gtt.c | 36 +++++++++++++++--------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 63569bd71fef..0d8155cf82e6 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -115,6 +115,24 @@ static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
 	 * WCB of the writes into the GGTT before it triggers the invalidate.
 	 */
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+
+	/*
+	 * Make sure the the writes into the GGTT have landed and the TLB
+	 * invalidation is complete before use. This is especially important
+	 * when switching mm, as the GPU will try to access the GSM while the
+	 * writes are still being flushed.
+	 *
+	 * Note for bxd_vtd_ggtt_wa:
+	 *
+	 * Make sure the internal GAM fifo has been cleared of all GTT
+	 * writes before exiting stop_machine(). This guarantees that
+	 * any aperture accesses waiting to start in another process
+	 * cannot back up behind the GTT writes causing a hang.
+	 * The register can be any arbitrary GAM register.
+	 */
+	WARN_ON_ONCE(intel_wait_for_register(dev_priv, GFX_FLSH_CNTL_GEN6,
+					     GFX_FLSH_CNTL_EN, 0,
+					     20));
 }
 
 static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv)
@@ -2413,20 +2431,6 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 		gen8_set_pte(&gtt_base[i], scratch_pte);
 }
 
-static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
-{
-	struct drm_i915_private *dev_priv = vm->i915;
-
-	/*
-	 * Make sure the internal GAM fifo has been cleared of all GTT
-	 * writes before exiting stop_machine(). This guarantees that
-	 * any aperture accesses waiting to start in another process
-	 * cannot back up behind the GTT writes causing a hang.
-	 * The register can be any arbitrary GAM register.
-	 */
-	POSTING_READ(GFX_FLSH_CNTL_GEN6);
-}
-
 struct insert_page {
 	struct i915_address_space *vm;
 	dma_addr_t addr;
@@ -2439,7 +2443,6 @@ static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
 	struct insert_page *arg = _arg;
 
 	gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
-	bxt_vtd_ggtt_wa(arg->vm);
 
 	return 0;
 }
@@ -2466,7 +2469,6 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 	struct insert_entries *arg = _arg;
 
 	gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, 0);
-	bxt_vtd_ggtt_wa(arg->vm);
 
 	return 0;
 }
@@ -2492,7 +2494,7 @@ static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
 	struct clear_range *arg = _arg;
 
 	gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
-	bxt_vtd_ggtt_wa(arg->vm);
+	i915_ggtt_invalidate(arg->vm->i915);
 
 	return 0;
 }
-- 
2.17.1