[PATCH 17/17] HAX: Ivybridge incoherence

[Chris Wilson]

Tried:

 - plain memory barriers after PTE writes (wmb, mb, clflush, WC,
   posting reads)

 - plain memory barriers before TAIL update (wmb, mb, posting read)

 - reordering PD LRI, extra LRI, extra invalidate, extra flushes, force
   context restores

 - serialising all PTE writes under a spinlock

This random sleep is the only thing to survive 10+ minutes. Similarly
applies to Braswell.

Hypothesis:

The async vma rearranged the execution flow such that the PTE updates
are immediate before the TAIL update (i.e. we go straight from PTE
update to execution) whereas before PTE updates were before we starting
writing into the ring. We've removed a lot of GGTT/WC writes between the
PTE and TAIL so the PTE writes are not coherent from the GPU.
---
 drivers/gpu/drm/i915/gt/intel_lrc.c | 59 +----------------------------
 drivers/gpu/drm/i915/i915_gem_gtt.c |  4 ++
 2 files changed, 5 insertions(+), 58 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 4dc0667f0c87..20ef828eba45 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -1871,60 +1871,6 @@ static int gen8_emit_init_breadcrumb(struct i915_request *rq)
 	return 0;
 }
 
-static int emit_pdps(struct i915_request *rq)
-{
-	const struct intel_engine_cs * const engine = rq->engine;
-	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->hw_context->vm);
-	int err, i;
-	u32 *cs;
-
-	GEM_BUG_ON(intel_vgpu_active(rq->i915));
-
-	/*
-	 * Beware ye of the dragons, this sequence is magic!
-	 *
-	 * Small changes to this sequence can cause anything from
-	 * GPU hangs to forcewake errors and machine lockups!
-	 */
-
-	/* Flush any residual operations from the context load */
-	err = engine->emit_flush(rq, EMIT_FLUSH);
-	if (err)
-		return err;
-
-	/* Magic required to prevent forcewake errors! */
-	err = engine->emit_flush(rq, EMIT_INVALIDATE);
-	if (err)
-		return err;
-
-	cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/* Ensure the LRI have landed before we invalidate & continue */
-	*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
-	for (i = GEN8_3LVL_PDPES; i--; ) {
-		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
-		u32 base = engine->mmio_base;
-
-		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
-		*cs++ = upper_32_bits(pd_daddr);
-		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
-		*cs++ = lower_32_bits(pd_daddr);
-	}
-	*cs++ = MI_NOOP;
-
-	intel_ring_advance(rq, cs);
-
-	/* Be doubly sure the LRI have landed before proceeding */
-	err = engine->emit_flush(rq, EMIT_FLUSH);
-	if (err)
-		return err;
-
-	/* Re-invalidate the TLB for luck */
-	return engine->emit_flush(rq, EMIT_INVALIDATE);
-}
-
 static int execlists_request_alloc(struct i915_request *request)
 {
 	struct intel_context *ce = request->hw_context;
@@ -1948,10 +1894,7 @@ static int execlists_request_alloc(struct i915_request *request)
 	 */
 
 	/* Unconditionally invalidate GPU caches and TLBs. */
-	if (i915_vm_is_4lvl(ce->vm))
-		ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
-	else
-		ret = emit_pdps(request);
+	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index dcbdab973f5d..be7cf409991e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -1359,6 +1359,7 @@ static void gen8_ppgtt_insert(struct i915_address_space *vm,
 
 		vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
 	}
+	msleep(2);
 }
 
 static int gen8_init_scratch(struct i915_address_space *vm)
@@ -1683,6 +1684,9 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 	} while (1);
 	kunmap_atomic(vaddr);
 
+	wmb();
+	msleep(20);
+
 	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
 }
 
-- 
2.23.0