[Mesa-dev] [PATCH 2/4] radeonsi: make sure CP DMA is idle at the end of IBs

[Marek Olšák]

From: Marek Olšák <marek.olsak at amd.com>

---
 src/gallium/drivers/radeonsi/si_cp_dma.c | 12 +++++++++++-
 src/gallium/drivers/radeonsi/si_gfx_cs.c |  5 ++++-
 src/gallium/drivers/radeonsi/si_pipe.h   |  1 +
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/gallium/drivers/radeonsi/si_cp_dma.c b/src/gallium/drivers/radeonsi/si_cp_dma.c
index 358b33c4eb1..b316637d94b 100644
--- a/src/gallium/drivers/radeonsi/si_cp_dma.c
+++ b/src/gallium/drivers/radeonsi/si_cp_dma.c
@@ -58,21 +58,20 @@ static inline unsigned cp_dma_max_byte_count(struct si_context *sctx)
  * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
  * clear value.
  */
 static void si_emit_cp_dma(struct si_context *sctx, uint64_t dst_va,
 			   uint64_t src_va, unsigned size, unsigned flags,
 			   enum si_coherency coher)
 {
 	struct radeon_winsys_cs *cs = sctx->gfx_cs;
 	uint32_t header = 0, command = 0;
 
-	assert(size);
 	assert(size <= cp_dma_max_byte_count(sctx));
 
 	if (sctx->chip_class >= GFX9)
 		command |= S_414_BYTE_COUNT_GFX9(size);
 	else
 		command |= S_414_BYTE_COUNT_GFX6(size);
 
 	/* Sync flags. */
 	if (flags & CP_DMA_SYNC)
 		header |= S_411_CP_SYNC(1);
@@ -121,20 +120,31 @@ static void si_emit_cp_dma(struct si_context *sctx, uint64_t dst_va,
 	 * This ensures that ME (CP DMA) is idle before PFP starts fetching
 	 * indices. If we wanted to execute CP DMA in PFP, this packet
 	 * should precede it.
 	 */
 	if (coher == SI_COHERENCY_SHADER && flags & CP_DMA_SYNC) {
 		radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
 		radeon_emit(cs, 0);
 	}
 }
 
+void si_cp_dma_wait_for_idle(struct si_context *sctx)
+{
+	/* Issue a dummy DMA that copies zero bytes.
+	 *
+	 * The DMA engine will see that there's no work to do and skip this
+	 * DMA request, however, the CP will see the sync flag and still wait
+	 * for all DMAs to complete.
+	 */
+	si_emit_cp_dma(sctx, 0, 0, 0, CP_DMA_SYNC, SI_COHERENCY_NONE);
+}
+
 static unsigned get_flush_flags(struct si_context *sctx, enum si_coherency coher)
 {
 	switch (coher) {
 	default:
 	case SI_COHERENCY_NONE:
 		return 0;
 	case SI_COHERENCY_SHADER:
 		return SI_CONTEXT_INV_SMEM_L1 |
 		       SI_CONTEXT_INV_VMEM_L1 |
 		       (sctx->chip_class == SI ? SI_CONTEXT_INV_GLOBAL_L2 : 0);
diff --git a/src/gallium/drivers/radeonsi/si_gfx_cs.c b/src/gallium/drivers/radeonsi/si_gfx_cs.c
index f99bc324c98..2d5e510b19e 100644
--- a/src/gallium/drivers/radeonsi/si_gfx_cs.c
+++ b/src/gallium/drivers/radeonsi/si_gfx_cs.c
@@ -104,21 +104,24 @@ void si_flush_gfx_cs(struct si_context *ctx, unsigned flags,
 	}
 
 	ctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
 			SI_CONTEXT_PS_PARTIAL_FLUSH;
 
 	/* DRM 3.1.0 doesn't flush TC for VI correctly. */
 	if (ctx->chip_class == VI && ctx->screen->info.drm_minor <= 1)
 		ctx->flags |= SI_CONTEXT_INV_GLOBAL_L2 |
 				SI_CONTEXT_INV_VMEM_L1;
 
-	si_emit_cache_flush(ctx);
+	/* Make sure CP DMA is idle at the end of IBs after L2 prefetches
+	 * because the kernel doesn't wait for it. */
+	if (ctx->chip_class >= CIK)
+		si_cp_dma_wait_for_idle(ctx);
 
 	if (ctx->current_saved_cs) {
 		si_trace_emit(ctx);
 		si_log_hw_flush(ctx);
 
 		/* Save the IB for debug contexts. */
 		si_save_cs(ws, cs, &ctx->current_saved_cs->gfx, true);
 		ctx->current_saved_cs->flushed = true;
 		ctx->current_saved_cs->time_flush = os_time_get_nano();
 	}
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index 8d4072c9c2e..0c90a6c6e46 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -883,20 +883,21 @@ void si_init_clear_functions(struct si_context *sctx);
 			   SI_CPDMA_SKIP_SYNC_BEFORE | \
 			   SI_CPDMA_SKIP_GFX_SYNC | \
 			   SI_CPDMA_SKIP_BO_LIST_UPDATE)
 
 enum si_coherency {
 	SI_COHERENCY_NONE, /* no cache flushes needed */
 	SI_COHERENCY_SHADER,
 	SI_COHERENCY_CB_META,
 };
 
+void si_cp_dma_wait_for_idle(struct si_context *sctx);
 void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
 		     uint64_t offset, uint64_t size, unsigned value,
 		     enum si_coherency coher);
 void si_copy_buffer(struct si_context *sctx,
 		    struct pipe_resource *dst, struct pipe_resource *src,
 		    uint64_t dst_offset, uint64_t src_offset, unsigned size,
 		    unsigned user_flags);
 void cik_prefetch_TC_L2_async(struct si_context *sctx, struct pipe_resource *buf,
 			      uint64_t offset, unsigned size);
 void cik_emit_prefetch_L2(struct si_context *sctx, bool vertex_stage_only);
-- 
2.15.1