[Intel-xe] [PATCH] drm/xe: Reinstate render / compute cache invalidation in ring ops

[Matt Roper]

On Tue, Mar 21, 2023 at 06:34:39PM -0700, Matthew Brost wrote:
> Render / compute engines have additional caches (not just TLBs) that
> need to be invalidated each batch, reinstate these invalidations in ring
> ops.
> 
> v2: Rebase for CI
> v3: Add missing changes to xe_gpu_commands.h
> 
> Suggested-by: Matt Roper <matthew.d.roper at intel.com>
> Signed-off-by: Matthew Brost <matthew.brost at intel.com>

Reviewed-by: Matt Roper <matthew.d.roper at intel.com>

> ---
>  drivers/gpu/drm/xe/regs/xe_gpu_commands.h |  5 ++++
>  drivers/gpu/drm/xe/xe_ring_ops.c          | 30 +++++++++++++++++++++++
>  2 files changed, 35 insertions(+)
> 
> diff --git a/drivers/gpu/drm/xe/regs/xe_gpu_commands.h b/drivers/gpu/drm/xe/regs/xe_gpu_commands.h
> index e60372a82723..9d6508d74d62 100644
> --- a/drivers/gpu/drm/xe/regs/xe_gpu_commands.h
> +++ b/drivers/gpu/drm/xe/regs/xe_gpu_commands.h
> @@ -66,20 +66,25 @@
>  #define   PVC_MS_MOCS_INDEX_MASK	GENMASK(6, 1)
>  
>  #define GFX_OP_PIPE_CONTROL(len)	((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2))
> +#define   PIPE_CONTROL_COMMAND_CACHE_INVALIDATE		(1<<29)
>  #define   PIPE_CONTROL_TILE_CACHE_FLUSH			(1<<28)
>  #define   PIPE_CONTROL_AMFS_FLUSH			(1<<25)
>  #define   PIPE_CONTROL_GLOBAL_GTT_IVB			(1<<24)
> +#define   PIPE_CONTROL_STORE_DATA_INDEX			(1<<21)
>  #define   PIPE_CONTROL_CS_STALL				(1<<20)
>  #define   PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET		(1<<19)
>  #define   PIPE_CONTROL_PSD_SYNC				(1<<17)
>  #define   PIPE_CONTROL_QW_WRITE				(1<<14)
>  #define   PIPE_CONTROL_DEPTH_STALL			(1<<13)
>  #define   PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH	(1<<12)
> +#define   PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE	(1<<11)
>  #define   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE		(1<<10)
>  #define   PIPE_CONTROL_INDIRECT_STATE_DISABLE		(1<<9)
>  #define   PIPE_CONTROL_FLUSH_ENABLE			(1<<7)
>  #define   PIPE_CONTROL_DC_FLUSH_ENABLE			(1<<5)
>  #define   PIPE_CONTROL_VF_CACHE_INVALIDATE		(1<<4)
> +#define   PIPE_CONTROL_CONST_CACHE_INVALIDATE		(1<<3)
> +#define   PIPE_CONTROL_STATE_CACHE_INVALIDATE		(1<<2)
>  #define   PIPE_CONTROL_STALL_AT_SCOREBOARD		(1<<1)
>  #define   PIPE_CONTROL_DEPTH_CACHE_FLUSH		(1<<0)
>  
> diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c
> index 4441fe60cd00..918e819e7c91 100644
> --- a/drivers/gpu/drm/xe/xe_ring_ops.c
> +++ b/drivers/gpu/drm/xe/xe_ring_ops.c
> @@ -106,6 +106,30 @@ static int emit_flush_invalidate(u32 flag, u32 *dw, int i)
>  	return i;
>  }
>  
> +static int emit_pipe_invalidate(u32 mask_flags, u32 *dw, int i)
> +{
> +	u32 flags = PIPE_CONTROL_CS_STALL |
> +		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
> +		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
> +		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
> +		PIPE_CONTROL_VF_CACHE_INVALIDATE |
> +		PIPE_CONTROL_CONST_CACHE_INVALIDATE |
> +		PIPE_CONTROL_STATE_CACHE_INVALIDATE |
> +		PIPE_CONTROL_QW_WRITE |
> +		PIPE_CONTROL_STORE_DATA_INDEX;
> +
> +	flags &= ~mask_flags;
> +
> +	dw[i++] = GFX_OP_PIPE_CONTROL(6);
> +	dw[i++] = flags;
> +	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR;
> +	dw[i++] = 0;
> +	dw[i++] = 0;
> +	dw[i++] = 0;
> +
> +	return i;
> +}
> +
>  #define MI_STORE_QWORD_IMM_GEN8_POSTED (MI_INSTR(0x20, 3) | (1 << 21))
>  
>  static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
> @@ -212,8 +236,14 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
>  	struct xe_gt *gt = job->engine->gt;
>  	struct xe_device *xe = gt_to_xe(gt);
>  	bool pvc = xe->info.platform == XE_PVC;
> +	u32 mask_flags = 0;
>  
>  	dw[i++] = preparser_disable(true);
> +	if (pvc)
> +		mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
> +	else if (job->engine->class == XE_ENGINE_CLASS_COMPUTE)
> +		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
> +	i = emit_pipe_invalidate(mask_flags, dw, i);
>  	/* Wa_1809175790 */
>  	if (!xe->info.has_flat_ccs)
>  		i = emit_aux_table_inv(gt, GEN12_GFX_CCS_AUX_NV.reg, dw, i);
> -- 
> 2.34.1
> 

-- 
Matt Roper
Graphics Software Engineer
Linux GPU Platform Enablement
Intel Corporation