[PATCH] drm/amdkfd: Support process XNACK mode dynamic change

[Felix Kuehling]

On 2023-01-12 16:18, Philip Yang wrote:
> Update queue qpd is done for the first queue creation of the process,
> if the device support XNACK mode per process, update qpd setup
> sh_mem_config based on the process XNACK mode, to support the process
> destory all queues, change XNACK mode, and then create queues.
>
> Add helper macro KFD_SUPPORT_XNACK_PER_PROCESS to remove duplicate code
> and add new ASICs support in future.
>
> Signed-off-by: Philip Yang <Philip.Yang at amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>


> ---
>   .../amd/amdkfd/kfd_device_queue_manager_v9.c  | 27 +++++++++----------
>   drivers/gpu/drm/amd/amdkfd/kfd_priv.h         |  2 ++
>   drivers/gpu/drm/amd/amdkfd/kfd_process.c      |  2 +-
>   3 files changed, 15 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
> index d119070956fb..8b2dd2670ab7 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
> @@ -59,30 +59,27 @@ static int update_qpd_v9(struct device_queue_manager *dqm,
>   
>   	/* check if sh_mem_config register already configured */
>   	if (qpd->sh_mem_config == 0) {
> -		qpd->sh_mem_config =
> -				SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
> +		qpd->sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
>   					SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
>   
> -		if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 2)) {
> -			/* Aldebaran can safely support different XNACK modes
> -			 * per process
> -			 */
> -			if (!pdd->process->xnack_enabled)
> -				qpd->sh_mem_config |=
> -					1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
> -		} else if (dqm->dev->noretry &&
> -			   !dqm->dev->use_iommu_v2) {
> -			qpd->sh_mem_config |=
> -				1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
> -		}
> +		if (dqm->dev->noretry && !dqm->dev->use_iommu_v2)
> +			qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
>   
>   		qpd->sh_mem_ape1_limit = 0;
>   		qpd->sh_mem_ape1_base = 0;
>   	}
>   
> +	if (KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev)) {
> +		if (!pdd->process->xnack_enabled)
> +			qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
> +		else
> +			qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
> +	}
> +
>   	qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
>   
> -	pr_debug("sh_mem_bases 0x%X\n", qpd->sh_mem_bases);
> +	pr_debug("sh_mem_bases 0x%X sh_mem_config 0x%X\n", qpd->sh_mem_bases,
> +		 qpd->sh_mem_config);
>   
>   	return 0;
>   }
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 552c3ac85a13..bfa30d12406b 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -206,6 +206,8 @@ enum cache_policy {
>   
>   #define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])
>   #define KFD_IS_SOC15(dev)   ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
> +#define KFD_SUPPORT_XNACK_PER_PROCESS(dev)\
> +		(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
>   
>   struct kfd_event_interrupt_class {
>   	bool (*interrupt_isr)(struct kfd_dev *dev,
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index 71db24fefe05..72df6286e240 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -1330,7 +1330,7 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
>   		 * per-process XNACK mode selection. But let the dev->noretry
>   		 * setting still influence the default XNACK mode.
>   		 */
> -		if (supported && KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
> +		if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev))
>   			continue;
>   
>   		/* GFXv10 and later GPUs do not support shader preemption