[PATCH 21/34] drm/amdkfd: add debug trap enabled flag to tma

[Felix Kuehling]

On 2023-03-27 14:43, Jonathan Kim wrote:
> From: Jay Cornwall <jay.cornwall at amd.com>
>
> Trap handler behavior will differ when a debugger is attached.
>
> Make the debug trap flag available in the trap handler TMA.
> Update it when the debug trap ioctl is invoked.
>
> v4: fix up comments to clarify flagging implementation.
>
> v3: Rebase for upstream
>
> v2:
> Add missing debug flag setup on APUs
>
> Signed-off-by: Jay Cornwall <jay.cornwall at amd.com>
> Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>
> Signed-off-by: Jonathan Kim <jonathan.kim at amd.com>
> ---
>   drivers/gpu/drm/amd/amdkfd/kfd_debug.c   | 11 +++++++++++
>   drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  2 ++
>   drivers/gpu/drm/amd/amdkfd/kfd_process.c | 15 +++++++++++++++
>   3 files changed, 28 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
> index f498987dc21d..c779acb9a623 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
> @@ -256,6 +256,8 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind
>   		if (unwind && i == unwind_count)
>   			break;
>   
> +		kfd_process_set_trap_debug_flag(&pdd->qpd, false);
> +
>   		/* GFX off is already disabled by debug activate if not RLC restore supported. */
>   		if (kfd_dbg_is_rlc_restore_supported(pdd->dev))
>   			amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
> @@ -351,6 +353,15 @@ int kfd_dbg_trap_activate(struct kfd_process *target)
>   		if (kfd_dbg_is_rlc_restore_supported(pdd->dev))
>   			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
>   
> +		/**

Not sure if I missed this in a previous review or it crept back in. This 
is not a well-formed doc comment. So it should not start with two *s. 
With that fixed, the patch is

Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>


> +		 * Setting the debug flag in the trap handler requires that the TMA has been
> +		 * allocated, which occurs during CWSR initialization.
> +		 * In the event that CWSR has not been initialized at this point, setting the
> +		 * flag will be called again during CWSR initialization if the target process
> +		 * is still debug enabled.
> +		 */
> +		kfd_process_set_trap_debug_flag(&pdd->qpd, true);
> +
>   		if (!pdd->dev->shared_resources.enable_mes)
>   			r = debug_refresh_runlist(pdd->dev->dqm);
>   		else
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index d092c81c9dc2..42a4502287f2 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -1105,6 +1105,8 @@ int kfd_init_apertures(struct kfd_process *process);
>   void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
>   				  uint64_t tba_addr,
>   				  uint64_t tma_addr);
> +void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
> +				     bool enabled);
>   
>   /* CWSR initialization */
>   int kfd_process_init_cwsr_apu(struct kfd_process *process, struct file *filep);
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index c6a4d01bb1b5..d26aa339fa6b 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -1303,6 +1303,8 @@ int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
>   
>   		memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
>   
> +		kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
> +
>   		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
>   		pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
>   			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
> @@ -1339,6 +1341,9 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
>   
>   	memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
>   
> +	kfd_process_set_trap_debug_flag(&pdd->qpd,
> +					pdd->process->debug_trap_enabled);
> +
>   	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
>   	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
>   		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
> @@ -1425,6 +1430,16 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
>   	return true;
>   }
>   
> +void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
> +				     bool enabled)
> +{
> +	if (qpd->cwsr_kaddr) {
> +		uint64_t *tma =
> +			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
> +		tma[2] = enabled;
> +	}
> +}
> +
>   /*
>    * On return the kfd_process is fully operational and will be freed when the
>    * mm is released