[PATCH 22/33] drm/amdkfd: update process interrupt handling for debug events

Felix Kuehling felix.kuehling at amd.com
Tue May 30 20:16:27 UTC 2023


Am 2023-05-25 um 13:27 schrieb Jonathan Kim:
> The debugger must be notified by any debugger subscribed exception
> that comes from hardware interrupts.
>
> If a debugger session exits, any exceptions it subscribed to may still
> have interrupts in the interrupt ring buffer or KGD/KFD pipeline.
> To prevent a new session from inheriting stale interrupts, when a new
> queue is created, open an interrupt drain and allow the IH ring to drain
> from a timestamped checkpoint.  Then inject a custom IV so that once
> the custom IV is picked up by the KFD, it's safe to close the drain
> and proceed with queue creation.
>
> The drain must also be on debug disable as SW interrupts may still
> be processed.  Drain at this time and clear all the exception status.
>
> The debugger may also not be attached nor subscibed to certain
> exceptions so forward them directly to the runtime.
>
> GFX10 also requires its own IV processing, hence the creation of
> kfd_int_process_v10.c.  This is because the IV from SQ interrupts are
> packed into a new continguous format unlike GFX9. To make this clear,
> a separate interrupting handling code file was created.
>
> v2: use new kfd_node struct in prototypes.
>
> Signed-off-by: Jonathan Kim <jonathan.kim at amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>


> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c    |  16 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h    |   2 +
>   drivers/gpu/drm/amd/amdkfd/Makefile           |   1 +
>   drivers/gpu/drm/amd/amdkfd/kfd_debug.c        |  84 ++++
>   drivers/gpu/drm/amd/amdkfd/kfd_debug.h        |   6 +
>   drivers/gpu/drm/amd/amdkfd/kfd_device.c       |   4 +-
>   .../gpu/drm/amd/amdkfd/kfd_int_process_v10.c  | 405 ++++++++++++++++++
>   .../gpu/drm/amd/amdkfd/kfd_int_process_v11.c  |  21 +-
>   .../gpu/drm/amd/amdkfd/kfd_int_process_v9.c   |  98 ++++-
>   drivers/gpu/drm/amd/amdkfd/kfd_priv.h         |  12 +
>   drivers/gpu/drm/amd/amdkfd/kfd_process.c      |  47 ++
>   .../amd/amdkfd/kfd_process_queue_manager.c    |   4 +
>   12 files changed, 680 insertions(+), 20 deletions(-)
>   create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index 66f80b9ab0c5..98cd52bb005f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -777,6 +777,22 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev, bo
>   	amdgpu_umc_poison_handler(adev, reset);
>   }
>   
> +int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
> +					uint32_t *payload)
> +{
> +	int ret;
> +
> +	/* Device or IH ring is not ready so bail. */
> +	ret = amdgpu_ih_wait_on_checkpoint_process_ts(adev, &adev->irq.ih);
> +	if (ret)
> +		return ret;
> +
> +	/* Send payload to fence KFD interrupts */
> +	amdgpu_amdkfd_interrupt(adev, payload);
> +
> +	return 0;
> +}
> +
>   bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev)
>   {
>   	if (adev->gfx.ras && adev->gfx.ras->query_utcl2_poison_status)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index 94cc456761e5..dd740e64e6e1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -250,6 +250,8 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,
>   					    struct amdgpu_device *src,
>   					    bool is_min);
>   int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min);
> +int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
> +					uint32_t *payload);
>   
>   /* Read user wptr from a specified user address space with page fault
>    * disabled. The memory must be pinned and mapped to the hardware when
> diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
> index 747754428073..2ec8f27c5366 100644
> --- a/drivers/gpu/drm/amd/amdkfd/Makefile
> +++ b/drivers/gpu/drm/amd/amdkfd/Makefile
> @@ -53,6 +53,7 @@ AMDKFD_FILES	:= $(AMDKFD_PATH)/kfd_module.o \
>   		$(AMDKFD_PATH)/kfd_events.o \
>   		$(AMDKFD_PATH)/cik_event_interrupt.o \
>   		$(AMDKFD_PATH)/kfd_int_process_v9.o \
> +		$(AMDKFD_PATH)/kfd_int_process_v10.o \
>   		$(AMDKFD_PATH)/kfd_int_process_v11.o \
>   		$(AMDKFD_PATH)/kfd_smi_events.o \
>   		$(AMDKFD_PATH)/kfd_crat.o \
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
> index 17e8e9edccbf..68b657398d41 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
> @@ -125,6 +125,64 @@ bool kfd_dbg_ev_raise(uint64_t event_mask,
>   	return is_subscribed;
>   }
>   
> +/* set pending event queue entry from ring entry  */
> +bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
> +				   unsigned int pasid,
> +				   uint32_t doorbell_id,
> +				   uint64_t trap_mask,
> +				   void *exception_data,
> +				   size_t exception_data_size)
> +{
> +	struct kfd_process *p;
> +	bool signaled_to_debugger_or_runtime = false;
> +
> +	p = kfd_lookup_process_by_pasid(pasid);
> +
> +	if (!p)
> +		return false;
> +
> +	if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true,
> +			      exception_data, exception_data_size)) {
> +		struct process_queue_manager *pqm;
> +		struct process_queue_node *pqn;
> +
> +		if (!!(trap_mask & KFD_EC_MASK_QUEUE) &&
> +		       p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) {
> +			mutex_lock(&p->mutex);
> +
> +			pqm = &p->pqm;
> +			list_for_each_entry(pqn, &pqm->queues,
> +							process_queue_list) {
> +
> +				if (!(pqn->q && pqn->q->device == dev &&
> +				      pqn->q->doorbell_id == doorbell_id))
> +					continue;
> +
> +				kfd_send_exception_to_runtime(p, pqn->q->properties.queue_id,
> +							      trap_mask);
> +
> +				signaled_to_debugger_or_runtime = true;
> +
> +				break;
> +			}
> +
> +			mutex_unlock(&p->mutex);
> +		} else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
> +			kfd_dqm_evict_pasid(dev->dqm, p->pasid);
> +			kfd_signal_vm_fault_event(dev, p->pasid, NULL,
> +							exception_data);
> +
> +			signaled_to_debugger_or_runtime = true;
> +		}
> +	} else {
> +		signaled_to_debugger_or_runtime = true;
> +	}
> +
> +	kfd_unref_process(p);
> +
> +	return signaled_to_debugger_or_runtime;
> +}
> +
>   int kfd_dbg_send_exception_to_runtime(struct kfd_process *p,
>   					unsigned int dev_id,
>   					unsigned int queue_id,
> @@ -281,6 +339,31 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind
>   	kfd_dbg_set_workaround(target, false);
>   }
>   
> +static void kfd_dbg_clean_exception_status(struct kfd_process *target)
> +{
> +	struct process_queue_manager *pqm;
> +	struct process_queue_node *pqn;
> +	int i;
> +
> +	for (i = 0; i < target->n_pdds; i++) {
> +		struct kfd_process_device *pdd = target->pdds[i];
> +
> +		kfd_process_drain_interrupts(pdd);
> +
> +		pdd->exception_status = 0;
> +	}
> +
> +	pqm = &target->pqm;
> +	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
> +		if (!pqn->q)
> +			continue;
> +
> +		pqn->q->properties.exception_status = 0;
> +	}
> +
> +	target->exception_status = 0;
> +}
> +
>   int kfd_dbg_trap_disable(struct kfd_process *target)
>   {
>   	if (!target->debug_trap_enabled)
> @@ -304,6 +387,7 @@ int kfd_dbg_trap_disable(struct kfd_process *target)
>   	}
>   
>   	target->debug_trap_enabled = false;
> +	kfd_dbg_clean_exception_status(target);
>   	kfd_unref_process(target);
>   
>   	return 0;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.h b/drivers/gpu/drm/amd/amdkfd/kfd_debug.h
> index fca928564948..5153ccbd7fd1 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.h
> @@ -27,6 +27,12 @@
>   
>   void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind_count);
>   int kfd_dbg_trap_activate(struct kfd_process *target);
> +bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
> +				   unsigned int pasid,
> +				   uint32_t doorbell_id,
> +				   uint64_t trap_mask,
> +				   void *exception_data,
> +				   size_t exception_data_size);
>   bool kfd_dbg_ev_raise(uint64_t event_mask,
>   			struct kfd_process *process, struct kfd_node *dev,
>   			unsigned int source_id, bool use_worker,
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> index 862a50f7b490..1eb57a4798f4 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> @@ -140,6 +140,8 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
>   	case IP_VERSION(9, 4, 1): /* ARCTURUS */
>   	case IP_VERSION(9, 4, 2): /* ALDEBARAN */
>   	case IP_VERSION(9, 4, 3): /* GC 9.4.3 */
> +		kfd->device_info.event_interrupt_class = &event_interrupt_class_v9;
> +		break;
>   	case IP_VERSION(10, 3, 1): /* VANGOGH */
>   	case IP_VERSION(10, 3, 3): /* YELLOW_CARP */
>   	case IP_VERSION(10, 3, 6): /* GC 10.3.6 */
> @@ -153,7 +155,7 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
>   	case IP_VERSION(10, 3, 2): /* NAVY_FLOUNDER */
>   	case IP_VERSION(10, 3, 4): /* DIMGREY_CAVEFISH */
>   	case IP_VERSION(10, 3, 5): /* BEIGE_GOBY */
> -		kfd->device_info.event_interrupt_class = &event_interrupt_class_v9;
> +		kfd->device_info.event_interrupt_class = &event_interrupt_class_v10;
>   		break;
>   	case IP_VERSION(11, 0, 0):
>   	case IP_VERSION(11, 0, 1):
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
> new file mode 100644
> index 000000000000..c7991e07b6be
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
> @@ -0,0 +1,405 @@
> +/*
> + * Copyright 2023 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "kfd_events.h"
> +#include "kfd_debug.h"
> +#include "soc15_int.h"
> +#include "kfd_device_queue_manager.h"
> +
> +/*
> + * GFX10 SQ Interrupts
> + *
> + * There are 3 encoding types of interrupts sourced from SQ sent as a 44-bit
> + * packet to the Interrupt Handler:
> + * Auto - Generated by the SQG (various cmd overflows, timestamps etc)
> + * Wave - Generated by S_SENDMSG through a shader program
> + * Error - HW generated errors (Illegal instructions, Memviols, EDC etc)
> + *
> + * The 44-bit packet is mapped as {context_id1[7:0],context_id0[31:0]} plus
> + * 4-bits for VMID (SOC15_VMID_FROM_IH_ENTRY) as such:
> + *
> + * - context_id1[7:6]
> + * Encoding type (0 = Auto, 1 = Wave, 2 = Error)
> + *
> + * - context_id0[24]
> + * PRIV bit indicates that Wave S_SEND or error occurred within trap
> + *
> + * - context_id0[22:0]
> + * 23-bit data with the following layout per encoding type:
> + * Auto - only context_id0[8:0] is used, which reports various interrupts
> + * generated by SQG.  The rest is 0.
> + * Wave - user data sent from m0 via S_SENDMSG
> + * Error - Error type (context_id0[22:19]), Error Details (rest of bits)
> + *
> + * The other context_id bits show coordinates (SE/SH/CU/SIMD/WGP) for wave
> + * S_SENDMSG and Errors.  These are 0 for Auto.
> + */
> +
> +enum SQ_INTERRUPT_WORD_ENCODING {
> +	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
> +	SQ_INTERRUPT_WORD_ENCODING_INST,
> +	SQ_INTERRUPT_WORD_ENCODING_ERROR,
> +};
> +
> +enum SQ_INTERRUPT_ERROR_TYPE {
> +	SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0,
> +	SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST,
> +	SQ_INTERRUPT_ERROR_TYPE_MEMVIOL,
> +	SQ_INTERRUPT_ERROR_TYPE_EDC_FED,
> +};
> +
> +/* SQ_INTERRUPT_WORD_AUTO_CTXID */
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE__SHIFT 0
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__WLT__SHIFT 1
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF0_FULL__SHIFT 2
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF1_FULL__SHIFT 3
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_UTC_ERROR__SHIFT 7
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID1__SE_ID__SHIFT 4
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID1__ENCODING__SHIFT 6
> +
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_MASK 0x00000001
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__WLT_MASK 0x00000002
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF0_FULL_MASK 0x00000004
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF1_FULL_MASK 0x00000008
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_UTC_ERROR_MASK 0x00000080
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID1__SE_ID_MASK 0x030
> +#define SQ_INTERRUPT_WORD_AUTO_CTXID1__ENCODING_MASK 0x0c0
> +
> +/* SQ_INTERRUPT_WORD_WAVE_CTXID */
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__DATA__SHIFT 0
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__SA_ID__SHIFT 23
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__PRIV__SHIFT 24
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__WAVE_ID__SHIFT 25
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__SIMD_ID__SHIFT 30
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID1__WGP_ID__SHIFT 0
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID1__SE_ID__SHIFT 4
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID1__ENCODING__SHIFT 6
> +
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__DATA_MASK 0x000007fffff
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__SA_ID_MASK 0x0000800000
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__PRIV_MASK 0x00001000000
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__WAVE_ID_MASK 0x0003e000000
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID0__SIMD_ID_MASK 0x000c0000000
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID1__WGP_ID_MASK 0x00f
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID1__SE_ID_MASK 0x030
> +#define SQ_INTERRUPT_WORD_WAVE_CTXID1__ENCODING_MASK 0x0c0
> +
> +#define KFD_CTXID0__ERR_TYPE_MASK 0x780000
> +#define KFD_CTXID0__ERR_TYPE__SHIFT 19
> +
> +/* GFX10 SQ interrupt ENC type bit (context_id1[7:6]) for wave s_sendmsg */
> +#define KFD_CONTEXT_ID1_ENC_TYPE_WAVE_MASK	0x40
> +/* GFX10 SQ interrupt PRIV bit (context_id0[24]) for s_sendmsg inside trap */
> +#define KFD_CONTEXT_ID0_PRIV_MASK		0x1000000
> +/*
> + * The debugger will send user data(m0) with PRIV=1 to indicate it requires
> + * notification from the KFD with the following queue id (DOORBELL_ID) and
> + * trap code (TRAP_CODE).
> + */
> +#define KFD_CONTEXT_ID0_DEBUG_DOORBELL_MASK	0x0003ff
> +#define KFD_CONTEXT_ID0_DEBUG_TRAP_CODE_SHIFT	10
> +#define KFD_CONTEXT_ID0_DEBUG_TRAP_CODE_MASK	0x07fc00
> +#define KFD_DEBUG_DOORBELL_ID(ctxid0)	((ctxid0) &	\
> +				KFD_CONTEXT_ID0_DEBUG_DOORBELL_MASK)
> +#define KFD_DEBUG_TRAP_CODE(ctxid0)	(((ctxid0) &	\
> +				KFD_CONTEXT_ID0_DEBUG_TRAP_CODE_MASK)	\
> +				>> KFD_CONTEXT_ID0_DEBUG_TRAP_CODE_SHIFT)
> +#define KFD_DEBUG_CP_BAD_OP_ECODE_MASK		0x3fffc00
> +#define KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT		10
> +#define KFD_DEBUG_CP_BAD_OP_ECODE(ctxid0) (((ctxid0) &			\
> +				KFD_DEBUG_CP_BAD_OP_ECODE_MASK)		\
> +				>> KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT)
> +
> +static void event_interrupt_poison_consumption(struct kfd_node *dev,
> +				uint16_t pasid, uint16_t client_id)
> +{
> +	int old_poison, ret = -EINVAL;
> +	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
> +
> +	if (!p)
> +		return;
> +
> +	/* all queues of a process will be unmapped in one time */
> +	old_poison = atomic_cmpxchg(&p->poison, 0, 1);
> +	kfd_unref_process(p);
> +	if (old_poison)
> +		return;
> +
> +	switch (client_id) {
> +	case SOC15_IH_CLIENTID_SE0SH:
> +	case SOC15_IH_CLIENTID_SE1SH:
> +	case SOC15_IH_CLIENTID_SE2SH:
> +	case SOC15_IH_CLIENTID_SE3SH:
> +	case SOC15_IH_CLIENTID_UTCL2:
> +		ret = kfd_dqm_evict_pasid(dev->dqm, pasid);
> +		break;
> +	case SOC15_IH_CLIENTID_SDMA0:
> +	case SOC15_IH_CLIENTID_SDMA1:
> +	case SOC15_IH_CLIENTID_SDMA2:
> +	case SOC15_IH_CLIENTID_SDMA3:
> +	case SOC15_IH_CLIENTID_SDMA4:
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	kfd_signal_poison_consumed_event(dev, pasid);
> +
> +	/* resetting queue passes, do page retirement without gpu reset
> +	 * resetting queue fails, fallback to gpu reset solution
> +	 */
> +	if (!ret) {
> +		dev_warn(dev->adev->dev,
> +			"RAS poison consumption, unmap queue flow succeeded: client id %d\n",
> +			client_id);
> +		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, false);
> +	} else {
> +		dev_warn(dev->adev->dev,
> +			"RAS poison consumption, fall back to gpu reset flow: client id %d\n",
> +			client_id);
> +		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, true);
> +	}
> +}
> +
> +static bool event_interrupt_isr_v10(struct kfd_node *dev,
> +					const uint32_t *ih_ring_entry,
> +					uint32_t *patched_ihre,
> +					bool *patched_flag)
> +{
> +	uint16_t source_id, client_id, pasid, vmid;
> +	const uint32_t *data = ih_ring_entry;
> +
> +	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
> +	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
> +
> +	/* Only handle interrupts from KFD VMIDs */
> +	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
> +	if (!KFD_IRQ_IS_FENCE(client_id, source_id) &&
> +	   (vmid < dev->vm_info.first_vmid_kfd ||
> +	    vmid > dev->vm_info.last_vmid_kfd))
> +		return false;
> +
> +	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
> +
> +	/* Only handle clients we care about */
> +	if (client_id != SOC15_IH_CLIENTID_GRBM_CP &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA0 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA1 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA2 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA3 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA4 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA5 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA6 &&
> +	    client_id != SOC15_IH_CLIENTID_SDMA7 &&
> +	    client_id != SOC15_IH_CLIENTID_VMC &&
> +	    client_id != SOC15_IH_CLIENTID_VMC1 &&
> +	    client_id != SOC15_IH_CLIENTID_UTCL2 &&
> +	    client_id != SOC15_IH_CLIENTID_SE0SH &&
> +	    client_id != SOC15_IH_CLIENTID_SE1SH &&
> +	    client_id != SOC15_IH_CLIENTID_SE2SH &&
> +	    client_id != SOC15_IH_CLIENTID_SE3SH)
> +		return false;
> +
> +	pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n",
> +		 client_id, source_id, vmid, pasid);
> +	pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
> +		 data[0], data[1], data[2], data[3],
> +		 data[4], data[5], data[6], data[7]);
> +
> +	/* If there is no valid PASID, it's likely a bug */
> +	if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt"))
> +		return 0;
> +
> +	/* Interrupt types we care about: various signals and faults.
> +	 * They will be forwarded to a work queue (see below).
> +	 */
> +	return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
> +		source_id == SOC15_INTSRC_SDMA_TRAP ||
> +		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
> +		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
> +		client_id == SOC15_IH_CLIENTID_VMC ||
> +		client_id == SOC15_IH_CLIENTID_VMC1 ||
> +		client_id == SOC15_IH_CLIENTID_UTCL2 ||
> +		KFD_IRQ_IS_FENCE(client_id, source_id);
> +}
> +
> +static void event_interrupt_wq_v10(struct kfd_node *dev,
> +					const uint32_t *ih_ring_entry)
> +{
> +	uint16_t source_id, client_id, pasid, vmid;
> +	uint32_t context_id0, context_id1;
> +	uint32_t encoding, sq_intr_err_type;
> +
> +	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
> +	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
> +	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
> +	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
> +	context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
> +	context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry);
> +
> +	if (client_id == SOC15_IH_CLIENTID_GRBM_CP ||
> +	    client_id == SOC15_IH_CLIENTID_SE0SH ||
> +	    client_id == SOC15_IH_CLIENTID_SE1SH ||
> +	    client_id == SOC15_IH_CLIENTID_SE2SH ||
> +	    client_id == SOC15_IH_CLIENTID_SE3SH) {
> +		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
> +			kfd_signal_event_interrupt(pasid, context_id0, 32);
> +		else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
> +			encoding = REG_GET_FIELD(context_id1,
> +						SQ_INTERRUPT_WORD_WAVE_CTXID1, ENCODING);
> +			switch (encoding) {
> +			case SQ_INTERRUPT_WORD_ENCODING_AUTO:
> +				pr_debug(
> +					"sq_intr: auto, se %d, ttrace %d, wlt %d, ttrac_buf0_full %d, ttrac_buf1_full %d, ttrace_utc_err %d\n",
> +					REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_AUTO_CTXID1,
> +							SE_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0,
> +							THREAD_TRACE),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0,
> +							WLT),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0,
> +							THREAD_TRACE_BUF0_FULL),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0,
> +							THREAD_TRACE_BUF1_FULL),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0,
> +							THREAD_TRACE_UTC_ERROR));
> +				break;
> +			case SQ_INTERRUPT_WORD_ENCODING_INST:
> +				pr_debug("sq_intr: inst, se %d, data 0x%x, sa %d, priv %d, wave_id %d, simd_id %d, wgp_id %d\n",
> +					REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1,
> +							SE_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							DATA),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							SA_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							PRIV),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							WAVE_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							SIMD_ID),
> +					REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1,
> +							WGP_ID));
> +				if (context_id0 & SQ_INTERRUPT_WORD_WAVE_CTXID0__PRIV_MASK) {
> +					if (kfd_set_dbg_ev_from_interrupt(dev, pasid,
> +							KFD_DEBUG_DOORBELL_ID(context_id0),
> +							KFD_DEBUG_TRAP_CODE(context_id0),
> +							NULL, 0))
> +						return;
> +				}
> +				break;
> +			case SQ_INTERRUPT_WORD_ENCODING_ERROR:
> +				sq_intr_err_type = REG_GET_FIELD(context_id0, KFD_CTXID0,
> +								ERR_TYPE);
> +				pr_warn("sq_intr: error, se %d, data 0x%x, sa %d, priv %d, wave_id %d, simd_id %d, wgp_id %d, err_type %d\n",
> +					REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1,
> +							SE_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							DATA),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							SA_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							PRIV),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							WAVE_ID),
> +					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0,
> +							SIMD_ID),
> +					REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1,
> +							WGP_ID),
> +					sq_intr_err_type);
> +				if (sq_intr_err_type != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
> +					sq_intr_err_type != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
> +					event_interrupt_poison_consumption(dev, pasid, source_id);
> +					return;
> +				}
> +				break;
> +			default:
> +				break;
> +			}
> +			kfd_signal_event_interrupt(pasid, context_id0 & 0x7fffff, 23);
> +		} else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) {
> +			kfd_set_dbg_ev_from_interrupt(dev, pasid,
> +				KFD_DEBUG_DOORBELL_ID(context_id0),
> +				KFD_EC_MASK(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0)),
> +				NULL,
> +				0);
> +		}
> +	} else if (client_id == SOC15_IH_CLIENTID_SDMA0 ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA1 ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA2 ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA3 ||
> +		   (client_id == SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid &&
> +		    KFD_GC_VERSION(dev) == IP_VERSION(10, 3, 0)) ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA4 ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA5 ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA6 ||
> +		   client_id == SOC15_IH_CLIENTID_SDMA7) {
> +		if (source_id == SOC15_INTSRC_SDMA_TRAP) {
> +			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
> +		} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
> +			event_interrupt_poison_consumption(dev, pasid, source_id);
> +			return;
> +		}
> +	} else if (client_id == SOC15_IH_CLIENTID_VMC ||
> +		   client_id == SOC15_IH_CLIENTID_VMC1 ||
> +		   client_id == SOC15_IH_CLIENTID_UTCL2) {
> +		struct kfd_vm_fault_info info = {0};
> +		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
> +		struct kfd_hsa_memory_exception_data exception_data;
> +
> +		if (client_id == SOC15_IH_CLIENTID_UTCL2 &&
> +				amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev)) {
> +			event_interrupt_poison_consumption(dev, pasid, client_id);
> +			return;
> +		}
> +
> +		info.vmid = vmid;
> +		info.mc_id = client_id;
> +		info.page_addr = ih_ring_entry[4] |
> +			(uint64_t)(ih_ring_entry[5] & 0xf) << 32;
> +		info.prot_valid = ring_id & 0x08;
> +		info.prot_read  = ring_id & 0x10;
> +		info.prot_write = ring_id & 0x20;
> +
> +		memset(&exception_data, 0, sizeof(exception_data));
> +		exception_data.gpu_id = dev->id;
> +		exception_data.va = (info.page_addr) << PAGE_SHIFT;
> +		exception_data.failure.NotPresent = info.prot_valid ? 1 : 0;
> +		exception_data.failure.NoExecute = info.prot_exec ? 1 : 0;
> +		exception_data.failure.ReadOnly = info.prot_write ? 1 : 0;
> +		exception_data.failure.imprecise = 0;
> +
> +		kfd_set_dbg_ev_from_interrupt(dev,
> +						pasid,
> +						-1,
> +						KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION),
> +						&exception_data,
> +						sizeof(exception_data));
> +	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
> +		kfd_process_close_interrupt_drain(pasid);
> +	}
> +}
> +
> +const struct kfd_event_interrupt_class event_interrupt_class_v10 = {
> +	.interrupt_isr = event_interrupt_isr_v10,
> +	.interrupt_wq = event_interrupt_wq_v10,
> +};
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
> index 0f0fdea4cd8a..1f92c6c6f181 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
> @@ -26,6 +26,7 @@
>   #include "kfd_device_queue_manager.h"
>   #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
>   #include "kfd_smi_events.h"
> +#include "kfd_debug.h"
>   
>   /*
>    * GFX11 SQ Interrupts
> @@ -238,7 +239,7 @@ static bool event_interrupt_isr_v11(struct kfd_node *dev,
>   	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
>   	/* Only handle interrupts from KFD VMIDs */
>   	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
> -	if (/*!KFD_IRQ_IS_FENCE(client_id, source_id) &&*/
> +	if (!KFD_IRQ_IS_FENCE(client_id, source_id) &&
>   	    (vmid < dev->vm_info.first_vmid_kfd ||
>   	    vmid > dev->vm_info.last_vmid_kfd))
>   		return false;
> @@ -267,7 +268,7 @@ static bool event_interrupt_isr_v11(struct kfd_node *dev,
>   		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
>   		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
>   		source_id == SOC21_INTSRC_SDMA_TRAP ||
> -		/* KFD_IRQ_IS_FENCE(client_id, source_id) || */
> +		KFD_IRQ_IS_FENCE(client_id, source_id) ||
>   		(((client_id == SOC21_IH_CLIENTID_VMC) ||
>   		 ((client_id == SOC21_IH_CLIENTID_GFX) &&
>   		  (source_id == UTCL2_1_0__SRCID__FAULT))) &&
> @@ -312,9 +313,9 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
>   		exception_data.failure.ReadOnly = info.prot_write ? 1 : 0;
>   		exception_data.failure.imprecise = 0;
>   
> -		/*kfd_set_dbg_ev_from_interrupt(dev, pasid, -1,
> +		kfd_set_dbg_ev_from_interrupt(dev, pasid, -1,
>   					      KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION),
> -					      &exception_data, sizeof(exception_data));*/
> +					      &exception_data, sizeof(exception_data));
>   		kfd_smi_event_update_vmfault(dev, pasid);
>   
>   	/* GRBM, SDMA, SE, PMM */
> @@ -324,11 +325,11 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
>   		/* CP */
>   		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
>   			kfd_signal_event_interrupt(pasid, context_id0, 32);
> -		/*else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
> +		else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
>   			kfd_set_dbg_ev_from_interrupt(dev, pasid,
>   				KFD_CTXID0_DOORBELL_ID(context_id0),
>   				KFD_EC_MASK(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0)),
> -				NULL, 0);*/
> +				NULL, 0);
>   
>   		/* SDMA */
>   		else if (source_id == SOC21_INTSRC_SDMA_TRAP)
> @@ -350,11 +351,11 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
>   				print_sq_intr_info_inst(context_id0, context_id1);
>   				sq_int_priv = REG_GET_FIELD(context_id0,
>   						SQ_INTERRUPT_WORD_WAVE_CTXID0, PRIV);
> -				/*if (sq_int_priv && (kfd_set_dbg_ev_from_interrupt(dev, pasid,
> +				if (sq_int_priv && (kfd_set_dbg_ev_from_interrupt(dev, pasid,
>   						KFD_CTXID0_DOORBELL_ID(context_id0),
>   						KFD_CTXID0_TRAP_CODE(context_id0),
>   						NULL, 0)))
> -					return;*/
> +					return;
>   				break;
>   			case SQ_INTERRUPT_WORD_ENCODING_ERROR:
>   				print_sq_intr_info_error(context_id0, context_id1);
> @@ -373,8 +374,8 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
>   			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
>   		}
>   
> -	/*} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
> -		kfd_process_close_interrupt_drain(pasid);*/
> +	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
> +		kfd_process_close_interrupt_drain(pasid);
>   	}
>   }
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
> index 8cf58be80f4e..d5c9f30552e3 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
> @@ -23,10 +23,40 @@
>   
>   #include "kfd_priv.h"
>   #include "kfd_events.h"
> +#include "kfd_debug.h"
>   #include "soc15_int.h"
>   #include "kfd_device_queue_manager.h"
>   #include "kfd_smi_events.h"
>   
> +/*
> + * GFX9 SQ Interrupts
> + *
> + * There are 3 encoding types of interrupts sourced from SQ sent as a 44-bit
> + * packet to the Interrupt Handler:
> + * Auto - Generated by the SQG (various cmd overflows, timestamps etc)
> + * Wave - Generated by S_SENDMSG through a shader program
> + * Error - HW generated errors (Illegal instructions, Memviols, EDC etc)
> + *
> + * The 44-bit packet is mapped as {context_id1[7:0],context_id0[31:0]} plus
> + * 4-bits for VMID (SOC15_VMID_FROM_IH_ENTRY) as such:
> + *
> + * - context_id0[27:26]
> + * Encoding type (0 = Auto, 1 = Wave, 2 = Error)
> + *
> + * - context_id0[13]
> + * PRIV bit indicates that Wave S_SEND or error occurred within trap
> + *
> + * - {context_id1[7:0],context_id0[31:28],context_id0[11:0]}
> + * 24-bit data with the following layout per encoding type:
> + * Auto - only context_id0[8:0] is used, which reports various interrupts
> + * generated by SQG.  The rest is 0.
> + * Wave - user data sent from m0 via S_SENDMSG
> + * Error - Error type (context_id1[7:4]), Error Details (rest of bits)
> + *
> + * The other context_id bits show coordinates (SE/SH/CU/SIMD/WAVE) for wave
> + * S_SENDMSG and Errors.  These are 0 for Auto.
> + */
> +
>   enum SQ_INTERRUPT_WORD_ENCODING {
>   	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
>   	SQ_INTERRUPT_WORD_ENCODING_INST,
> @@ -84,12 +114,32 @@ enum SQ_INTERRUPT_ERROR_TYPE {
>   #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK 0x03000000
>   #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK 0x0c000000
>   
> +/* GFX9 SQ interrupt 24-bit data from context_id<0,1> */
>   #define KFD_CONTEXT_ID_GET_SQ_INT_DATA(ctx0, ctx1)                             \
>   	((ctx0 & 0xfff) | ((ctx0 >> 16) & 0xf000) | ((ctx1 << 16) & 0xff0000))
>   
>   #define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000
>   #define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
>   
> +/*
> + * The debugger will send user data(m0) with PRIV=1 to indicate it requires
> + * notification from the KFD with the following queue id (DOORBELL_ID) and
> + * trap code (TRAP_CODE).
> + */
> +#define KFD_INT_DATA_DEBUG_DOORBELL_MASK	0x0003ff
> +#define KFD_INT_DATA_DEBUG_TRAP_CODE_SHIFT	10
> +#define KFD_INT_DATA_DEBUG_TRAP_CODE_MASK	0x07fc00
> +#define KFD_DEBUG_DOORBELL_ID(sq_int_data)	((sq_int_data) &	\
> +				KFD_INT_DATA_DEBUG_DOORBELL_MASK)
> +#define KFD_DEBUG_TRAP_CODE(sq_int_data)	(((sq_int_data) &	\
> +				KFD_INT_DATA_DEBUG_TRAP_CODE_MASK)	\
> +				>> KFD_INT_DATA_DEBUG_TRAP_CODE_SHIFT)
> +#define KFD_DEBUG_CP_BAD_OP_ECODE_MASK		0x3fffc00
> +#define KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT		10
> +#define KFD_DEBUG_CP_BAD_OP_ECODE(ctxid0)	(((ctxid0) &		\
> +				KFD_DEBUG_CP_BAD_OP_ECODE_MASK)		\
> +				>> KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT)
> +
>   static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
>   				uint16_t pasid, uint16_t client_id)
>   {
> @@ -168,14 +218,16 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev,
>   	uint16_t source_id, client_id, pasid, vmid;
>   	const uint32_t *data = ih_ring_entry;
>   
> +	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
> +	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
> +
>   	/* Only handle interrupts from KFD VMIDs */
>   	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
> -	if (vmid < dev->vm_info.first_vmid_kfd ||
> -	    vmid > dev->vm_info.last_vmid_kfd)
> +	if (!KFD_IRQ_IS_FENCE(client_id, source_id) &&
> +	   (vmid < dev->vm_info.first_vmid_kfd ||
> +	    vmid > dev->vm_info.last_vmid_kfd))
>   		return false;
>   
> -	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
> -	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
>   	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
>   
>   	/* Only handle clients we care about */
> @@ -194,7 +246,8 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev,
>   	    client_id != SOC15_IH_CLIENTID_SE0SH &&
>   	    client_id != SOC15_IH_CLIENTID_SE1SH &&
>   	    client_id != SOC15_IH_CLIENTID_SE2SH &&
> -	    client_id != SOC15_IH_CLIENTID_SE3SH)
> +	    client_id != SOC15_IH_CLIENTID_SE3SH &&
> +	    !KFD_IRQ_IS_FENCE(client_id, source_id))
>   		return false;
>   
>   	/* This is a known issue for gfx9. Under non HWS, pasid is not set
> @@ -247,6 +300,7 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev,
>   		source_id == SOC15_INTSRC_SDMA_ECC ||
>   		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
>   		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
> +		KFD_IRQ_IS_FENCE(client_id, source_id) ||
>   		((client_id == SOC15_IH_CLIENTID_VMC ||
>   		client_id == SOC15_IH_CLIENTID_VMC1 ||
>   		client_id == SOC15_IH_CLIENTID_UTCL2) &&
> @@ -302,6 +356,13 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
>   					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
>   					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
>   					sq_int_data);
> +				if (context_id0 & SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK) {
> +					if (kfd_set_dbg_ev_from_interrupt(dev, pasid,
> +							KFD_DEBUG_DOORBELL_ID(sq_int_data),
> +							KFD_DEBUG_TRAP_CODE(sq_int_data),
> +							NULL, 0))
> +						return;
> +				}
>   				break;
>   			case SQ_INTERRUPT_WORD_ENCODING_ERROR:
>   				sq_intr_err = REG_GET_FIELD(sq_int_data, KFD_SQ_INT_DATA, ERR_TYPE);
> @@ -324,8 +385,12 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
>   				break;
>   			}
>   			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
> -		} else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
> -			kfd_signal_hw_exception_event(pasid);
> +		} else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) {
> +			kfd_set_dbg_ev_from_interrupt(dev, pasid,
> +				KFD_DEBUG_DOORBELL_ID(context_id0),
> +				KFD_EC_MASK(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0)),
> +				NULL, 0);
> +		}
>   	} else if (client_id == SOC15_IH_CLIENTID_SDMA0 ||
>   		   client_id == SOC15_IH_CLIENTID_SDMA1 ||
>   		   client_id == SOC15_IH_CLIENTID_SDMA2 ||
> @@ -345,6 +410,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
>   		   client_id == SOC15_IH_CLIENTID_UTCL2) {
>   		struct kfd_vm_fault_info info = {0};
>   		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
> +		struct kfd_hsa_memory_exception_data exception_data;
>   
>   		if (client_id == SOC15_IH_CLIENTID_UTCL2 &&
>   		    amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev)) {
> @@ -360,9 +426,23 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
>   		info.prot_read  = ring_id & 0x10;
>   		info.prot_write = ring_id & 0x20;
>   
> +		memset(&exception_data, 0, sizeof(exception_data));
> +		exception_data.gpu_id = dev->id;
> +		exception_data.va = (info.page_addr) << PAGE_SHIFT;
> +		exception_data.failure.NotPresent = info.prot_valid ? 1 : 0;
> +		exception_data.failure.NoExecute = info.prot_exec ? 1 : 0;
> +		exception_data.failure.ReadOnly = info.prot_write ? 1 : 0;
> +		exception_data.failure.imprecise = 0;
> +
> +		kfd_set_dbg_ev_from_interrupt(dev,
> +						pasid,
> +						-1,
> +						KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION),
> +						&exception_data,
> +						sizeof(exception_data));
>   		kfd_smi_event_update_vmfault(dev, pasid);
> -		kfd_dqm_evict_pasid(dev->dqm, pasid);
> -		kfd_signal_vm_fault_event(dev, pasid, &info, NULL);
> +	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
> +		kfd_process_close_interrupt_drain(pasid);
>   	}
>   }
>   
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index a02fb939614a..cd2d56e5cdf0 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -963,6 +963,10 @@ struct kfd_process {
>   	uint64_t exception_enable_mask;
>   	uint64_t exception_status;
>   
> +	/* Used to drain stale interrupts */
> +	wait_queue_head_t wait_irq_drain;
> +	bool irq_drain_is_open;
> +
>   	/* shared virtual memory registered by this process */
>   	struct svm_range_list svms;
>   
> @@ -1144,12 +1148,19 @@ int kfd_numa_node_to_apic_id(int numa_node_id);
>   void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
>   
>   /* Interrupts */
> +#define	KFD_IRQ_FENCE_CLIENTID	0xff
> +#define	KFD_IRQ_FENCE_SOURCEID	0xff
> +#define	KFD_IRQ_IS_FENCE(client, source)				\
> +				((client) == KFD_IRQ_FENCE_CLIENTID &&	\
> +				(source) == KFD_IRQ_FENCE_SOURCEID)
>   int kfd_interrupt_init(struct kfd_node *dev);
>   void kfd_interrupt_exit(struct kfd_node *dev);
>   bool enqueue_ih_ring_entry(struct kfd_node *kfd, const void *ih_ring_entry);
>   bool interrupt_is_wanted(struct kfd_node *dev,
>   				const uint32_t *ih_ring_entry,
>   				uint32_t *patched_ihre, bool *flag);
> +int kfd_process_drain_interrupts(struct kfd_process_device *pdd);
> +void kfd_process_close_interrupt_drain(unsigned int pasid);
>   
>   /* amdkfd Apertures */
>   int kfd_init_apertures(struct kfd_process *process);
> @@ -1421,6 +1432,7 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
>   /* Events */
>   extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
>   extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
> +extern const struct kfd_event_interrupt_class event_interrupt_class_v10;
>   extern const struct kfd_event_interrupt_class event_interrupt_class_v11;
>   
>   extern const struct kfd_device_global_init_class device_global_init_class_cik;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index 2a60c630ab5d..e3a27c31f61d 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -862,6 +862,8 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
>   		kfd_procfs_add_sysfs_stats(process);
>   		kfd_procfs_add_sysfs_files(process);
>   		kfd_procfs_add_sysfs_counters(process);
> +
> +		init_waitqueue_head(&process->wait_irq_drain);
>   	}
>   out:
>   	if (!IS_ERR(process))
> @@ -2136,6 +2138,51 @@ void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
>   	}
>   }
>   
> +/* assumes caller holds process lock. */
> +int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
> +{
> +	uint32_t irq_drain_fence[8];
> +	int r = 0;
> +
> +	if (!KFD_IS_SOC15(pdd->dev))
> +		return 0;
> +
> +	pdd->process->irq_drain_is_open = true;
> +
> +	memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
> +	irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
> +							KFD_IRQ_FENCE_CLIENTID;
> +	irq_drain_fence[3] = pdd->process->pasid;
> +
> +	/* ensure stale irqs scheduled KFD interrupts and send drain fence. */
> +	if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
> +						     irq_drain_fence)) {
> +		pdd->process->irq_drain_is_open = false;
> +		return 0;
> +	}
> +
> +	r = wait_event_interruptible(pdd->process->wait_irq_drain,
> +				     !READ_ONCE(pdd->process->irq_drain_is_open));
> +	if (r)
> +		pdd->process->irq_drain_is_open = false;
> +
> +	return r;
> +}
> +
> +void kfd_process_close_interrupt_drain(unsigned int pasid)
> +{
> +	struct kfd_process *p;
> +
> +	p = kfd_lookup_process_by_pasid(pasid);
> +
> +	if (!p)
> +		return;
> +
> +	WRITE_ONCE(p->irq_drain_is_open, false);
> +	wake_up_all(&p->wait_irq_drain);
> +	kfd_unref_process(p);
> +}
> +
>   struct send_exception_work_handler_workarea {
>   	struct work_struct work;
>   	struct kfd_process *p;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
> index 43d432b5c5bc..70852a200d8f 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
> @@ -337,6 +337,10 @@ int pqm_create_queue(struct process_queue_manager *pqm,
>   		kq->queue->properties.queue_id = *qid;
>   		pqn->kq = kq;
>   		pqn->q = NULL;
> +		retval = kfd_process_drain_interrupts(pdd);
> +		if (retval)
> +			break;
> +
>   		retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
>   							kq, &pdd->qpd);
>   		break;


More information about the amd-gfx mailing list