[PATCH] drm/amdgpu: add ih waiter on process until checkpoint

[Felix Kuehling]

On 2021-03-09 11:19 a.m., Jonathan Kim wrote:
> Add IH function to allow caller to wait until ring entries are processed
> until the checkpoint write pointer.
>
> This will be primarily used by HMM to drain pending page fault interrupts
> before memory unmap to prevent HMM from handling stale interrupts.
>
> v3: Scrap busy loop and change to wait_event.
>
> v2: Update title and description to clarify use.
> Add rptr/wptr wrap counter checks to guarantee ring entries are processed
> until the checkpoint.
>
> Suggested-by: Christian Koenig <christian.koenig at amd.com>
> Signed-off-by: Jonathan Kim <jonathan.kim at amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>


> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c | 49 ++++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h |  5 +++
>   2 files changed, 54 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
> index dc852af4f3b7..1024065f1f03 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
> @@ -99,6 +99,8 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
>   		ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4;
>   		ih->rptr_cpu = &adev->wb.wb[rptr_offs];
>   	}
> +
> +	init_waitqueue_head(&ih->wait_process);
>   	return 0;
>   }
>   
> @@ -160,6 +162,52 @@ void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv,
>   	}
>   }
>   
> +/* Waiter helper that checks current rptr matches or passes checkpoint wptr */
> +static bool amdgpu_ih_has_checkpoint_processed(struct amdgpu_device *adev,
> +					struct amdgpu_ih_ring *ih,
> +					uint32_t checkpoint_wptr,
> +					uint32_t *prev_rptr)
> +{
> +	uint32_t cur_rptr = ih->rptr | (*prev_rptr & ~ih->ptr_mask);
> +
> +	/* rptr has wrapped. */
> +	if (cur_rptr < *prev_rptr)
> +		cur_rptr += ih->ptr_mask + 1;
> +	*prev_rptr = cur_rptr;
> +
> +	return cur_rptr >= checkpoint_wptr;
> +}
> +
> +/**
> + * amdgpu_ih_wait_on_checkpoint_process - wait to process IVs up to checkpoint
> + *
> + * @adev: amdgpu_device pointer
> + * @ih: ih ring to process
> + *
> + * Used to ensure ring has processed IVs up to the checkpoint write pointer.
> + */
> +int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
> +					struct amdgpu_ih_ring *ih)
> +{
> +	uint32_t checkpoint_wptr, rptr;
> +
> +	if (!ih->enabled || adev->shutdown)
> +		return -ENODEV;
> +
> +	checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);
> +	/* Order wptr with rptr. */
> +	rmb();
> +	rptr = READ_ONCE(ih->rptr);
> +
> +	/* wptr has wrapped. */
> +	if (rptr > checkpoint_wptr)
> +		checkpoint_wptr += ih->ptr_mask + 1;
> +
> +	return wait_event_interruptible(ih->wait_process,
> +				amdgpu_ih_has_checkpoint_processed(adev, ih,
> +						checkpoint_wptr, &rptr));
> +}
> +
>   /**
>    * amdgpu_ih_process - interrupt handler
>    *
> @@ -195,6 +243,7 @@ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
>   	}
>   
>   	amdgpu_ih_set_rptr(adev, ih);
> +	wake_up_all(&ih->wait_process);
>   	atomic_set(&ih->lock, 0);
>   
>   	/* make sure wptr hasn't changed while processing */
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
> index 6ed4a85fc7c3..87ec6d20dbe0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
> @@ -66,6 +66,9 @@ struct amdgpu_ih_ring {
>   	unsigned		rptr;
>   	atomic_t		lock;
>   	struct amdgpu_ih_regs	ih_regs;
> +
> +	/* For waiting on IH processing at checkpoint. */
> +	wait_queue_head_t wait_process;
>   };
>   
>   /* provided by the ih block */
> @@ -87,6 +90,8 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
>   void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih);
>   void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv,
>   			  unsigned int num_dw);
> +int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
> +					struct amdgpu_ih_ring *ih);
>   int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih);
>   void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev,
>   				struct amdgpu_ih_ring *ih,