[PATCH 07/30] drm/amdgpu: track ring state associated with a fence
Alex Deucher
alexdeucher at gmail.com
Thu Jun 26 17:25:01 UTC 2025
On Thu, Jun 26, 2025 at 7:03 AM Christian König
<christian.koenig at amd.com> wrote:
>
>
>
> On 25.06.25 23:06, Alex Deucher wrote:
> > We need to know the wptr and sequence number associated
> > with a fence so that we can re-emit the unprocessed state
> > after a ring reset. Pre-allocate storage space for
> > the ring buffer contents and add helpers to save off
> > and re-emit the unprocessed state so that it can be
> > re-emitted after the queue is reset.
> >
> > Signed-off-by: Alex Deucher <alexander.deucher at amd.com>
> > ---
> > drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 99 +++++++++++++++++++++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 15 +++-
> > drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 4 +-
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 59 ++++++++++++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 17 ++++
> > 5 files changed, 191 insertions(+), 3 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
> > index 2d6b2b486baf7..db14bdc7053ca 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
> > @@ -120,11 +120,13 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
> > am_fence = kzalloc(sizeof(*am_fence), GFP_KERNEL);
> > if (!am_fence)
> > return -ENOMEM;
> > + am_fence->context = 0;
> > } else {
> > am_fence = af;
> > }
> > fence = &am_fence->base;
> > am_fence->ring = ring;
> > + am_fence->wptr = 0;
> >
> > seq = ++ring->fence_drv.sync_seq;
> > if (af) {
> > @@ -253,6 +255,7 @@ bool amdgpu_fence_process(struct amdgpu_ring *ring)
> >
> > do {
> > struct dma_fence *fence, **ptr;
> > + struct amdgpu_fence *am_fence;
> >
> > ++last_seq;
> > last_seq &= drv->num_fences_mask;
> > @@ -265,6 +268,13 @@ bool amdgpu_fence_process(struct amdgpu_ring *ring)
> > if (!fence)
> > continue;
> >
> > + /* Save the wptr in the fence driver so we know what the last processed
> > + * wptr was. This is required for re-emitting the ring state for
> > + * queues that are reset but are not guilty and thus have no guilty fence.
> > + */
> > + am_fence = container_of(fence, struct amdgpu_fence, base);
> > + if (am_fence->wptr)
> > + drv->last_wptr = am_fence->wptr;
>
> That looks fishy. We don't need the last wptr, but the first processed one.
I will rename it to signalled_wptr, but I think it is correct. This
is the wptr associated with the last fence that signalled. That is
the starting point for saving the ring contents.
Alex
>
> > dma_fence_signal(fence);
> > dma_fence_put(fence);
> > pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
> > @@ -727,6 +737,95 @@ void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
> > amdgpu_fence_process(ring);
> > }
> >
> > +
> > +/**
> > + * Kernel queue reset handling
> > + *
> > + * The driver can reset individual queues for most engines, but those queues
> > + * may contain work from multiple contexts. Resetting the queue will reset
> > + * lose all of that state. In order to minimize the collatoral damage, the
>
> Typo "collateral".
>
> > + * driver will save the ring contents which are not associated with the guilty
> > + * context prior to resetting the queue. After resetting the queue the queue
> > + * contents from the other contexts is re-emitted to the rings so that it can
> > + * be processed by the engine. To handle this, we save the queue's write
> > + * pointer (wptr) in the fences associated with each context. If we get a
> > + * queue timeout, we can then use the wptrs from the fences to determine
> > + * which data needs to be saved out of the queue's ring buffer.
> > + */
> > +
> > +/**
> > + * amdgpu_fence_driver_guilty_force_completion - force signal of specified sequence
> > + *
> > + * @fence: fence of the ring to signal
> > + *
> > + */
> > +void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *fence)
> > +{
> > + dma_fence_set_error(&fence->base, -ETIME);
> > + amdgpu_fence_write(fence->ring, fence->base.seqno);
> > + amdgpu_fence_process(fence->ring);
> > +}
> > +
> > +void amdgpu_fence_save_wptr(struct dma_fence *fence)
> > +{
> > + struct amdgpu_fence *am_fence = container_of(fence, struct amdgpu_fence, base);
> > +
> > + am_fence->wptr = am_fence->ring->wptr;
> > +}
> > +
> > +static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,
> > + unsigned int idx,
>
> idx is unused.
>
> > + u64 start_wptr, u32 end_wptr)
> > +{
> > + unsigned int first_idx = start_wptr & ring->buf_mask;
> > + unsigned int last_idx = end_wptr & ring->buf_mask;
> > + unsigned int i, count;
> > +
> > + /* Backup the contents of the ring buffer. */
> > + for (i = first_idx, count = 0; i != last_idx; ++i, i &= ring->buf_mask, ++count)
>
> count should start at ring->ring_backup_entries_to_copy.
>
> > + ring->ring_backup[count] = ring->ring[i];
> > + ring->ring_backup_entries_to_copy = count;
> > +}
> > +
> > +void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
> > + struct amdgpu_fence *guilty_fence)
> > +{
> > + struct amdgpu_fence *fence;
> > + struct dma_fence *unprocessed, **ptr;
>
> ptr need an __rcu annotation or otherwise the automated tools will start complaining.
>
> > + u64 wptr, i, seqno;
> > +
> > + if (guilty_fence) {
> > + seqno = guilty_fence->base.seqno;
> > + wptr = guilty_fence->wptr;
>
> I think we can just always go over the whole ring buffer here. E.g. dropping this if branch.
>
>
> > + } else {
> > + seqno = amdgpu_fence_read(ring);
> > + wptr = ring->fence_drv.last_wptr;
>
> Again that looks fishy. We shouldn't start at the last wptr, but the first.
>
> > + }
> > + ring->ring_backup_entries_to_copy = 0;
> > + for (i = seqno + 1; i <= ring->fence_drv.sync_seq; ++i) {
> > + ptr = &ring->fence_drv.fences[i & ring->fence_drv.num_fences_mask];
> > + rcu_read_lock();
> > + unprocessed = rcu_dereference(*ptr);
> > +
> > + if (unprocessed && !dma_fence_is_signaled(unprocessed)) {
> > + fence = container_of(unprocessed, struct amdgpu_fence, base);
>
> We should probably make ring->fence_drv.fences an array of amdgpu_fences.
>
> > +
> > + /* save everything if the ring is not guilty, otherwise
> > + * just save the content from other contexts.
> > + */
> > + if (fence->wptr &&
> > + (!guilty_fence || (fence->context != guilty_fence->context))) {
> > + amdgpu_ring_backup_unprocessed_command(ring,
> > + ring->ring_backup_entries_to_copy,
> > + wptr,
> > + fence->wptr);
> > + wptr = fence->wptr;
>
> Wptr also needs to be updated for already signaled fences and fences which are skipped because they belong to the guilty context.
>
> Christian.
>
> > + }
> > + }
> > + rcu_read_unlock();
> > + }
> > +}
> > +
> > /*
> > * Common fence implementation
> > */
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> > index 206b70acb29a0..d0f838fde2ae5 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> > @@ -139,7 +139,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
> > int vmid = AMDGPU_JOB_GET_VMID(job);
> > bool need_pipe_sync = false;
> > unsigned int cond_exec;
> > -
> > unsigned int i;
> > int r = 0;
> >
> > @@ -156,6 +155,11 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
> > gds_va = job->gds_va;
> > init_shadow = job->init_shadow;
> > af = &job->hw_fence;
> > + /* Save the context of the job for reset handling.
> > + * The driver needs this so it can skip the ring
> > + * contents for guilty contexts.
> > + */
> > + af->context = job->base.s_fence ? job->base.s_fence->finished.context : 0;
> > } else {
> > vm = NULL;
> > fence_ctx = 0;
> > @@ -309,6 +313,15 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
> >
> > amdgpu_ring_ib_end(ring);
> > amdgpu_ring_commit(ring);
> > +
> > + /* Save the wptr associated with this fence.
> > + * This must be last for resets to work properly
> > + * as we need to save the wptr associated with this
> > + * fence so we know what rings contents to backup
> > + * after we reset the queue.
> > + */
> > + amdgpu_fence_save_wptr(*f);
> > +
> > return 0;
> > }
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> > index f0b7080dccb8d..45febdc2f3493 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> > @@ -89,8 +89,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
> > {
> > struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
> > struct amdgpu_job *job = to_amdgpu_job(s_job);
> > - struct amdgpu_task_info *ti;
> > struct amdgpu_device *adev = ring->adev;
> > + struct amdgpu_task_info *ti;
> > int idx, r;
> >
> > if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
> > @@ -135,7 +135,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
> > } else if (amdgpu_gpu_recovery && ring->funcs->reset) {
> > dev_err(adev->dev, "Starting %s ring reset\n",
> > s_job->sched->name);
> > - r = amdgpu_ring_reset(ring, job->vmid, NULL);
> > + r = amdgpu_ring_reset(ring, job->vmid, &job->hw_fence);
> > if (!r) {
> > atomic_inc(&ring->adev->gpu_reset_counter);
> > dev_err(adev->dev, "Ring %s reset succeeded\n",
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> > index 426834806fbf2..0985eba010e17 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> > @@ -333,6 +333,12 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
> > /* Initialize cached_rptr to 0 */
> > ring->cached_rptr = 0;
> >
> > + if (!ring->ring_backup) {
> > + ring->ring_backup = kvzalloc(ring->ring_size, GFP_KERNEL);
> > + if (!ring->ring_backup)
> > + return -ENOMEM;
> > + }
> > +
> > /* Allocate ring buffer */
> > if (ring->ring_obj == NULL) {
> > r = amdgpu_bo_create_kernel(adev, ring->ring_size + ring->funcs->extra_dw, PAGE_SIZE,
> > @@ -342,6 +348,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
> > (void **)&ring->ring);
> > if (r) {
> > dev_err(adev->dev, "(%d) ring create failed\n", r);
> > + kvfree(ring->ring_backup);
> > return r;
> > }
> > amdgpu_ring_clear_ring(ring);
> > @@ -385,6 +392,8 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
> > amdgpu_bo_free_kernel(&ring->ring_obj,
> > &ring->gpu_addr,
> > (void **)&ring->ring);
> > + kvfree(ring->ring_backup);
> > + ring->ring_backup = NULL;
> >
> > dma_fence_put(ring->vmid_wait);
> > ring->vmid_wait = NULL;
> > @@ -753,3 +762,53 @@ bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring)
> >
> > return true;
> > }
> > +
> > +static int amdgpu_ring_reemit_unprocessed_commands(struct amdgpu_ring *ring)
> > +{
> > + unsigned int i;
> > + int r;
> > +
> > + /* re-emit the unprocessed ring contents */
> > + if (ring->ring_backup_entries_to_copy) {
> > + r = amdgpu_ring_alloc(ring, ring->ring_backup_entries_to_copy);
> > + if (r)
> > + return r;
> > + for (i = 0; i < ring->ring_backup_entries_to_copy; i++)
> > + amdgpu_ring_write(ring, ring->ring_backup[i]);
> > + amdgpu_ring_commit(ring);
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +void amdgpu_ring_reset_helper_begin(struct amdgpu_ring *ring,
> > + struct amdgpu_fence *guilty_fence)
> > +{
> > + /* Stop the scheduler to prevent anybody else from touching the ring buffer. */
> > + drm_sched_wqueue_stop(&ring->sched);
> > + /* back up the non-guilty commands */
> > + amdgpu_ring_backup_unprocessed_commands(ring, guilty_fence);
> > +}
> > +
> > +int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
> > + struct amdgpu_fence *guilty_fence)
> > +{
> > + int r;
> > +
> > + /* verify that the ring is functional */
> > + r = amdgpu_ring_test_ring(ring);
> > + if (r)
> > + return r;
> > +
> > + /* signal the fence of the bad job */
> > + if (guilty_fence)
> > + amdgpu_fence_driver_guilty_force_completion(guilty_fence);
> > + /* Re-emit the non-guilty commands */
> > + r = amdgpu_ring_reemit_unprocessed_commands(ring);
> > + if (r)
> > + /* if we fail to reemit, force complete all fences */
> > + amdgpu_fence_driver_force_completion(ring);
> > + /* Start the scheduler again */
> > + drm_sched_wqueue_start(&ring->sched);
> > + return 0;
> > +}
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > index 784ba2ec354c7..2b5546d15a6be 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> > @@ -118,6 +118,7 @@ struct amdgpu_fence_driver {
> > /* sync_seq is protected by ring emission lock */
> > uint32_t sync_seq;
> > atomic_t last_seq;
> > + u64 last_wptr;
> > bool initialized;
> > struct amdgpu_irq_src *irq_src;
> > unsigned irq_type;
> > @@ -141,6 +142,11 @@ struct amdgpu_fence {
> > /* RB, DMA, etc. */
> > struct amdgpu_ring *ring;
> > ktime_t start_timestamp;
> > +
> > + /* wptr for the fence for resets */
> > + u64 wptr;
> > + /* fence context for resets */
> > + u64 context;
> > };
> >
> > extern const struct drm_sched_backend_ops amdgpu_sched_ops;
> > @@ -148,6 +154,8 @@ extern const struct drm_sched_backend_ops amdgpu_sched_ops;
> > void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring);
> > void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error);
> > void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
> > +void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *fence);
> > +void amdgpu_fence_save_wptr(struct dma_fence *fence);
> >
> > int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring);
> > int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
> > @@ -284,6 +292,9 @@ struct amdgpu_ring {
> >
> > struct amdgpu_bo *ring_obj;
> > uint32_t *ring;
> > + /* backups for resets */
> > + uint32_t *ring_backup;
> > + unsigned int ring_backup_entries_to_copy;
> > unsigned rptr_offs;
> > u64 rptr_gpu_addr;
> > volatile u32 *rptr_cpu_addr;
> > @@ -550,4 +561,10 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev);
> > void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
> > int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
> > bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring);
> > +void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
> > + struct amdgpu_fence *guilty_fence);
> > +void amdgpu_ring_reset_helper_begin(struct amdgpu_ring *ring,
> > + struct amdgpu_fence *guilty_fence);
> > +int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
> > + struct amdgpu_fence *guilty_fence);
> > #endif
>
More information about the amd-gfx
mailing list