[PATCH v2 7/9] drm/xe: Add GT TLB invalidation jobs
Summers, Stuart
stuart.summers at intel.com
Thu Jul 17 15:58:01 UTC 2025
On Tue, 2025-07-15 at 18:08 -0700, Matthew Brost wrote:
> > On Tue, Jul 15, 2025 at 05:09:01PM -0600, Summers, Stuart wrote:
> > > > On Wed, 2025-07-02 at 16:42 -0700, Matthew Brost wrote:
> > > > > > Add GT TLB invalidation jobs which issue GT TLB
> > > > > > invalidations. > > > Built
> > > > > > on
> > > > > > top of Xe generic dependency scheduler.
> > > > > >
> > > > > > v2:
> > > > > > - Fix checkpatch
> > > > > >
> > > > > > Suggested-by: Thomas Hellström
> > > > > > <thomas.hellstrom at linux.intel.com>
> > > > > > Signed-off-by: Matthew Brost <matthew.brost at intel.com>
> > > > > > ---
> > > > > > drivers/gpu/drm/xe/Makefile | 1 +
> > > > > > drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c | 271
> > > > > > +++++++++++++++++++++++
> > > > > > drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h | 34 +++
> > > > > > 3 files changed, 306 insertions(+)
> > > > > > create mode 100644
> > > > > > drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c
> > > > > > create mode 100644
> > > > > > drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h
> > > > > >
> > > > > > diff --git a/drivers/gpu/drm/xe/Makefile
> > > > > > b/drivers/gpu/drm/xe/Makefile
> > > > > > index 0edcfc770c0d..5aad44a3b5fd 100644
> > > > > > --- a/drivers/gpu/drm/xe/Makefile
> > > > > > +++ b/drivers/gpu/drm/xe/Makefile
> > > > > > @@ -55,6 +55,7 @@ xe-y += xe_bb.o \
> > > > > > xe_gt_sysfs.o \
> > > > > > xe_gt_throttle.o \
> > > > > > xe_gt_tlb_invalidation.o \
> > > > > > + xe_gt_tlb_inval_job.o \
> > > > > > xe_gt_topology.o \
> > > > > > xe_guc.o \
> > > > > > xe_guc_ads.o \
> > > > > > diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c
> > > > > > b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c
> > > > > > new file mode 100644
> > > > > > index 000000000000..428d20f16ec2
> > > > > > --- /dev/null
> > > > > > +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c
> > > > > > @@ -0,0 +1,271 @@
> > > > > > +// SPDX-License-Identifier: MIT
> > > > > > +/*
> > > > > > + * Copyright © 2025 Intel Corporation
> > > > > > + */
> > > > > > +
> > > > > > +#include "xe_dep_job_types.h"
> > > > > > +#include "xe_dep_scheduler.h"
> > > > > > +#include "xe_exec_queue.h"
> > > > > > +#include "xe_gt.h"
> > > > > > +#include "xe_gt_tlb_invalidation.h"
> > > > > > +#include "xe_gt_tlb_inval_job.h"
> > > > > > +#include "xe_migrate.h"
> > > > > > +#include "xe_pm.h"
> > > > > > +
> > > > > > +/** struct xe_gt_tlb_inval_job - GT TLB invalidation job
> > > > > > */
> > > > > > +struct xe_gt_tlb_inval_job {
> > > > > > + /** @dep: base generic dependency Xe job */
> > > > > > + struct xe_dep_job dep;
> > > > > > + /** @gt: GT to invalidate */
> > > > > > + struct xe_gt *gt;
> > > > > > + /** @q: exec queue issuing the invalidate */
> > > > > > + struct xe_exec_queue *q;
> > > > > > + /** @refcount: ref count of this job */
> > > > > > + struct kref refcount;
> > > > > > + /**
> > > > > > + * @fence: dma fence to indicate completion. 1 way
> > > > > > relationship - job
> > > > > > + * can safely reference fence, fence cannot safely
> > > > > > > > > reference
> > > > > > job.
> > > > > > + */
> > > > > > + struct dma_fence *fence;
> > > > > > + /** @start: Start address to invalidate */
> > > > > > + u64 start;
> > > > > > + /** @end: End address to invalidate */
> > > > > > + u64 end;
> > > > > > + /** @asid: Address space ID to invalidate */
> > > > > > + u32 asid;
> > > > > > + /** @fence_armed: Fence has been armed */
> > > > > > + bool fence_armed;
> > > > > > +};
> > > > > > +
> > > > > > +static struct dma_fence *xe_gt_tlb_inval_job_run(struct >
> > > > > > > > xe_dep_job
> > > > > > *dep_job)
> > > > > > +{
> > > > > > + struct xe_gt_tlb_inval_job *job =
> > > > > > + container_of(dep_job, typeof(*job), dep);
> > > > > > + struct xe_gt_tlb_invalidation_fence *ifence =
> > > > > > + container_of(job->fence, typeof(*ifence),
> > > > > > base);
> > > > > > +
> > > > > > + xe_gt_tlb_invalidation_range(job->gt, ifence, job-
> > > > > > >start,
> > > > > > + job->end, job->asid);
> > > > > > +
> > > > > > + return job->fence;
> > > > > > +}
> > > > > > +
> > > > > > +static void xe_gt_tlb_inval_job_free(struct xe_dep_job
> > > > > > *dep_job)
> > > > > > +{
> > > > > > + struct xe_gt_tlb_inval_job *job =
> > > > > > + container_of(dep_job, typeof(*job), dep);
> > > > > > +
> > > > > > + /* Pairs with get in xe_gt_tlb_inval_job_push */
> > > > > > + xe_gt_tlb_inval_job_put(job);
> > > > > > +}
> > > > > > +
> > > > > > +static const struct xe_dep_job_ops dep_job_ops = {
> > > > > > + .run_job = xe_gt_tlb_inval_job_run,
> > > > > > + .free_job = xe_gt_tlb_inval_job_free,
> > > > > > +};
> > > > > > +
> > > > > > +static int xe_gt_tlb_inval_context(struct xe_gt *gt)
> > > > > > +{
> > > > > > + return xe_gt_is_media_type(gt) ?
> > > > > > XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT :
> > > > > > + XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT;
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * xe_gt_tlb_inval_job_create() - GT TLB invalidation job
> > > > > > create
> > > > > > + * @gt: GT to invalidate
> > > > > > + * @q: exec queue issuing the invalidate
> > > > > > + * @start: Start address to invalidate
> > > > > > + * @end: End address to invalidate
> > > > > > + * @asid: Address space ID to invalidate
> > > > > > + *
> > > > > > + * Create a GT TLB invalidation job and initialize
> > > > > > internal > > > fields.
> > > > > > The caller is
> > > > > > + * responsible for releasing the creation reference.
> > > > > > + *
> > > > > > + * Return: GT TLB invalidation job object or ERR_PTR
> > > > > > + */
> > > > > > +struct xe_gt_tlb_inval_job
> > > > > > *xe_gt_tlb_inval_job_create(struct
> > > > > > xe_exec_queue *q,
> > > > > > +
> > > > > > struct > > > xe_gt
> > > > > > *gt,
> > > > > > + u64
> > > > > > start, > > > u64
> > > > > > end,
> > > > > > + u32
> > > > > > asid)
> > > > > > +{
> > > > > > + struct xe_gt_tlb_inval_job *job;
> > > > > > + struct xe_dep_scheduler *dep_scheduler =
> > > > > > + q-
> > > > > > > > tlb_inval[xe_gt_tlb_inval_context(gt)].dep_scheduler;
> > > > > > + struct drm_sched_entity *entity =
> > > > > > + xe_dep_scheduler_entity(dep_scheduler);
> > > > > > + struct xe_gt_tlb_invalidation_fence *ifence;
> > > > > > + int err;
> > > > > > +
> > > > > > + job = kmalloc(sizeof(*job), GFP_KERNEL);
> > > > > > + if (!job)
> > > > > > + return ERR_PTR(-ENOMEM);
> > > > > > +
> > > > > > + job->q = q;
> > > > > > + job->gt = gt;
> > > > > > + job->start = start;
> > > > > > + job->end = end;
> > > > > > + job->asid = asid;
> > > > > > + job->fence_armed = false;
> > > > > > + job->dep.ops = &dep_job_ops;
> > > > > > + kref_init(&job->refcount);
> > > > > > + xe_exec_queue_get(q);
> > > > > > +
> > > > > > + ifence = kmalloc(sizeof(*ifence), GFP_KERNEL);
> > > > > > + if (!ifence) {
> > > > > > + err = -ENOMEM;
> > > > > > + goto err_job;
> > > > > > + }
> > > > > > + job->fence = &ifence->base;
> > > > > > +
> > > > > > + err = drm_sched_job_init(&job->dep.drm, entity, 1,
> > > > > > NULL,
> > > > > > + q->xef ? q->xef->drm-
> > > > > > >client_id > > > :
> > > > > > 0);
> > > > > > + if (err)
> > > > > > + goto err_fence;
> > > > > > +
> > > > > > + xe_pm_runtime_get_noresume(gt_to_xe(job->gt));
> > > > > > + return job;
> > > > > > +
> > > > > > +err_fence:
> > > > > > + kfree(ifence);
> > > > > > +err_job:
> > > > > > + xe_exec_queue_put(q);
> > > > > > + kfree(job);
> > > > > > +
> > > > > > + return ERR_PTR(err);
> > > > > > +}
> > > > > > +
> > > > > > +static void xe_gt_tlb_inval_job_destroy(struct kref *ref)
> > > > > > +{
> > > > > > + struct xe_gt_tlb_inval_job *job = container_of(ref,
> > > > > > typeof(*job),
> > > > > > +
> > > > > > refcount);
> > > > > > + struct xe_gt_tlb_invalidation_fence *ifence =
> > > > > > + container_of(job->fence, typeof(*ifence),
> > > > > > base);
> > > > > > + struct xe_device *xe = gt_to_xe(job->gt);
> > > > > > + struct xe_exec_queue *q = job->q;
> > > > > > +
> > > > > > + if (!job->fence_armed)
> > > > > > + kfree(ifence);
> > > > > > + else
> > > > > > + /* Ref from
> > > > > > xe_gt_tlb_invalidation_fence_init */
> > > > > > + dma_fence_put(job->fence);
> > > > > > +
> > > > > > + drm_sched_job_cleanup(&job->dep.drm);
> > > > > > + kfree(job);
> > > > > > + xe_exec_queue_put(q); /* Pairs with get from
> > > > > > xe_gt_tlb_inval_job_create */
> > > > > > + xe_pm_runtime_put(xe); /* Pairs with get from
> > > > > > xe_gt_tlb_inval_job_create */
> > > >
> > > > This patch also looks great to me. My only concern here is it
> > > > feels
> > > > weird reordering these puts and kfrees from how we allocated
> > > > them > > in
> > > > xe_gt_tlb_inval_job_create. It does look functional though
> > > > because > > the
> > > > kfree of the job and exec queue/runtime put are really
> > > > independent, > > but
> > > > if for some reason that changed in the future (however
> > > > unlikely) > > and we
> > > > wanted to link those, it seems safer to do the cleanup in the >
> > > > > reverse
> > > > of the order you did the allocation. Let me know what you think
> > > > > > here.
> >
> > They are independent. The ordering is just mirroring what
> > xe_sched_job_destroy does but in bith cases could be reordered if
> > desired. I'd rather leave it as is for now, so the functions that >
> > create
> > / destroy jobs look more or less the same in Xe.
Yeah makes sense to me. Like I said I don't think this is a functional
problem or we'd have a lot of other issues. But eventually it would be
nice to clean this up.
Reviewed-by: Stuart Summers <stuart.summers at intel.com>
> >
> > Matt
> >
> > > >
> > > > Thanks,
> > > > Stuart
> > > >
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * xe_gt_tlb_inval_alloc_dep() - GT TLB invalidation job
> > > > > > alloc
> > > > > > dependency
> > > > > > + * @job: GT TLB invalidation job to alloc dependency for
> > > > > > + *
> > > > > > + * Allocate storage for a dependency in the GT TLB
> > > > > > invalidation
> > > > > > fence. This
> > > > > > + * function should be called at most once per job and must
> > > > > > be > > > paired
> > > > > > with
> > > > > > + * xe_gt_tlb_inval_job_push being called with a real > > >
> > > > > > (non-signaled)
> > > > > > fence.
> > > > > > + *
> > > > > > + * Return: 0 on success, -errno on failure
> > > > > > + */
> > > > > > +int xe_gt_tlb_inval_job_alloc_dep(struct
> > > > > > xe_gt_tlb_inval_job > > > *job)
> > > > > > +{
> > > > > > + xe_assert(gt_to_xe(job->gt), !xa_load(&job-
> > > > > > > > dep.drm.dependencies, 0));
> > > > > > +
> > > > > > + return drm_sched_job_add_dependency(&job->dep.drm,
> > > > > > + > > >
> > > > > > dma_fence_get_stub());
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * xe_gt_tlb_inval_job_push() - GT TLB invalidation job
> > > > > > push
> > > > > > + * @job: GT TLB invalidation job to push
> > > > > > + * @m: The migration object being used
> > > > > > + * @fence: Dependency for GT TLB invalidation job
> > > > > > + *
> > > > > > + * Pushes a GT TLB invalidation job for execution, using
> > > > > > @fence > > > as a
> > > > > > dependency.
> > > > > > + * Storage for @fence must be preallocated with
> > > > > > xe_gt_tlb_inval_job_alloc_dep
> > > > > > + * prior to this call if @fence is not signaled. Takes a >
> > > > > > > > reference
> > > > > > to the job’s
> > > > > > + * finished fence, which the caller is responsible for > >
> > > > > > > releasing,
> > > > > > and retutn it
> > > > > > + * to the caller. This function is safe to be called in
> > > > > > the path > > > of
> > > > > > reclaim.
> > > > > > + *
> > > > > > + * Return: Job's finished fence
> > > > > > + */
> > > > > > +struct dma_fence *xe_gt_tlb_inval_job_push(struct
> > > > > > xe_gt_tlb_inval_job *job,
> > > > > > + struct
> > > > > > xe_migrate *m,
> > > > > > + struct dma_fence
> > > > > > > > > *fence)
> > > > > > +{
> > > > > > + struct xe_gt_tlb_invalidation_fence *ifence =
> > > > > > + container_of(job->fence, typeof(*ifence),
> > > > > > base);
> > > > > > +
> > > > > > + if (!dma_fence_is_signaled(fence)) {
> > > > > > + void *ptr;
> > > > > > +
> > > > > > + /*
> > > > > > + * Can be in path of reclaim, hence the > >
> > > > > > > preallocation
> > > > > > of fence
> > > > > > + * storage in
> > > > > > xe_gt_tlb_inval_job_alloc_dep. > > > Verify
> > > > > > caller did
> > > > > > + * this correctly.
> > > > > > + */
> > > > > > + xe_assert(gt_to_xe(job->gt),
> > > > > > + xa_load(&job-
> > > > > > >dep.drm.dependencies, 0) > > > ==
> > > > > > + dma_fence_get_stub());
> > > > > > +
> > > > > > + dma_fence_get(fence); /* ref released
> > > > > > once
> > > > > > dependency processed by scheduler */
> > > > > > + ptr = xa_store(&job->dep.drm.dependencies,
> > > > > > 0, > > > fence,
> > > > > > + GFP_ATOMIC);
> > > > > > + xe_assert(gt_to_xe(job->gt),
> > > > > > !xa_is_err(ptr));
> > > > > > + }
> > > > > > +
> > > > > > + xe_gt_tlb_inval_job_get(job); /* Pairs with put
> > > > > > in > > > free_job
> > > > > > */
> > > > > > + job->fence_armed = true;
> > > > > > +
> > > > > > + /*
> > > > > > + * We need the migration lock to protect the seqnos
> > > > > > (job > > > and
> > > > > > + * invalidation fence) and the spsc queue, only
> > > > > > taken on
> > > > > > migration
> > > > > > + * queue, user queues protected dma-resv VM lock.
> > > > > > + */
> > > > > > + xe_migrate_job_lock(m, job->q);
> > > > > > +
> > > > > > + /* Creation ref pairs with put in > > >
> > > > > > xe_gt_tlb_inval_job_destroy
> > > > > > */
> > > > > > + xe_gt_tlb_invalidation_fence_init(job->gt, ifence,
> > > > > > > > > false);
> > > > > > + dma_fence_get(job->fence); /* Pairs with put
> > > > > > in DRM
> > > > > > scheduler */
> > > > > > +
> > > > > > + drm_sched_job_arm(&job->dep.drm);
> > > > > > + /*
> > > > > > + * caller ref, get must be done before job push as
> > > > > > it > > > could
> > > > > > immediately
> > > > > > + * signal and free.
> > > > > > + */
> > > > > > + dma_fence_get(&job->dep.drm.s_fence->finished);
> > > > > > + drm_sched_entity_push_job(&job->dep.drm);
> > > > > > +
> > > > > > + xe_migrate_job_unlock(m, job->q);
> > > > > > +
> > > > > > + /*
> > > > > > + * Not using job->fence, as it has its own dma-
> > > > > > fence > > > context,
> > > > > > which does
> > > > > > + * not allow GT TLB invalidation fences on the same
> > > > > > > > > queue, GT
> > > > > > tuple to
> > > > > > + * be squashed in dma-resv/DRM scheduler. Instead,
> > > > > > we use > > > the
> > > > > > DRM scheduler
> > > > > > + * context and job's finished fence, which enables
> > > > > > > > > squashing.
> > > > > > + */
> > > > > > + return &job->dep.drm.s_fence->finished;
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * xe_gt_tlb_inval_job_get() - Get a reference to GT TLB
> > > > > > invalidation job
> > > > > > + * @job: GT TLB invalidation job object
> > > > > > + *
> > > > > > + * Increment the GT TLB invalidation job's reference count
> > > > > > + */
> > > > > > +void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job
> > > > > > *job)
> > > > > > +{
> > > > > > + kref_get(&job->refcount);
> > > > > > +}
> > > > > > +
> > > > > > +/**
> > > > > > + * xe_gt_tlb_inval_job_put() - Put a reference to GT TLB
> > > > > > invalidation job
> > > > > > + * @job: GT TLB invalidation job object
> > > > > > + *
> > > > > > + * Decrement the GT TLB invalidation job's reference
> > > > > > count, call
> > > > > > + * xe_gt_tlb_inval_job_destroy when reference count == 0.
> > > > > > Skips
> > > > > > decrement if
> > > > > > + * input @job is NULL or IS_ERR.
> > > > > > + */
> > > > > > +void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job
> > > > > > *job)
> > > > > > +{
> > > > > > + if (job && !IS_ERR(job))
> > > > > > + kref_put(&job->refcount,
> > > > > > xe_gt_tlb_inval_job_destroy);
> > > > > > +}
> > > > > > diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h
> > > > > > b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h
> > > > > > new file mode 100644
> > > > > > index 000000000000..883896194a34
> > > > > > --- /dev/null
> > > > > > +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h
> > > > > > @@ -0,0 +1,34 @@
> > > > > > +/* SPDX-License-Identifier: MIT */
> > > > > > +/*
> > > > > > + * Copyright © 2025 Intel Corporation
> > > > > > + */
> > > > > > +
> > > > > > +#ifndef _XE_GT_TLB_INVAL_JOB_H_
> > > > > > +#define _XE_GT_TLB_INVAL_JOB_H_
> > > > > > +
> > > > > > +#include <linux/types.h>
> > > > > > +
> > > > > > +struct dma_fence;
> > > > > > +struct drm_sched_job;
> > > > > > +struct kref;
> > > > > > +struct xe_exec_queue;
> > > > > > +struct xe_gt;
> > > > > > +struct xe_gt_tlb_inval_job;
> > > > > > +struct xe_migrate;
> > > > > > +
> > > > > > +struct xe_gt_tlb_inval_job
> > > > > > *xe_gt_tlb_inval_job_create(struct
> > > > > > xe_exec_queue *q,
> > > > > > +
> > > > > > struct > > > xe_gt
> > > > > > *gt,
> > > > > > + u64
> > > > > > start, > > > u64
> > > > > > end,
> > > > > > + u32
> > > > > > asid);
> > > > > > +
> > > > > > +int xe_gt_tlb_inval_job_alloc_dep(struct
> > > > > > xe_gt_tlb_inval_job > > > *job);
> > > > > > +
> > > > > > +struct dma_fence *xe_gt_tlb_inval_job_push(struct
> > > > > > xe_gt_tlb_inval_job *job,
> > > > > > + struct
> > > > > > xe_migrate *m,
> > > > > > + struct dma_fence
> > > > > > > > > *fence);
> > > > > > +
> > > > > > +void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job
> > > > > > *job);
> > > > > > +
> > > > > > +void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job
> > > > > > *job);
> > > > > > +
> > > > > > +#endif
> > > >
More information about the Intel-xe
mailing list