[PATCH v2 2/9] drm/xe: Add generic dependecy jobs / scheduler

[Francois Dugast]

On Wed, Jul 02, 2025 at 04:42:15PM -0700, Matthew Brost wrote:
> Add generic dependecy jobs / scheduler which serves as wrapper for DRM
> scheduler. Useful when we want delay a generic operation until a
> dma-fence signals.
> 
> Existing use cases could be destroying of resources based fences /
> dma-resv, the preempt rebind worker, and pipelined GT TLB invalidations.
> 
> Written in such a way it could be moved to DRM subsystem if needed.
> 
> Signed-off-by: Matthew Brost <matthew.brost at intel.com>
> ---
>  drivers/gpu/drm/xe/Makefile           |   1 +
>  drivers/gpu/drm/xe/xe_dep_job_types.h |  29 ++++++
>  drivers/gpu/drm/xe/xe_dep_scheduler.c | 145 ++++++++++++++++++++++++++
>  drivers/gpu/drm/xe/xe_dep_scheduler.h |  21 ++++
>  4 files changed, 196 insertions(+)
>  create mode 100644 drivers/gpu/drm/xe/xe_dep_job_types.h
>  create mode 100644 drivers/gpu/drm/xe/xe_dep_scheduler.c
>  create mode 100644 drivers/gpu/drm/xe/xe_dep_scheduler.h
> 
> diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
> index 1d97e5b63f4e..0edcfc770c0d 100644
> --- a/drivers/gpu/drm/xe/Makefile
> +++ b/drivers/gpu/drm/xe/Makefile
> @@ -28,6 +28,7 @@ $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \
>  xe-y += xe_bb.o \
>  	xe_bo.o \
>  	xe_bo_evict.o \
> +	xe_dep_scheduler.o \
>  	xe_devcoredump.o \
>  	xe_device.o \
>  	xe_device_sysfs.o \
> diff --git a/drivers/gpu/drm/xe/xe_dep_job_types.h b/drivers/gpu/drm/xe/xe_dep_job_types.h
> new file mode 100644
> index 000000000000..c6a484f24c8c
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_dep_job_types.h
> @@ -0,0 +1,29 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright © 2025 Intel Corporation
> + */
> +
> +#ifndef _XE_DEP_JOB_TYPES_H_
> +#define _XE_DEP_JOB_TYPES_H_
> +
> +#include <drm/gpu_scheduler.h>
> +
> +struct xe_dep_job;
> +
> +/** struct xe_dep_job_ops - Generic Xe dependency job operations */
> +struct xe_dep_job_ops {
> +	/** @run_job: Run generic Xe dependency job */
> +	struct dma_fence *(*run_job)(struct xe_dep_job *job);
> +	/** @free_job: Free generic Xe dependency job */
> +	void (*free_job)(struct xe_dep_job *job);
> +};
> +
> +/** struct xe_dep_job - Generic dependency Xe job */
> +struct xe_dep_job {
> +	/** @drm: base DRM scheduler job */
> +	struct drm_sched_job drm;
> +	/** @ops: dependency job operations */
> +	const struct xe_dep_job_ops *ops;
> +};
> +
> +#endif
> diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c
> new file mode 100644
> index 000000000000..fbd55577d787
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c
> @@ -0,0 +1,145 @@
> +// SPDX-License-Identifier: MIT
> +/*
> + * Copyright © 2025 Intel Corporation
> + */
> +
> +#include <linux/slab.h>
> +
> +#include <drm/gpu_scheduler.h>
> +
> +#include "xe_dep_job_types.h"
> +#include "xe_dep_scheduler.h"
> +#include "xe_device_types.h"
> +
> +/**
> + * DOC: Xe Dependency Scheduler
> + *
> + * The Xe dependency scheduler is a simple wrapper built around the DRM
> + * scheduler to execute jobs once their dependencies are resolved (i.e., all
> + * input fences specified as dependencies are signaled). The jobs that are
> + * executed contain virtual functions to run (execute) and free the job,
> + * allowing a single dependency scheduler to handle jobs performing different
> + * operations.
> + *
> + * Example use cases include deferred resource freeing, TLB invalidations after
> + * bind jobs, etc.
> + */

This is already well documentated but as this code might eventually get
promoted to DRM, maybe we could add generic pseudo-code showing how it is
intended to be used, for example:

 * .. code-block:: c
 *
 *      struct my_job {
 *        struct xe_dep_job dep;
 *        struct dma_fence *fence;
 *        ...
 *      }
 *      
 *      
 *      static struct dma_fence *my_job_run(struct xe_dep_job *dep_job)
 *      {
 *        struct my_job *job = container_of(dep_job, typeof(*job), dep);
 *        
 *        // start the job and get a fence
 *        ...
 *        
 *        return job->fence;
 *      }
 *      
 *      static void my_job_free(struct xe_dep_job *dep_job)
 *      {
 *        ...
 *      }
 *      
 *      static const struct xe_dep_job_ops my_job_ops = {
 *      	.run_job = my_job_run,
 *      	.free_job = my_job_free,
 *      };
 *      
 *      void init()
 *      {
 *        struct xe_dep_scheduler *dep_scheduler = xe_dep_scheduler_create(xe, wq, name, 16);
 *        ...
 *      }
 *      
 *      struct my_job *create()
 *      {
 *        struct drm_sched_entity *entity = xe_dep_scheduler_entity(dep_scheduler);
 *        struct my_job *job;
 *        ...
 *      
 *        job->dep.ops = &dep_job_ops;
 *        drm_sched_job_init(&job->dep.drm, entity, ...);
 *        
 *        return job;
 *      }
 *      
 *      void cleanup()
 *      {
 *        xe_dep_scheduler_fini(dep_scheduler);
 *        ...
 *      }

> +
> +/** struct xe_dep_scheduler - Generic Xe dependency scheduler */
> +struct xe_dep_scheduler {
> +	/** @sched: DRM GPU scheduler */
> +	struct drm_gpu_scheduler sched;
> +	/** @entity: DRM scheduler entity  */
> +	struct drm_sched_entity entity;
> +	/** @rcu: For safe freeing of exported dma fences */
> +	struct rcu_head rcu;

Is it used in the series?

Francois

> +};
> +
> +static struct dma_fence *xe_dep_scheduler_run_job(struct drm_sched_job *drm_job)
> +{
> +	struct xe_dep_job *dep_job =
> +		container_of(drm_job, typeof(*dep_job), drm);
> +
> +	return dep_job->ops->run_job(dep_job);
> +}
> +
> +static void xe_dep_scheduler_free_job(struct drm_sched_job *drm_job)
> +{
> +	struct xe_dep_job *dep_job =
> +		container_of(drm_job, typeof(*dep_job), drm);
> +
> +	dep_job->ops->free_job(dep_job);
> +}
> +
> +static const struct drm_sched_backend_ops sched_ops = {
> +	.run_job = xe_dep_scheduler_run_job,
> +	.free_job = xe_dep_scheduler_free_job,
> +};
> +
> +/**
> + * xe_dep_scheduler_create() - Generic Xe dependency scheduler create
> + * @xe: Xe device
> + * @submit_wq: Submit workqueue struct (can be NULL)
> + * @name: Name of dependency scheduler
> + * @job_limit: Max dependency jobs that can be scheduled
> + *
> + * Create a generic Xe dependency scheduler and initialize internal DRM
> + * scheduler objects.
> + *
> + * Return: Generic Xe dependency scheduler object or ERR_PTR
> + */
> +struct xe_dep_scheduler *
> +xe_dep_scheduler_create(struct xe_device *xe,
> +			struct workqueue_struct *submit_wq,
> +			const char *name, u32 job_limit)
> +{
> +	struct xe_dep_scheduler *dep_scheduler;
> +	struct drm_gpu_scheduler *sched;
> +	const struct drm_sched_init_args args = {
> +		.ops = &sched_ops,
> +		.submit_wq = submit_wq,
> +		.num_rqs = 1,
> +		.credit_limit = job_limit,
> +		.timeout = MAX_SCHEDULE_TIMEOUT,
> +		.name = name,
> +		.dev = xe->drm.dev,
> +	};
> +	int err;
> +
> +	dep_scheduler = kzalloc(sizeof(*dep_scheduler), GFP_KERNEL);
> +	if (!dep_scheduler)
> +		return ERR_PTR(-ENOMEM);
> +
> +	err = drm_sched_init(&dep_scheduler->sched, &args);
> +	if (err)
> +		goto err_free;
> +
> +	sched = &dep_scheduler->sched;
> +	err = drm_sched_entity_init(&dep_scheduler->entity, 0,
> +				    (struct drm_gpu_scheduler **)&sched, 1,
> +				    NULL);
> +	if (err)
> +		goto err_sched;
> +
> +	init_rcu_head(&dep_scheduler->rcu);
> +
> +	return dep_scheduler;
> +
> +err_sched:
> +	drm_sched_fini(&dep_scheduler->sched);
> +err_free:
> +	kfree(dep_scheduler);
> +
> +	return ERR_PTR(err);
> +}
> +
> +/**
> + * xe_dep_scheduler_fini() - Generic Xe dependency scheduler finalize
> + * @dep_scheduler: Generic Xe dependency scheduler object
> + *
> + * Finalize internal DRM scheduler objects and free generic Xe dependency
> + * scheduler object
> + */
> +void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler)
> +{
> +	drm_sched_entity_fini(&dep_scheduler->entity);
> +	drm_sched_fini(&dep_scheduler->sched);
> +	/*
> +	 * RCU free due sched being exported via DRM scheduler fences
> +	 * (timeline name).
> +	 */
> +	kfree_rcu(dep_scheduler, rcu);
> +}
> +
> +/**
> + * xe_dep_scheduler_entity() - Retrieve a generic Xe dependency scheduler
> + *                             DRM scheduler entity
> + * @dep_scheduler: Generic Xe dependency scheduler object
> + *
> + * Return: The generic Xe dependency scheduler's DRM scheduler entity
> + */
> +struct drm_sched_entity *
> +xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler)
> +{
> +	return &dep_scheduler->entity;
> +}
> diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.h b/drivers/gpu/drm/xe/xe_dep_scheduler.h
> new file mode 100644
> index 000000000000..853961eec64b
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.h
> @@ -0,0 +1,21 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright © 2025 Intel Corporation
> + */
> +
> +#include <linux/types.h>
> +
> +struct drm_sched_entity;
> +struct workqueue_struct;
> +struct xe_dep_scheduler;
> +struct xe_device;
> +
> +struct xe_dep_scheduler *
> +xe_dep_scheduler_create(struct xe_device *xe,
> +			struct workqueue_struct *submit_wq,
> +			const char *name, u32 job_limit);
> +
> +void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler);
> +
> +struct drm_sched_entity *
> +xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler);
> -- 
> 2.34.1
>