[PATCH 3/4] drm/amdgpu: suspend gfx userqueues

Shashank Sharma shashank.sharma at amd.com
Wed May 8 20:28:36 UTC 2024


This patch adds suspend support for gfx userqueues. It typically does
the following:
- adds an enable_signaling function for the eviction fence, so that it
  can trigger the userqueue suspend,
- adds a delayed function for suspending the userqueues, to suspend all
  the queues under this userq manager and signals the eviction fence,
- adds reference of userq manager in the eviction fence container so
  that it can be used in the suspend function.

Cc: Alex Deucher <alexander.deucher at amd.com>
Cc: Christian Koenig <christian.koenig at amd.com>
Signed-off-by: Shashank Sharma <shashank.sharma at amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h           |  1 +
 .../drm/amd/amdgpu/amdgpu_eviction_fence.c    |  6 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h    |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 77 +++++++++++++++++++
 .../gpu/drm/amd/include/amdgpu_userqueue.h    |  6 ++
 5 files changed, 91 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index a37193fc9ddc..1856fe11dd05 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -469,6 +469,7 @@ struct amdgpu_eviction_fence {
 	struct dma_fence base;
 	spinlock_t	 lock;
 	char		 timeline_name[TASK_COMM_LEN];
+	struct amdgpu_userq_mgr *uq_mgr;
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index 1a03f040ccc8..3f806e44f614 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -39,10 +39,16 @@ amdgpu_ev_fence_get_timeline_name(struct dma_fence *f)
 	return ef->timeline_name;
 }
 
+static bool amdgpu_ev_fence_enable_signaling(struct dma_fence *f)
+{
+	return !amdgpu_userqueue_enable_signaling(f);
+}
+
 static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
 	.use_64bit_seqno = true,
 	.get_driver_name = amdgpu_ev_fence_get_driver_name,
 	.get_timeline_name = amdgpu_ev_fence_get_timeline_name,
+	.enable_signaling = amdgpu_ev_fence_enable_signaling,
 };
 
 struct amdgpu_eviction_fence *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index fa03d9e4874c..8c13de7f2a19 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -111,6 +111,7 @@ struct amdgpu_bo {
 #endif
 	struct kgd_mem                  *kfd_bo;
 
+
 	/*
 	 * For GPUs with spatial partitioning, xcp partition number, -1 means
 	 * any partition. For other ASICs without spatial partition, always 0
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
index 7a89f378c97f..fdbd542e7f53 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
@@ -23,11 +23,16 @@
  */
 #include <linux/atomic.h>
 #include <drm/drm_syncobj.h>
+#include <drm/drm_exec.h>
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_userqueue.h"
 #include "amdgpu_userq_fence.h"
 
+#define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name)
+#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr)
+#define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base)
+
 static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa)
 {
 	struct amdgpu_userq_fence_driver *fence_drv;
@@ -226,6 +231,7 @@ int amdgpu_userqueue_update_bo_mapping(struct drm_file *filp, struct amdgpu_bo *
 	}
 
 	drm_syncobj_add_point(syncobj, chain, bo_va->last_pt_update, (uint64_t)point);
+
 	return 0;
 }
 
@@ -392,12 +398,83 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
 	return r;
 }
 
+static int
+amdgpu_userqueue_suspend_all(struct amdgpu_userq_mgr *uq_mgr)
+{
+	struct amdgpu_device *adev = uq_mgr->adev;
+	const struct amdgpu_userq_funcs *userq_funcs;
+	struct amdgpu_usermode_queue *queue;
+	int queue_id, ret;
+
+	userq_funcs = adev->userq_funcs[AMDGPU_HW_IP_GFX];
+
+	/* Suspend all the queues for this process */
+	idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) {
+		ret = userq_funcs->suspend(uq_mgr, queue);
+		if (ret)
+			DRM_ERROR("Failed to suspend queue\n");
+	}
+
+	return ret;
+}
+
+static void
+amdgpu_userqueue_suspend_worker(struct work_struct *work)
+{
+	int ret;
+	struct dma_fence *fence;
+	struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, suspend_work.work);
+	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
+
+	mutex_lock(&uq_mgr->userq_mutex);
+	ret = amdgpu_userqueue_suspend_all(uq_mgr);
+	if (ret) {
+		DRM_ERROR("Failed to evict userqueue\n");
+		goto unlock;
+	}
+
+	/* Signal current eviction fence */
+	fence = dma_fence_get(&fpriv->ev_fence->base);
+	ret = dma_fence_signal(fence);
+	dma_fence_put(fence);
+	if (ret) {
+		DRM_ERROR("Can't signal eviction fence to suspend");
+		return;
+	}
+
+unlock:
+	mutex_unlock(&uq_mgr->userq_mutex);
+}
+
+int amdgpu_userqueue_enable_signaling(struct dma_fence *f)
+{
+	struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f);
+	struct amdgpu_userq_mgr *uq_mgr = ev_fence->uq_mgr;
+
+	/* Schedule a suspend work in 1 ms */
+	schedule_delayed_work(&uq_mgr->suspend_work,
+			      msecs_to_jiffies(AMDGPU_USERQ_SUSPEND_TIME_MS));
+
+	return 0;
+}
+
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev)
 {
+	struct amdgpu_fpriv *fpriv;
+
 	mutex_init(&userq_mgr->userq_mutex);
 	idr_init_base(&userq_mgr->userq_idr, 1);
 	userq_mgr->adev = adev;
 
+	fpriv = uq_mgr_to_fpriv(userq_mgr);
+	if (!fpriv->ev_fence) {
+		DRM_ERROR("Eviction fence not initialized yet\n");
+		return -EINVAL;
+	}
+
+	/* This reference is required for suspend work */
+	fpriv->ev_fence->uq_mgr = userq_mgr;
+	INIT_DELAYED_WORK(&userq_mgr->suspend_work, amdgpu_userqueue_suspend_worker);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
index afaf93faa824..647e63bf03ab 100644
--- a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
+++ b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
@@ -26,6 +26,8 @@
 #define AMDGPU_USERQUEUE_H_
 
 #define AMDGPU_MAX_USERQ_COUNT 512
+#define AMDGPU_USERQ_SUSPEND_TIME_MS 1
+#define AMDGPU_USERQ_RESUME_TIME_MS 100
 
 struct amdgpu_mqd_prop;
 
@@ -69,6 +71,8 @@ struct amdgpu_userq_mgr {
 	struct idr			userq_idr;
 	struct mutex			userq_mutex;
 	struct amdgpu_device		*adev;
+
+	struct delayed_work		suspend_work;
 };
 
 int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
@@ -86,4 +90,6 @@ void amdgpu_userqueue_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
 
 int amdgpu_userqueue_update_bo_mapping(struct drm_file *filp, struct amdgpu_bo *bo,
 				       uint32_t syncobj_handle, uint16_t point);
+
+int amdgpu_userqueue_enable_signaling(struct dma_fence *f);
 #endif
-- 
2.43.2



More information about the amd-gfx mailing list