[PATCH 5/5] drm/amdgpu: Add TDR queue for ring

[Lijo Lazar]

Add a TDR queue for rings to handle job timeouts. Ring's scheduler will
use this queue to for running job timeout handlers. Timeout handler will
then use the appropriate reset domain to handle recovery.

Signed-off-by: Lijo Lazar <lijo.lazar at amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    | 16 ++++++++++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c   |  5 +++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |  1 +
 4 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 4aee867ec59f..78db74b7a49b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2307,7 +2307,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
 
 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
 				   ring->num_hw_submission, 0,
-				   timeout, adev->reset_domain->wq,
+				   timeout, ring->tdr_queue,
 				   ring->sched_score, ring->name,
 				   adev->dev);
 		if (r) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 78476bc75b4e..e081c0056a60 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -31,6 +31,16 @@
 #include "amdgpu_trace.h"
 #include "amdgpu_reset.h"
 
+static void amdgpu_job_recover(struct amdgpu_reset_context *reset_context)
+{
+	int r;
+
+	r = amdgpu_device_gpu_recover(reset_context->reset_req_dev,
+				      reset_context->job, reset_context);
+	if (r)
+		DRM_ERROR("GPU Recovery Failed: %d\n", r);
+}
+
 static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 {
 	struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
@@ -38,7 +48,6 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	struct amdgpu_task_info ti;
 	struct amdgpu_device *adev = ring->adev;
 	int idx;
-	int r;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
 		DRM_INFO("%s - device unplugged skipping recovery on scheduler:%s",
@@ -75,9 +84,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 		reset_context.reset_req_dev = adev;
 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
 
-		r = amdgpu_device_gpu_recover(ring->adev, job, &reset_context);
-		if (r)
-			DRM_ERROR("GPU Recovery Failed: %d\n", r);
+		amdgpu_reset_exec_work(ring->adev, &reset_context,
+				       amdgpu_job_recover);
 	} else {
 		drm_sched_suspend_timeout(&ring->sched);
 		if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 80d6e132e409..9b2e5e6e9388 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -357,6 +357,8 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		num_sched = &adev->gpu_sched[hw_ip][hw_prio].num_scheds;
 		adev->gpu_sched[hw_ip][hw_prio].sched[(*num_sched)++] =
 			&ring->sched;
+
+		ring->tdr_queue = create_singlethread_workqueue(ring->name);
 	}
 
 	return 0;
@@ -399,6 +401,9 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
 
 	if (!ring->is_mes_queue)
 		ring->adev->rings[ring->idx] = NULL;
+
+	if (ring->tdr_queue)
+		destroy_workqueue(ring->tdr_queue);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index e3f98554bb3c..3c60af19ad05 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -299,6 +299,7 @@ struct amdgpu_ring {
 	bool            is_sw_ring;
 	unsigned int    entry_index;
 
+	struct workqueue_struct *tdr_queue;
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
-- 
2.25.1