[PATCH 2/3] drm/amd: Poll for GFX core to be off

[Lazar, Lijo]

[AMD Official Use Only - General]

Is this register GFX_IMU_MSG_FLAGS outside of GFXOFF domain?

Thanks,
Lijo

-----Original Message-----
From: amd-gfx <amd-gfx-bounces at lists.freedesktop.org> On Behalf Of Mario Limonciello
Sent: Tuesday, May 16, 2023 11:22 PM
To: amd-gfx at lists.freedesktop.org
Cc: Tsao, Anson <anson.tsao at amd.com>; Huang, Tim <Tim.Huang at amd.com>; Martinez, Juan <Juan.Martinez at amd.com>; Limonciello, Mario <Mario.Limonciello at amd.com>; Gong, Richard <Richard.Gong at amd.com>
Subject: [PATCH 2/3] drm/amd: Poll for GFX core to be off

If GFXOFF was flushed during suspend entry it may take some time for GFX core to be powered down.  Ensure that it's powered off before continuing any operations that may try to utilize related IP. This avoids hangs from stopping RLC as well as problems with fence interrupts timing out during s2idle entry and exit.

Cc: stable at vger.kernel.org # 6.1+
Tested-by: Juan Martinez <Juan.Martinez at amd.com>
Tested-by: Anson Tsao <anson.tsao at amd.com>
Suggested-by: Tim Huang <tim.huang at amd.com>
Signed-off-by: Mario Limonciello <mario.limonciello at amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 20 ++++++++++++++++++--
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c     | 18 ++++++++++++++++++
 drivers/gpu/drm/amd/include/amd_shared.h   |  1 +
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 059139f1f973..17fc053405ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3055,12 +3055,28 @@ static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)  {
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
+	int r, i;
 
 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
 
-	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
-		adev->gfx.gfx_off_state = true;
+	r = amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true);
+	if (r) {
+		DRM_ERROR("failed to enable gfxoff: %d\n", r);
+		return;
+	}
+	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
+		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_GFX)
+			continue;
+		if (!adev->ip_blocks[i].version->funcs->wait_for_off)
+			continue;
+		r = adev->ip_blocks[i].version->funcs->wait_for_off((void *)adev);
+		if (r) {
+			DRM_ERROR("failed to wait for gfxoff: %d\n", r);
+			return;
+		}
+	}
+	adev->gfx.gfx_off_state = true;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 4b7224de879e..dcbdb2641086 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -4434,6 +4434,23 @@ static int gfx_v11_0_wait_for_idle(void *handle)
 	return -ETIMEDOUT;
 }
 
+
+static int gfx_v11_0_wait_for_off(void *handle) {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 tmp;
+	int i;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32_SOC15(GC, 0, regGFX_IMU_MSG_FLAGS);
+		if (!(tmp & 0x06))
+			return 0;
+		udelay(1);
+	}
+	dev_dbg(adev->dev, "GFX IMU is %x\n", tmp);
+	return -ETIMEDOUT;
+}
+
 static int gfx_v11_0_soft_reset(void *handle)  {
 	u32 grbm_soft_reset = 0;
@@ -6109,6 +6126,7 @@ static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
 	.resume = gfx_v11_0_resume,
 	.is_idle = gfx_v11_0_is_idle,
 	.wait_for_idle = gfx_v11_0_wait_for_idle,
+	.wait_for_off = gfx_v11_0_wait_for_off,
 	.soft_reset = gfx_v11_0_soft_reset,
 	.check_soft_reset = gfx_v11_0_check_soft_reset,
 	.post_soft_reset = gfx_v11_0_post_soft_reset, diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index f175e65b853a..ce2e2b6fd6ff 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -298,6 +298,7 @@ struct amd_ip_funcs {
 	int (*resume)(void *handle);
 	bool (*is_idle)(void *handle);
 	int (*wait_for_idle)(void *handle);
+	int (*wait_for_off)(void *handle);
 	bool (*check_soft_reset)(void *handle);
 	int (*pre_soft_reset)(void *handle);
 	int (*soft_reset)(void *handle);
--
2.34.1