[PATCH 3/5] drm/amdgpu: Use asynchronous polling to handle umc_v12_0 poisoning
Chai, Thomas
YiPeng.Chai at amd.com
Thu Jan 18 05:29:30 UTC 2024
[AMD Official Use Only - General]
-----------------
Best Regards,
Thomas
_____________________________________________
From: Zhou1, Tao <Tao.Zhou1 at amd.com>
Sent: Thursday, January 18, 2024 11:24 AM
To: Chai, Thomas <YiPeng.Chai at amd.com>; Zhang, Hawking <Hawking.Zhang at amd.com>; amd-gfx at lists.freedesktop.org
Cc: Li, Candice <Candice.Li at amd.com>; Wang, Yang(Kevin) <KevinYang.Wang at amd.com>; Yang, Stanley <Stanley.Yang at amd.com>
Subject: RE: [PATCH 3/5] drm/amdgpu: Use asynchronous polling to handle umc_v12_0 poisoning
[AMD Official Use Only - General]
_____________________________________________
From: Chai, Thomas <YiPeng.Chai at amd.com<mailto:YiPeng.Chai at amd.com>>
Sent: Thursday, January 18, 2024 11:06 AM
To: Zhang, Hawking <Hawking.Zhang at amd.com<mailto:Hawking.Zhang at amd.com>>; amd-gfx at lists.freedesktop.org<mailto:amd-gfx at lists.freedesktop.org>
Cc: Zhou1, Tao <Tao.Zhou1 at amd.com<mailto:Tao.Zhou1 at amd.com>>; Li, Candice <Candice.Li at amd.com<mailto:Candice.Li at amd.com>>; Wang, Yang(Kevin) <KevinYang.Wang at amd.com<mailto:KevinYang.Wang at amd.com>>; Yang, Stanley <Stanley.Yang at amd.com<mailto:Stanley.Yang at amd.com>>
Subject: RE: [PATCH 3/5] drm/amdgpu: Use asynchronous polling to handle umc_v12_0 poisoning
[AMD Official Use Only - General]
-----------------
Best Regards,
Thomas
_____________________________________________
From: Zhang, Hawking <Hawking.Zhang at amd.com<mailto:Hawking.Zhang at amd.com>>
Sent: Wednesday, January 17, 2024 7:54 PM
To: Chai, Thomas <YiPeng.Chai at amd.com<mailto:YiPeng.Chai at amd.com>>; amd-gfx at lists.freedesktop.org<mailto:amd-gfx at lists.freedesktop.org>
Cc: Zhou1, Tao <Tao.Zhou1 at amd.com<mailto:Tao.Zhou1 at amd.com>>; Li, Candice <Candice.Li at amd.com<mailto:Candice.Li at amd.com>>; Wang, Yang(Kevin) <KevinYang.Wang at amd.com<mailto:KevinYang.Wang at amd.com>>; Yang, Stanley <Stanley.Yang at amd.com<mailto:Stanley.Yang at amd.com>>
Subject: RE: [PATCH 3/5] drm/amdgpu: Use asynchronous polling to handle umc_v12_0 poisoning
[AMD Official Use Only - General]
Please check my comments inline
Regards,
Hawking
-----Original Message-----
From: Chai, Thomas <YiPeng.Chai at amd.com<mailto:YiPeng.Chai at amd.com>>
Sent: Tuesday, January 16, 2024 16:21
To: amd-gfx at lists.freedesktop.org<mailto:amd-gfx at lists.freedesktop.org>
Cc: Chai, Thomas <YiPeng.Chai at amd.com<mailto:YiPeng.Chai at amd.com>>; Zhang, Hawking <Hawking.Zhang at amd.com<mailto:Hawking.Zhang at amd.com>>; Zhou1, Tao <Tao.Zhou1 at amd.com<mailto:Tao.Zhou1 at amd.com>>; Li, Candice <Candice.Li at amd.com<mailto:Candice.Li at amd.com>>; Wang, Yang(Kevin) <KevinYang.Wang at amd.com<mailto:KevinYang.Wang at amd.com>>; Yang, Stanley <Stanley.Yang at amd.com<mailto:Stanley.Yang at amd.com>>; Chai, Thomas <YiPeng.Chai at amd.com<mailto:YiPeng.Chai at amd.com>>
Subject: [PATCH 3/5] drm/amdgpu: Use asynchronous polling to handle umc_v12_0 poisoning
Use asynchronous polling to handle umc_v12_0 poisoning.
Signed-off-by: YiPeng Chai <YiPeng.Chai at amd.com<mailto:YiPeng.Chai at amd.com>>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 5 +
drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c | 143 +++++++++++++++++++-----
drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h | 3 +
3 files changed, 120 insertions(+), 31 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 856206e95842..44929281840e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -118,6 +118,8 @@ const char *get_ras_block_str(struct ras_common_if *ras_block)
/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
+#define MAX_UMC_POISON_POLLING_TIME_ASYNC 100 //ms
+
enum amdgpu_ras_retire_page_reservation {
AMDGPU_RAS_RETIRE_PAGE_RESERVED,
AMDGPU_RAS_RETIRE_PAGE_PENDING,
@@ -2670,6 +2672,9 @@ static int amdgpu_ras_page_retirement_thread(void *param)
atomic_read(&con->page_retirement_req_cnt));
atomic_dec(&con->page_retirement_req_cnt);
+
+ amdgpu_umc_poison_retire_page_polling_timeout(adev,
+ false, MAX_UMC_POISON_POLLING_TIME_ASYNC);
}
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 9d1cf41cf483..2dde29cb807d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -23,6 +23,7 @@
#include "amdgpu.h"
#include "umc_v6_7.h"
+#define MAX_UMC_POISON_POLLING_TIME_SYNC 20 //ms
static int amdgpu_umc_convert_error_address(struct amdgpu_device *adev,
struct ras_err_data *err_data, uint64_t err_addr, @@ -85,17 +86,14 @@ int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev,
return ret;
}
-static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
- void *ras_error_status,
- struct amdgpu_iv_entry *entry,
- bool reset)
+static void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
+ void *ras_error_status)
{
struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
int ret = 0;
unsigned long err_count;
-
- kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+ mutex_lock(&con->page_retirement_lock);
ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc));
if (ret == -EOPNOTSUPP) {
if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && @@ -163,19 +161,86 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
con->update_channel_flag = false;
}
}
-
- if (reset) {
- /* use mode-2 reset for poison consumption */
- if (!entry)
- con->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE2_RESET;
- amdgpu_ras_reset_gpu(adev);
- }
}
kfree(err_data->err_addr);
+
+ mutex_unlock(&con->page_retirement_lock);
+}
+
+static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
+ void *ras_error_status,
+ struct amdgpu_iv_entry *entry,
+ bool reset)
+{
+ struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
+ struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+ kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+ amdgpu_umc_handle_bad_pages(adev, ras_error_status);
+
+ if (err_data->ue_count && reset) {
+ /* use mode-2 reset for poison consumption */
+ if (!entry)
+ con->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE2_RESET;
>[Hawking]: Shall we do further check on con->poison_supported flag to decide issuing mode-2 or mode-1.
> [thomas] This "gpu reset" code is not belong to page retirement range, but reuse old code. About how to reset gpu, we can do it with another patch in future.
>[Tao] in fact, the condition "if (!entry)" is for poison mode. If poison isn't supported, only umc ue error needs page retirement, it will be called in related cb interface and the entry would not be null.
[thomas] "if (!entry)" is useful for the umc version less than v12 in umc poison handler.
+ amdgpu_ras_reset_gpu(adev);
+ }
+
return AMDGPU_RAS_SUCCESS;
}
+int amdgpu_umc_poison_retire_page_polling_timeout(struct amdgpu_device *adev,
+ bool reset, uint32_t timeout_ms)
[Hawking] int amdgpu_umc_bad_page_polling_timeout(struct amdgpu_device *adev, boot reset, uint32_t timeout_ms)
[thomas] OK
+{
+ struct ras_err_data err_data;
+ struct ras_common_if head = {
+ .block = AMDGPU_RAS_BLOCK__UMC,
+ };
+ struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head);
+ uint32_t timeout = timeout_ms;
+
+ memset(&err_data, 0, sizeof(err_data));
+ amdgpu_ras_error_data_init(&err_data);
+
+ do {
+
+ amdgpu_umc_handle_bad_pages(adev, &err_data);
+
+ if (timeout && !err_data.de_count) {
+ msleep(1);
+ timeout--;
+ }
+
+ } while (timeout && !err_data.de_count);
+
+ if (!timeout)
+ dev_warn(adev->dev, "Page retirment executed, but did not find bad
+pages\n");
[Hawking] dev_warn (adev->dev, "can't find bad pages\n");
[thomas] OK
+
+ if (err_data.de_count)
+ dev_info(adev->dev, "Page retirment: ue:%ld, ce:%ld, de:%ld\n",
+ err_data.ue_count, err_data.ce_count, err_data.de_count);
[Hawking] dev_info(adev->dev, "%ld new deferred hardware errors detected\n", err_data.de_count)
[thomas] OK
+
+ if (obj) {
+ obj->err_data.ue_count += err_data.ue_count;
+ obj->err_data.ce_count += err_data.ce_count;
+ obj->err_data.de_count += err_data.de_count;
+ }
+
+ amdgpu_ras_error_data_fini(&err_data);
+
+ kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+
+ if (reset) {
+ struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+ /* use mode-2 reset for poison consumption */
+ con->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE2_RESET;
[Hawking]: Shall we do further check on con->poison_supported flag to decide issuing mode-2 or mode-1.
[thomas] This "gpu reset" code is not belong to page retirement range, but reuse old code. About how to reset gpu, we can do it with another patch in future.
+ amdgpu_ras_reset_gpu(adev);
+ }
+
+ return 0;
+}
+
int amdgpu_umc_poison_handler(struct amdgpu_device *adev, bool reset) {
int ret = AMDGPU_RAS_SUCCESS;
@@ -193,25 +258,41 @@ int amdgpu_umc_poison_handler(struct amdgpu_device *adev, bool reset)
}
if (!amdgpu_sriov_vf(adev)) {
- struct ras_err_data err_data;
- struct ras_common_if head = {
- .block = AMDGPU_RAS_BLOCK__UMC,
- };
- struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head);
-
- ret = amdgpu_ras_error_data_init(&err_data);
- if (ret)
- return ret;
-
- ret = amdgpu_umc_do_page_retirement(adev, &err_data, NULL, reset);
-
- if (ret == AMDGPU_RAS_SUCCESS && obj) {
- obj->err_data.ue_count += err_data.ue_count;
- obj->err_data.ce_count += err_data.ce_count;
- obj->err_data.de_count += err_data.de_count;
- }
+ if (amdgpu_ip_version(adev, UMC_HWIP, 0) != IP_VERSION(12, 0, 0)) {
[Hawking]: if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) {
[thomas] OK
+ struct ras_err_data err_data;
+ struct ras_common_if head = {
+ .block = AMDGPU_RAS_BLOCK__UMC,
+ };
+ struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head);
+
+ ret = amdgpu_ras_error_data_init(&err_data);
+ if (ret)
+ return ret;
+
+ ret = amdgpu_umc_do_page_retirement(adev, &err_data, NULL, reset);
- amdgpu_ras_error_data_fini(&err_data);
+ if (ret == AMDGPU_RAS_SUCCESS && obj) {
+ obj->err_data.ue_count += err_data.ue_count;
+ obj->err_data.ce_count += err_data.ce_count;
+ obj->err_data.de_count += err_data.de_count;
+ }
+
+ amdgpu_ras_error_data_fini(&err_data);
+ } else {
+ if (reset) {
+ amdgpu_umc_poison_retire_page_polling_timeout(adev,
+ reset, MAX_UMC_POISON_POLLING_TIME_SYNC);
+ } else {
+ struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+ dev_info(adev->dev, "Page retirement pending request count: %d\n",
+ atomic_read(&con->page_retirement_req_cnt));
[Hawking]: might remove the printed message here.
[thomas] OK
+
+ atomic_inc(&con->page_retirement_req_cnt);
+
+ wake_up(&con->page_retirement_wq);
+ }
+ }
} else {
if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
adev->virt.ops->ras_poison_handler(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index 417a6726c71b..ee487adce7e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -118,4 +118,7 @@ int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev,
int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
umc_func func, void *data);
+
+int amdgpu_umc_poison_retire_page_polling_timeout(struct amdgpu_device *adev,
+ bool reset, uint32_t timeout_ms);
#endif
--
2.34.1
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.freedesktop.org/archives/amd-gfx/attachments/20240118/03572391/attachment-0001.htm>
More information about the amd-gfx
mailing list