[PATCH 4/4] drm/xe: Introduce the wedged_mode debugfs
Rodrigo Vivi
rodrigo.vivi at intel.com
Wed Apr 3 15:07:32 UTC 2024
So, the wedged mode can be selected per device at runtime,
before the tests or before reproducing the issue.
v2: - s/busted/wedged
- some locking consistency
Cc: Lucas De Marchi <lucas.demarchi at intel.com>
Cc: Alan Previn <alan.previn.teres.alexis at intel.com>
Signed-off-by: Rodrigo Vivi <rodrigo.vivi at intel.com>
---
drivers/gpu/drm/xe/xe_debugfs.c | 56 ++++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_device.c | 41 ++++++++++++++------
drivers/gpu/drm/xe/xe_device.h | 4 +-
drivers/gpu/drm/xe/xe_device_types.h | 11 +++++-
drivers/gpu/drm/xe/xe_gt.c | 2 +-
drivers/gpu/drm/xe/xe_guc.c | 2 +-
drivers/gpu/drm/xe/xe_guc_ads.c | 52 +++++++++++++++++++++++++-
drivers/gpu/drm/xe/xe_guc_ads.h | 1 +
drivers/gpu/drm/xe/xe_guc_submit.c | 4 +-
9 files changed, 150 insertions(+), 23 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index 8abdf3c17e1d..eb0932105e80 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -12,6 +12,7 @@
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_gt_debugfs.h"
+#include "xe_guc_ads.h"
#include "xe_pm.h"
#include "xe_step.h"
@@ -106,6 +107,58 @@ static const struct file_operations forcewake_all_fops = {
.release = forcewake_release,
};
+static ssize_t wedged_mode_show(struct file *f, char __user *ubuf,
+ size_t size, loff_t *pos)
+{
+ struct xe_device *xe = file_inode(f)->i_private;
+ char buf[32];
+ int len = 0;
+
+ mutex_lock(&xe->wedged.lock);
+ len = scnprintf(buf, sizeof(buf), "%d\n", xe->wedged.mode);
+ mutex_unlock(&xe->wedged.lock);
+
+ return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+
+static ssize_t wedged_mode_set(struct file *f, const char __user *ubuf,
+ size_t size, loff_t *pos)
+{
+ struct xe_device *xe = file_inode(f)->i_private;
+ struct xe_gt *gt;
+ u32 wedged_mode;
+ ssize_t ret;
+ u8 id;
+
+ ret = kstrtouint_from_user(ubuf, size, 0, &wedged_mode);
+ if (ret)
+ return ret;
+
+ if (wedged_mode > 2)
+ return -EINVAL;
+
+ mutex_lock(&xe->wedged.lock);
+ xe->wedged.mode = wedged_mode;
+ if (wedged_mode == 2) {
+ for_each_gt(gt, xe, id) {
+ ret = xe_guc_ads_scheduler_policy_disable_reset(>->uc.guc.ads);
+ if (ret) {
+ drm_err(&xe->drm, "Failed to update GuC ADS scheduler policy. GPU might still reset even on the wedged_mode=2\n");
+ break;
+ }
+ }
+ }
+ mutex_unlock(&xe->wedged.lock);
+
+ return size;
+}
+
+static const struct file_operations wedged_mode_fops = {
+ .owner = THIS_MODULE,
+ .read = wedged_mode_show,
+ .write = wedged_mode_set,
+};
+
void xe_debugfs_register(struct xe_device *xe)
{
struct ttm_device *bdev = &xe->ttm;
@@ -123,6 +176,9 @@ void xe_debugfs_register(struct xe_device *xe)
debugfs_create_file("forcewake_all", 0400, root, xe,
&forcewake_all_fops);
+ debugfs_create_file("wedged_mode", 0400, root, xe,
+ &wedged_mode_fops);
+
for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) {
man = ttm_manager_type(bdev, mem_type);
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 8e380a404a26..31bece971a13 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -436,6 +436,9 @@ int xe_device_probe_early(struct xe_device *xe)
if (err)
return err;
+ mutex_init(&xe->wedged.lock);
+ xe->wedged.mode = xe_modparam.wedged_mode;
+
return 0;
}
@@ -778,26 +781,37 @@ u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
}
/**
- * xe_device_declare_wedged - Declare device wedged
+ * xe_device_hint_wedged - Get a hint and possibly declare device as wedged
* @xe: xe device instance
+ * @in_timeout_path: hint coming from a timeout path
*
- * This is a final state that can only be cleared with a module
+ * The wedged state is a final on that can only be cleared with a module
* re-probe (unbind + bind).
* In this state every IOCTL will be blocked so the GT cannot be used.
- * In general it will be called upon any critical error such as gt reset
- * failure or guc loading failure.
- * If xe.wedged module parameter is set to 2, this function will be called
- * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
- * snapshot capture. In this mode, GT reset won't be attempted so the state of
- * the issue is preserved for further debugging.
+ * In general device will be declared wedged only at critical
+ * error paths such as gt reset failure or guc loading failure.
+ * Hints are also expected from every single execution timeout (a.k.a. GPU hang)
+ * right after devcoredump snapshot capture. Then, device can be declared wedged
+ * if wedged_mode is set to 2. In this mode, GT reset won't be attempted so the
+ * state of the issue is preserved for further debugging.
+ *
+ * Return: True if device has been just declared wedged. False otherwise
*/
-void xe_device_declare_wedged(struct xe_device *xe)
+bool xe_device_hint_wedged(struct xe_device *xe, bool in_timeout_path)
{
- if (xe_modparam.wedged_mode == 0)
- return;
+ bool ret = false;
+
+ mutex_lock(&xe->wedged.lock);
- if (!atomic_xchg(&xe->wedged, 1)) {
+ if (xe->wedged.mode == 0)
+ goto out;
+
+ if (in_timeout_path && xe->wedged.mode != 2)
+ goto out;
+
+ if (!atomic_xchg(&xe->wedged.flag, 1)) {
xe->needs_flr_on_fini = true;
+ ret = true;
drm_err(&xe->drm,
"CRITICAL: Xe has declared device %s as wedged.\n"
"IOCTLs and executions are blocked until device is probed again with unbind and bind operations:\n"
@@ -807,4 +821,7 @@ void xe_device_declare_wedged(struct xe_device *xe)
dev_name(xe->drm.dev), dev_name(xe->drm.dev),
dev_name(xe->drm.dev));
}
+out:
+ mutex_unlock(&xe->wedged.lock);
+ return ret;
}
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index 0fea5c18f76d..e3ea8a43e7f9 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -178,9 +178,9 @@ u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);
static inline bool xe_device_wedged(struct xe_device *xe)
{
- return atomic_read(&xe->wedged);
+ return atomic_read(&xe->wedged.flag);
}
-void xe_device_declare_wedged(struct xe_device *xe);
+bool xe_device_hint_wedged(struct xe_device *xe, bool in_timeout_path);
#endif
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 0430bd51a5dd..39ebe683c33e 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -455,8 +455,15 @@ struct xe_device {
/** @needs_flr_on_fini: requests function-reset on fini */
bool needs_flr_on_fini;
- /** @wedged: Xe device faced a critical error and is now blocked. */
- atomic_t wedged;
+ /** @wedged: Struct to control Wedged States and mode */
+ struct {
+ /** @wedged.flag: Xe device faced a critical error and is now blocked. */
+ atomic_t flag;
+ /** @wedged.mode: Mode controlled by kernel parameter and debugfs */
+ int mode;
+ /** @wedged.lock: To protect @wedged.mode */
+ struct mutex lock;
+ } wedged;
/* private: */
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index 0844081b88ef..da16f4273877 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -688,7 +688,7 @@ static int gt_reset(struct xe_gt *gt)
err_fail:
xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
- xe_device_declare_wedged(gt_to_xe(gt));
+ xe_device_hint_wedged(gt_to_xe(gt), false);
return err;
}
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index a7cd65a9cf32..3951e9197cf5 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -500,7 +500,7 @@ static void guc_wait_ucode(struct xe_guc *guc)
SOFT_SCRATCH(13)));
}
- xe_device_declare_wedged(xe);
+ xe_device_hint_wedged(xe, false);
} else {
drm_dbg(&xe->drm, "GuC successfully loaded");
}
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index 37f30c333c93..fe6104c5cec0 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -7,6 +7,7 @@
#include <drm/drm_managed.h>
+#include "abi/guc_actions_abi.h"
#include "regs/xe_engine_regs.h"
#include "regs/xe_gt_regs.h"
#include "regs/xe_guc_regs.h"
@@ -14,11 +15,11 @@
#include "xe_gt.h"
#include "xe_gt_ccs_mode.h"
#include "xe_guc.h"
+#include "xe_guc_ct.h"
#include "xe_hw_engine.h"
#include "xe_lrc.h"
#include "xe_map.h"
#include "xe_mmio.h"
-#include "xe_module.h"
#include "xe_platform_types.h"
/* Slack of a few additional entries per engine */
@@ -314,6 +315,7 @@ int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads)
static void guc_policies_init(struct xe_guc_ads *ads)
{
+ struct xe_device *xe = ads_to_xe(ads);
u32 global_flags = 0;
ads_blob_write(ads, policies.dpc_promote_time,
@@ -321,8 +323,10 @@ static void guc_policies_init(struct xe_guc_ads *ads)
ads_blob_write(ads, policies.max_num_work_items,
GLOBAL_POLICY_MAX_NUM_WI);
- if (xe_modparam.wedged_mode == 2)
+ mutex_lock(&xe->wedged.lock);
+ if (xe->wedged.mode == 2)
global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
+ mutex_unlock(&xe->wedged.lock);
ads_blob_write(ads, policies.global_flags, global_flags);
ads_blob_write(ads, policies.is_valid, 1);
@@ -678,3 +682,47 @@ void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads)
{
guc_populate_golden_lrc(ads);
}
+
+static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset)
+{
+ struct xe_guc_ct *ct = &ads_to_guc(ads)->ct;
+ u32 action[] = {
+ XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
+ policy_offset
+ };
+
+ return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
+}
+
+int xe_guc_ads_scheduler_policy_disable_reset(struct xe_guc_ads *ads)
+{
+ struct xe_device *xe = ads_to_xe(ads);
+ struct xe_gt *gt = ads_to_gt(ads);
+ struct xe_tile *tile = gt_to_tile(gt);
+ struct guc_policies *policies;
+ struct xe_bo *bo;
+ int ret = 0;
+
+ policies = kmalloc(sizeof(*policies), GFP_KERNEL);
+ if (!policies)
+ return -ENOMEM;
+
+ policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time);
+ policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items);
+ policies->is_valid = 1;
+ if (xe->wedged.mode == 2)
+ policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
+
+ bo = xe_managed_bo_create_from_data(xe, tile, policies, sizeof(struct guc_policies),
+ XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+ XE_BO_FLAG_GGTT);
+ if (IS_ERR(bo)) {
+ ret = PTR_ERR(bo);
+ goto out;
+ }
+
+ ret = guc_ads_action_update_policies(ads, xe_bo_ggtt_addr(bo));
+out:
+ kfree(policies);
+ return ret;
+}
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.h b/drivers/gpu/drm/xe/xe_guc_ads.h
index 138ef6267671..7c45c40fab34 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.h
+++ b/drivers/gpu/drm/xe/xe_guc_ads.h
@@ -13,5 +13,6 @@ int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads);
void xe_guc_ads_populate(struct xe_guc_ads *ads);
void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads);
void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads);
+int xe_guc_ads_scheduler_policy_disable_reset(struct xe_guc_ads *ads);
#endif
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index 0eba01582f7c..f42f1b567067 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -35,7 +35,6 @@
#include "xe_macros.h"
#include "xe_map.h"
#include "xe_mocs.h"
-#include "xe_module.h"
#include "xe_ring_ops_types.h"
#include "xe_sched_job.h"
#include "xe_trace.h"
@@ -962,8 +961,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
trace_xe_sched_job_timedout(job);
- if (xe_modparam.wedged_mode == 2) {
- xe_device_declare_wedged(xe);
+ if (xe_device_hint_wedged(xe, true)) {
guc_submit_device_wedged(q);
return DRM_GPU_SCHED_STAT_ENODEV;
}
--
2.44.0
More information about the Intel-xe
mailing list