[PATCH 2/3] drm/i915/guc: Move GuC workqueue allocations to init_early
Michał Winiarski
michal.winiarski at intel.com
Thu Dec 7 13:38:41 UTC 2017
This moves the workqueue alloc outside of struct mutex, which gets rid
of the following lockdep splat:
======================================================
WARNING: possible circular locking dependency detected
4.15.0-rc2-CI-Patchwork_7428+ #1 Not tainted
------------------------------------------------------
debugfs_test/1351 is trying to acquire lock:
(&dev->struct_mutex){+.+.}, at: [<000000009d90d1a3>] i915_mutex_lock_interruptible+0x47/0x130 [i915]
but task is already holding lock:
(&mm->mmap_sem){++++}, at: [<000000005df01c1e>] __do_page_fault+0x106/0x560
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #6 (&mm->mmap_sem){++++}:
__might_fault+0x63/0x90
_copy_to_user+0x1e/0x70
filldir+0x8c/0xf0
dcache_readdir+0xeb/0x160
iterate_dir+0xe6/0x150
SyS_getdents+0xa0/0x130
entry_SYSCALL_64_fastpath+0x1c/0x89
-> #5 (&sb->s_type->i_mutex_key#5){++++}:
lockref_get+0x9/0x20
-> #4 ((completion)&req.done){+.+.}:
wait_for_common+0x54/0x210
devtmpfs_create_node+0x130/0x150
device_add+0x5ad/0x5e0
device_create_groups_vargs+0xd4/0xe0
device_create+0x35/0x40
msr_device_create+0x22/0x40
cpuhp_invoke_callback+0xc5/0xbf0
cpuhp_thread_fun+0x167/0x210
smpboot_thread_fn+0x17f/0x270
kthread+0x173/0x1b0
ret_from_fork+0x24/0x30
-> #3 (cpuhp_state-up){+.+.}:
cpuhp_issue_call+0x132/0x1c0
__cpuhp_setup_state_cpuslocked+0x12f/0x2a0
__cpuhp_setup_state+0x3a/0x50
page_writeback_init+0x3a/0x5c
start_kernel+0x393/0x3e2
secondary_startup_64+0xa5/0xb0
-> #2 (cpuhp_state_mutex){+.+.}:
__mutex_lock+0x81/0x9b0
__cpuhp_setup_state_cpuslocked+0x4b/0x2a0
__cpuhp_setup_state+0x3a/0x50
page_alloc_init+0x1f/0x26
start_kernel+0x139/0x3e2
secondary_startup_64+0xa5/0xb0
-> #1 (cpu_hotplug_lock.rw_sem){++++}:
cpus_read_lock+0x34/0xa0
apply_workqueue_attrs+0xd/0x40
__alloc_workqueue_key+0x2c7/0x4e1
intel_guc_submission_init+0x10c/0x650 [i915]
intel_uc_init_hw+0x29e/0x460 [i915]
i915_gem_init_hw+0xca/0x290 [i915]
i915_gem_init+0x115/0x3a0 [i915]
i915_driver_load+0x9a8/0x16c0 [i915]
i915_pci_probe+0x2e/0x90 [i915]
pci_device_probe+0x9c/0x120
driver_probe_device+0x2a3/0x480
__driver_attach+0xd9/0xe0
bus_for_each_dev+0x57/0x90
bus_add_driver+0x168/0x260
driver_register+0x52/0xc0
do_one_initcall+0x39/0x150
do_init_module+0x56/0x1ef
load_module+0x231c/0x2d70
SyS_finit_module+0xa5/0xe0
entry_SYSCALL_64_fastpath+0x1c/0x89
-> #0 (&dev->struct_mutex){+.+.}:
lock_acquire+0xaf/0x200
__mutex_lock+0x81/0x9b0
i915_mutex_lock_interruptible+0x47/0x130 [i915]
i915_gem_fault+0x201/0x760 [i915]
__do_fault+0x15/0x70
__handle_mm_fault+0x85b/0xe40
handle_mm_fault+0x14f/0x2f0
__do_page_fault+0x2d1/0x560
page_fault+0x22/0x30
other info that might help us debug this:
Chain exists of:
&dev->struct_mutex --> &sb->s_type->i_mutex_key#5 --> &mm->mmap_sem
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(&mm->mmap_sem);
lock(&sb->s_type->i_mutex_key#5);
lock(&mm->mmap_sem);
lock(&dev->struct_mutex);
*** DEADLOCK ***
1 lock held by debugfs_test/1351:
#0: (&mm->mmap_sem){++++}, at: [<000000005df01c1e>] __do_page_fault+0x106/0x560
stack backtrace:
CPU: 2 PID: 1351 Comm: debugfs_test Not tainted 4.15.0-rc2-CI-Patchwork_7428+ #1
Hardware name: /NUC6i5SYB, BIOS SYSKLi35.86A.0057.2017.0119.1758 01/19/2017
Call Trace:
dump_stack+0x5f/0x86
print_circular_bug+0x230/0x3b0
check_prev_add+0x439/0x7b0
? lockdep_init_map_crosslock+0x20/0x20
? unwind_get_return_address+0x16/0x30
? __lock_acquire+0x1385/0x15a0
__lock_acquire+0x1385/0x15a0
lock_acquire+0xaf/0x200
? i915_mutex_lock_interruptible+0x47/0x130 [i915]
__mutex_lock+0x81/0x9b0
? i915_mutex_lock_interruptible+0x47/0x130 [i915]
? i915_mutex_lock_interruptible+0x47/0x130 [i915]
? i915_mutex_lock_interruptible+0x47/0x130 [i915]
i915_mutex_lock_interruptible+0x47/0x130 [i915]
? __pm_runtime_resume+0x4f/0x80
i915_gem_fault+0x201/0x760 [i915]
__do_fault+0x15/0x70
__handle_mm_fault+0x85b/0xe40
handle_mm_fault+0x14f/0x2f0
__do_page_fault+0x2d1/0x560
page_fault+0x22/0x30
RIP: 0033:0x7f98d6f49116
RSP: 002b:00007ffd6ffc3278 EFLAGS: 00010283
RAX: 00007f98d39a2bc0 RBX: 0000000000000000 RCX: 0000000000001680
RDX: 0000000000001680 RSI: 00007ffd6ffc3400 RDI: 00007f98d39a2bc0
RBP: 00007ffd6ffc33a0 R08: 0000000000000000 R09: 00000000000005a0
R10: 000055e847c2a830 R11: 0000000000000002 R12: 0000000000000001
R13: 000055e847c1d040 R14: 00007ffd6ffc3400 R15: 00007f98d6752ba0
Testcase: igt/debugfs_test/read_all_entries
Signed-off-by: Michał Winiarski <michal.winiarski at intel.com>
Cc: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen at linux.intel.com>
Cc: Michal Wajdeczko <michal.wajdeczko at intel.com>
---
drivers/gpu/drm/i915/i915_drv.c | 10 ++++--
drivers/gpu/drm/i915/intel_guc.c | 46 ++++++++++++++++++++++++-
drivers/gpu/drm/i915/intel_guc.h | 3 +-
drivers/gpu/drm/i915/intel_guc_log.c | 52 ++++++++++++++++-------------
drivers/gpu/drm/i915/intel_guc_log.h | 2 ++
drivers/gpu/drm/i915/intel_guc_submission.c | 14 ++------
drivers/gpu/drm/i915/intel_guc_submission.h | 2 ++
drivers/gpu/drm/i915/intel_uc.c | 16 +++++++--
drivers/gpu/drm/i915/intel_uc.h | 3 +-
9 files changed, 106 insertions(+), 42 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 5b1fd5f1defb..15093d75045d 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -905,12 +905,15 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
mutex_init(&dev_priv->wm.wm_mutex);
mutex_init(&dev_priv->pps_mutex);
- intel_uc_init_early(dev_priv);
+ ret = intel_uc_init_early(dev_priv);
+ if (ret < 0)
+ goto err_engines;
+
i915_memcpy_init_early(dev_priv);
ret = i915_workqueues_init(dev_priv);
if (ret < 0)
- goto err_engines;
+ goto err_uc;
/* This must be called before any calls to HAS_PCH_* */
intel_detect_pch(dev_priv);
@@ -938,6 +941,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
err_irq:
intel_irq_fini(dev_priv);
i915_workqueues_cleanup(dev_priv);
+err_uc:
+ intel_uc_cleanup_early(dev_priv);
err_engines:
i915_engines_cleanup(dev_priv);
return ret;
@@ -951,6 +956,7 @@ static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
{
i915_gem_load_cleanup(dev_priv);
intel_irq_fini(dev_priv);
+ intel_uc_cleanup_early(dev_priv);
i915_workqueues_cleanup(dev_priv);
i915_engines_cleanup(dev_priv);
}
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 57589c0842d1..d81b4c2058d6 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -59,14 +59,58 @@ void intel_guc_init_send_regs(struct intel_guc *guc)
guc->send_regs.fw_domains = fw_domains;
}
-void intel_guc_init_early(struct intel_guc *guc)
+static int guc_workqueues_init(struct intel_guc *guc)
{
+ int ret;
+
+ ret = intel_guc_log_work_create(guc);
+ if (ret)
+ return ret;
+ GEM_BUG_ON(!guc->log.runtime.flush_wq);
+
+ if (intel_uc_is_using_guc_submission()) {
+ ret = intel_guc_preempt_work_create(guc);
+ if (ret) {
+ intel_guc_log_work_destroy(guc);
+ return ret;
+ }
+ GEM_BUG_ON(!guc->preempt_wq);
+ }
+
+ return 0;
+}
+
+static void guc_workqueues_cleanup(struct intel_guc *guc)
+{
+ if (intel_uc_is_using_guc_submission())
+ intel_guc_preempt_work_destroy(guc);
+
+ intel_guc_log_work_destroy(guc);
+}
+
+int intel_guc_init_early(struct intel_guc *guc)
+{
+ int ret;
+
intel_guc_fw_init_early(guc);
intel_guc_ct_init_early(&guc->ct);
+ ret = guc_workqueues_init(guc);
+ if (ret) {
+ DRM_ERROR("Couldn't allocate workqueues for GuC\n");
+ return ret;
+ }
+
mutex_init(&guc->send_mutex);
guc->send = intel_guc_send_nop;
guc->notify = gen8_guc_raise_irq;
+
+ return 0;
+}
+
+void intel_guc_cleanup_early(struct intel_guc *guc)
+{
+ guc_workqueues_cleanup(guc);
}
static int guc_shared_data_create(struct intel_guc *guc)
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 400c741bc718..f0342ca1730e 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -116,7 +116,8 @@ static inline u32 guc_ggtt_offset(struct i915_vma *vma)
return offset;
}
-void intel_guc_init_early(struct intel_guc *guc);
+int intel_guc_init_early(struct intel_guc *guc);
+void intel_guc_cleanup_early(struct intel_guc *guc);
void intel_guc_init_send_regs(struct intel_guc *guc);
void intel_guc_init_params(struct intel_guc *guc);
int intel_guc_communication_init(struct intel_guc *guc);
diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c
index 1a2c5eed9929..18128a5c0e66 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/intel_guc_log.c
@@ -351,6 +351,34 @@ static void capture_logs_work(struct work_struct *work)
guc_log_capture_logs(guc);
}
+int intel_guc_log_work_create(struct intel_guc *guc)
+{
+ INIT_WORK(&guc->log.runtime.flush_work, capture_logs_work);
+
+ /*
+ * GuC log buffer flush work item has to do register access to
+ * send the ack to GuC and this work item, if not synced before
+ * suspend, can potentially get executed after the GFX device is
+ * suspended.
+ * By marking the WQ as freezable, we don't have to bother about
+ * flushing of this work item from the suspend hooks, the pending
+ * work item if any will be either executed before the suspend
+ * or scheduled later on resume. This way the handling of work
+ * item can be kept same between system suspend & rpm suspend.
+ */
+ guc->log.runtime.flush_wq = alloc_ordered_workqueue("i915-guc_log",
+ WQ_HIGHPRI | WQ_FREEZABLE);
+ if (!guc->log.runtime.flush_wq)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void intel_guc_log_work_destroy(struct intel_guc *guc)
+{
+ destroy_workqueue(guc->log.runtime.flush_wq);
+}
+
static bool guc_log_has_runtime(struct intel_guc *guc)
{
return guc->log.runtime.buf_addr != NULL;
@@ -410,31 +438,8 @@ static int guc_log_runtime_create(struct intel_guc *guc)
GEM_BUG_ON(guc_log_relay_chan->subbuf_size < subbuf_size);
guc->log.runtime.relay_chan = guc_log_relay_chan;
- INIT_WORK(&guc->log.runtime.flush_work, capture_logs_work);
-
- /*
- * GuC log buffer flush work item has to do register access to
- * send the ack to GuC and this work item, if not synced before
- * suspend, can potentially get executed after the GFX device is
- * suspended.
- * By marking the WQ as freezable, we don't have to bother about
- * flushing of this work item from the suspend hooks, the pending
- * work item if any will be either executed before the suspend
- * or scheduled later on resume. This way the handling of work
- * item can be kept same between system suspend & rpm suspend.
- */
- guc->log.runtime.flush_wq = alloc_ordered_workqueue("i915-guc_log",
- WQ_HIGHPRI | WQ_FREEZABLE);
- if (!guc->log.runtime.flush_wq) {
- DRM_ERROR("Couldn't allocate the wq for GuC logging\n");
- ret = -ENOMEM;
- goto err_relaychan;
- }
-
return 0;
-err_relaychan:
- relay_close(guc->log.runtime.relay_chan);
err_vaddr:
i915_gem_object_unpin_map(guc->log.vma->obj);
guc->log.runtime.buf_addr = NULL;
@@ -450,7 +455,6 @@ static void guc_log_runtime_destroy(struct intel_guc *guc)
if (!guc_log_has_runtime(guc))
return;
- destroy_workqueue(guc->log.runtime.flush_wq);
relay_close(guc->log.runtime.relay_chan);
i915_gem_object_unpin_map(guc->log.vma->obj);
guc->log.runtime.buf_addr = NULL;
diff --git a/drivers/gpu/drm/i915/intel_guc_log.h b/drivers/gpu/drm/i915/intel_guc_log.h
index f512cf79339b..f43deb75d493 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.h
+++ b/drivers/gpu/drm/i915/intel_guc_log.h
@@ -52,6 +52,8 @@ struct intel_guc_log {
int intel_guc_log_create(struct intel_guc *guc);
void intel_guc_log_destroy(struct intel_guc *guc);
+int intel_guc_log_work_create(struct intel_guc *guc);
+void intel_guc_log_work_destroy(struct intel_guc *guc);
int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val);
void i915_guc_log_register(struct drm_i915_private *dev_priv);
void i915_guc_log_unregister(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
index c020560c395e..1d797578158e 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -1187,7 +1187,7 @@ static void guc_ads_destroy(struct intel_guc *guc)
i915_vma_unpin_and_release(&guc->ads_vma);
}
-static int guc_preempt_work_create(struct intel_guc *guc)
+int intel_guc_preempt_work_create(struct intel_guc *guc)
{
struct drm_i915_private *dev_priv = guc_to_i915(guc);
struct intel_engine_cs *engine;
@@ -1219,7 +1219,7 @@ static int guc_preempt_work_create(struct intel_guc *guc)
return 0;
}
-static void guc_preempt_work_destroy(struct intel_guc *guc)
+void intel_guc_preempt_work_destroy(struct intel_guc *guc)
{
struct drm_i915_private *dev_priv = guc_to_i915(guc);
struct intel_engine_cs *engine;
@@ -1256,20 +1256,13 @@ int intel_guc_submission_init(struct intel_guc *guc)
if (ret < 0)
goto err_stage_desc_pool;
- ret = guc_preempt_work_create(guc);
- if (ret)
- goto err_log;
- GEM_BUG_ON(!guc->preempt_wq);
-
ret = guc_ads_create(guc);
if (ret < 0)
- goto err_wq;
+ goto err_log;
GEM_BUG_ON(!guc->ads_vma);
return 0;
-err_wq:
- guc_preempt_work_destroy(guc);
err_log:
intel_guc_log_destroy(guc);
err_stage_desc_pool:
@@ -1280,7 +1273,6 @@ int intel_guc_submission_init(struct intel_guc *guc)
void intel_guc_submission_fini(struct intel_guc *guc)
{
guc_ads_destroy(guc);
- guc_preempt_work_destroy(guc);
intel_guc_log_destroy(guc);
guc_stage_desc_pool_destroy(guc);
}
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.h b/drivers/gpu/drm/i915/intel_guc_submission.h
index 021fe85c8f71..fb081cefef93 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.h
+++ b/drivers/gpu/drm/i915/intel_guc_submission.h
@@ -77,5 +77,7 @@ int intel_guc_submission_init(struct intel_guc *guc);
int intel_guc_submission_enable(struct intel_guc *guc);
void intel_guc_submission_disable(struct intel_guc *guc);
void intel_guc_submission_fini(struct intel_guc *guc);
+int intel_guc_preempt_work_create(struct intel_guc *guc);
+void intel_guc_preempt_work_destroy(struct intel_guc *guc);
#endif
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index b2abd46dae80..93224b094188 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -109,10 +109,22 @@ void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
GEM_BUG_ON(i915_modparams.enable_guc < 0);
}
-void intel_uc_init_early(struct drm_i915_private *dev_priv)
+int intel_uc_init_early(struct drm_i915_private *dev_priv)
{
- intel_guc_init_early(&dev_priv->guc);
+ int ret;
+
+ ret = intel_guc_init_early(&dev_priv->guc);
+ if (ret)
+ return ret;
+
intel_huc_init_early(&dev_priv->huc);
+
+ return 0;
+}
+
+void intel_uc_cleanup_early(struct drm_i915_private *dev_priv)
+{
+ intel_guc_cleanup_early(&dev_priv->guc);
}
void intel_uc_init_fw(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h
index 7a59e2486e9e..d0467852da19 100644
--- a/drivers/gpu/drm/i915/intel_uc.h
+++ b/drivers/gpu/drm/i915/intel_uc.h
@@ -29,7 +29,8 @@
#include "i915_params.h"
void intel_uc_sanitize_options(struct drm_i915_private *dev_priv);
-void intel_uc_init_early(struct drm_i915_private *dev_priv);
+int intel_uc_init_early(struct drm_i915_private *dev_priv);
+void intel_uc_cleanup_early(struct drm_i915_private *dev_priv);
void intel_uc_init_mmio(struct drm_i915_private *dev_priv);
void intel_uc_init_fw(struct drm_i915_private *dev_priv);
void intel_uc_fini_fw(struct drm_i915_private *dev_priv);
--
2.14.3
More information about the Intel-gfx-trybot
mailing list