[PATCH 11/11] engine-pm
Chris Wilson
chris at chris-wilson.co.uk
Fri Apr 5 01:00:22 UTC 2019
---
drivers/gpu/drm/i915/Makefile | 2 +
drivers/gpu/drm/i915/gt/intel_context.c | 13 +-
drivers/gpu/drm/i915/gt/intel_engine.h | 3 -
drivers/gpu/drm/i915/gt/intel_engine_cs.c | 81 +-----
drivers/gpu/drm/i915/gt/intel_engine_pm.c | 103 ++++++++
drivers/gpu/drm/i915/gt/intel_engine_pm.h | 17 ++
drivers/gpu/drm/i915/gt/intel_engine_types.h | 5 +
drivers/gpu/drm/i915/gt/intel_gt_pm.c | 82 ++++++
drivers/gpu/drm/i915/gt/intel_gt_pm.h | 22 ++
drivers/gpu/drm/i915/gt/intel_hangcheck.c | 7 +
drivers/gpu/drm/i915/i915_debugfs.c | 12 +-
drivers/gpu/drm/i915/i915_drv.h | 8 +-
drivers/gpu/drm/i915/i915_gem.c | 6 +-
drivers/gpu/drm/i915/i915_gem.h | 3 -
drivers/gpu/drm/i915/i915_gem_evict.c | 47 +---
drivers/gpu/drm/i915/i915_gem_pm.c | 237 ++++++------------
drivers/gpu/drm/i915/i915_gem_pm.h | 3 -
drivers/gpu/drm/i915/i915_request.c | 10 +-
drivers/gpu/drm/i915/i915_request.h | 2 +-
drivers/gpu/drm/i915/intel_wakeref.h | 3 +-
.../gpu/drm/i915/selftests/i915_gem_context.c | 12 +-
.../gpu/drm/i915/selftests/i915_gem_object.c | 29 ++-
.../gpu/drm/i915/selftests/igt_flush_test.c | 2 +-
.../gpu/drm/i915/selftests/mock_gem_device.c | 5 -
24 files changed, 371 insertions(+), 343 deletions(-)
create mode 100644 drivers/gpu/drm/i915/gt/intel_engine_pm.c
create mode 100644 drivers/gpu/drm/i915/gt/intel_engine_pm.h
create mode 100644 drivers/gpu/drm/i915/gt/intel_gt_pm.c
create mode 100644 drivers/gpu/drm/i915/gt/intel_gt_pm.h
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 858642c7bc40..dd8d923aa1c6 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -71,6 +71,8 @@ gt-y += \
gt/intel_breadcrumbs.o \
gt/intel_context.o \
gt/intel_engine_cs.o \
+ gt/intel_engine_pm.o \
+ gt/intel_gt_pm.o \
gt/intel_hangcheck.o \
gt/intel_lrc.o \
gt/intel_reset.o \
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index 4681f294a897..b82b95bdcc96 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -10,6 +10,7 @@
#include "intel_context.h"
#include "intel_engine.h"
+#include "intel_engine_pm.h"
static struct i915_global_context {
struct i915_global base;
@@ -269,19 +270,11 @@ int __init i915_global_context_init(void)
int __intel_context_enter(struct intel_context *ce)
{
- struct drm_i915_private *i915 = ce->gem_context->i915;
-
- if (!i915->gt.active_requests++)
- i915_gem_unpark(i915);
-
+ intel_engine_pm_get(ce->engine);
return 0;
}
void __intel_context_exit(struct intel_context *ce)
{
- struct drm_i915_private *i915 = ce->gem_context->i915;
-
- GEM_BUG_ON(!i915->gt.active_requests);
- if (!--i915->gt.active_requests)
- i915_gem_park(i915);
+ intel_engine_pm_put(ce->engine);
}
diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h
index 362552226a72..e458b8f26cfa 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine.h
@@ -468,9 +468,6 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
void intel_engine_lost_context(struct intel_engine_cs *engine);
-void intel_engines_park(struct drm_i915_private *i915);
-void intel_engines_unpark(struct drm_i915_private *i915);
-
void intel_engines_reset_default_submission(struct drm_i915_private *i915);
unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index 2e2352408c62..75547770a96c 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -27,6 +27,7 @@
#include "i915_drv.h"
#include "intel_engine.h"
+#include "intel_engine_pm.h"
#include "intel_lrc.h"
#include "intel_reset.h"
@@ -588,6 +589,7 @@ int intel_engine_setup_common(struct intel_engine_cs *engine)
intel_engine_init_hangcheck(engine);
intel_engine_init_batch_pool(engine);
intel_engine_init_cmd_parser(engine);
+ intel_engine_init__pm(engine);
return 0;
@@ -1153,85 +1155,6 @@ void intel_engines_sanitize(struct drm_i915_private *i915, bool force)
intel_engine_reset(engine, false);
}
-/**
- * intel_engines_park: called when the GT is transitioning from busy->idle
- * @i915: the i915 device
- *
- * The GT is now idle and about to go to sleep (maybe never to wake again?).
- * Time for us to tidy and put away our toys (release resources back to the
- * system).
- */
-void intel_engines_park(struct drm_i915_private *i915)
-{
- struct intel_engine_cs *engine;
- enum intel_engine_id id;
-
- for_each_engine(engine, i915, id) {
- /* Flush the residual irq tasklets first. */
- intel_engine_disarm_breadcrumbs(engine);
- tasklet_kill(&engine->execlists.tasklet);
-
- /*
- * We are committed now to parking the engines, make sure there
- * will be no more interrupts arriving later and the engines
- * are truly idle.
- */
- if (wait_for(intel_engine_is_idle(engine), 10)) {
- struct drm_printer p = drm_debug_printer(__func__);
-
- dev_err(i915->drm.dev,
- "%s is not idle before parking\n",
- engine->name);
- intel_engine_dump(engine, &p, NULL);
- }
-
- /* Must be reset upon idling, or we may miss the busy wakeup. */
- GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
-
- if (engine->park)
- engine->park(engine);
-
- if (engine->pinned_default_state) {
- i915_gem_object_unpin_map(engine->default_state);
- engine->pinned_default_state = NULL;
- }
-
- i915_gem_batch_pool_fini(&engine->batch_pool);
- engine->execlists.no_priolist = false;
- }
-
- i915->gt.active_engines = 0;
-}
-
-/**
- * intel_engines_unpark: called when the GT is transitioning from idle->busy
- * @i915: the i915 device
- *
- * The GT was idle and now about to fire up with some new user requests.
- */
-void intel_engines_unpark(struct drm_i915_private *i915)
-{
- struct intel_engine_cs *engine;
- enum intel_engine_id id;
-
- for_each_engine(engine, i915, id) {
- void *map;
-
- /* Pin the default state for fast resets from atomic context. */
- map = NULL;
- if (engine->default_state)
- map = i915_gem_object_pin_map(engine->default_state,
- I915_MAP_WB);
- if (!IS_ERR_OR_NULL(map))
- engine->pinned_default_state = map;
-
- if (engine->unpark)
- engine->unpark(engine);
-
- intel_engine_init_hangcheck(engine);
- }
-}
-
/**
* intel_engine_lost_context: called when the GPU is reset into unknown state
* @engine: the engine
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
new file mode 100644
index 000000000000..cfba29826004
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -0,0 +1,103 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#include "i915_drv.h"
+
+#include "intel_engine.h"
+#include "intel_engine_pm.h"
+#include "intel_gt_pm.h"
+
+static bool __intel_engine_pm_get(struct intel_wakeref_count *wc)
+{
+ struct intel_engine_cs *engine =
+ container_of(wc, typeof(*engine), wakeref);
+ void *map;
+
+ intel_gt_pm_get(engine->i915);
+
+ /* Pin the default state for fast resets from atomic context. */
+ map = NULL;
+ if (engine->default_state)
+ map = i915_gem_object_pin_map(engine->default_state,
+ I915_MAP_WB);
+ if (!IS_ERR_OR_NULL(map))
+ engine->pinned_default_state = map;
+
+ if (engine->unpark)
+ engine->unpark(engine);
+
+ intel_engine_init_hangcheck(engine);
+ return true;
+}
+
+void intel_engine_pm_get(struct intel_engine_cs *engine)
+{
+ intel_wakeref_get_once(engine->i915,
+ &engine->wakeref,
+ __intel_engine_pm_get);
+}
+
+static bool switch_to_kernel_context(struct intel_engine_cs *engine)
+{
+ struct i915_request *rq;
+
+ rq = i915_request_alloc(engine, engine->kernel_context->gem_context);
+ if (IS_ERR(rq))
+ return false;
+
+ i915_request_add(rq);
+ return true;
+}
+
+static bool __intel_engine_pm_put(struct intel_wakeref_count *wc)
+{
+ struct intel_engine_cs *engine =
+ container_of(wc, typeof(*engine), wakeref);
+
+ if (engine->wakeref_serial != engine->serial) {
+ engine->wakeref_serial = engine->serial + 1;
+ atomic_inc(&wc->count);
+
+ if (switch_to_kernel_context(engine)) {
+ atomic_dec(&wc->count);
+ return false;
+ }
+
+ engine->wakeref_serial--;
+ if (!atomic_dec_and_test(&wc->count))
+ return false;
+ }
+
+ intel_engine_disarm_breadcrumbs(engine);
+
+ /* Must be reset upon idling, or we may miss the busy wakeup. */
+ GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
+
+ if (engine->park)
+ engine->park(engine);
+
+ if (engine->pinned_default_state) {
+ i915_gem_object_unpin_map(engine->default_state);
+ engine->pinned_default_state = NULL;
+ }
+
+ engine->execlists.no_priolist = false;
+
+ intel_gt_pm_put(engine->i915);
+ return true;
+}
+
+void intel_engine_pm_put(struct intel_engine_cs *engine)
+{
+ intel_wakeref_put_once(engine->i915,
+ &engine->wakeref,
+ __intel_engine_pm_put);
+}
+
+void intel_engine_init__pm(struct intel_engine_cs *engine)
+{
+ intel_wakeref_init(&engine->wakeref, INTEL_WAKEREF_ENGINE);
+}
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.h b/drivers/gpu/drm/i915/gt/intel_engine_pm.h
new file mode 100644
index 000000000000..d984df98053f
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.h
@@ -0,0 +1,17 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef INTEL_ENGINE_PM_H
+#define INTEL_ENGINE_PM_H
+
+struct intel_engine_cs;
+
+void intel_engine_pm_get(struct intel_engine_cs *engine);
+void intel_engine_pm_put(struct intel_engine_cs *engine);
+
+void intel_engine_init__pm(struct intel_engine_cs *engine);
+
+#endif /* INTEL_ENGINE_PM_H */
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 51faf6d40ba5..d1b3eb1d3bc3 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -20,6 +20,7 @@
#include "i915_selftest.h"
#include "i915_timeline_types.h"
#include "intel_sseu.h"
+#include "intel_wakeref.h"
#include "intel_workarounds_types.h"
#define I915_MAX_SLICES 3
@@ -282,6 +283,10 @@ struct intel_engine_cs {
struct intel_context *kernel_context; /* pinned */
struct intel_context *preempt_context; /* pinned; optional */
+ unsigned long serial;
+
+ unsigned long wakeref_serial;
+ struct intel_wakeref_count wakeref;
struct drm_i915_gem_object *default_state;
void *pinned_default_state;
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
new file mode 100644
index 000000000000..e3b7b4c9ca52
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
@@ -0,0 +1,82 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "intel_gt_pm.h"
+#include "intel_wakeref.h"
+
+static void pm_notify(struct drm_i915_private *i915, int state)
+{
+ blocking_notifier_call_chain(&i915->gt.pm_notifications,
+ state, i915);
+}
+
+static bool __intel_gt_pm_get_once(struct intel_wakeref_count *wc)
+{
+ struct drm_i915_private *i915 =
+ container_of(wc, typeof(*i915), gt.wakeref);
+
+ /*
+ * It seems that the DMC likes to transition between the DC states a lot
+ * when there are no connected displays (no active power domains) during
+ * command submission.
+ *
+ * This activity has negative impact on the performance of the chip with
+ * huge latencies observed in the interrupt handler and elsewhere.
+ *
+ * Work around it by grabbing a GT IRQ power domain whilst there is any
+ * GT activity, preventing any DC state transitions.
+ */
+ i915->gt.awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
+ GEM_BUG_ON(!i915->gt.awake);
+
+ intel_enable_gt_powersave(i915);
+
+ i915_update_gfx_val(i915);
+ if (INTEL_GEN(i915) >= 6)
+ gen6_rps_busy(i915);
+
+ i915_pmu_gt_unparked(i915);
+
+ i915_queue_hangcheck(i915);
+
+ pm_notify(i915, INTEL_GT_UNPARK);
+ return true;
+}
+
+void intel_gt_pm_get(struct drm_i915_private *i915)
+{
+ intel_wakeref_get_once(i915, &i915->gt.wakeref, __intel_gt_pm_get_once);
+}
+
+static bool __intel_gt_pm_put_once(struct intel_wakeref_count *wc)
+{
+ struct drm_i915_private *i915 =
+ container_of(wc, typeof(*i915), gt.wakeref);
+ intel_wakeref_t wakeref = fetch_and_zero(&i915->gt.awake);
+
+ pm_notify(i915, INTEL_GT_PARK);
+
+ i915_pmu_gt_parked(i915);
+ if (INTEL_GEN(i915) >= 6)
+ gen6_rps_idle(i915);
+
+ GEM_BUG_ON(!wakeref);
+ intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref);
+
+ return true;
+}
+
+void intel_gt_pm_put(struct drm_i915_private *i915)
+{
+ intel_wakeref_put_once(i915, &i915->gt.wakeref, __intel_gt_pm_put_once);
+}
+
+void intel_gt_pm_init(struct drm_i915_private *i915)
+{
+ intel_wakeref_init(&i915->gt.wakeref, INTEL_WAKEREF_GT);
+ BLOCKING_INIT_NOTIFIER_HEAD(&i915->gt.pm_notifications);
+}
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
new file mode 100644
index 000000000000..245637fb69d2
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef INTEL_GT_PM_H
+#define INTEL_GT_PM_H
+
+struct drm_i915_private;
+
+enum {
+ INTEL_GT_UNPARK,
+ INTEL_GT_PARK,
+};
+
+void intel_gt_pm_get(struct drm_i915_private *i915);
+void intel_gt_pm_put(struct drm_i915_private *i915);
+
+void intel_gt_pm_init(struct drm_i915_private *i915);
+
+#endif /* INTEL_GT_PM_H */
diff --git a/drivers/gpu/drm/i915/gt/intel_hangcheck.c b/drivers/gpu/drm/i915/gt/intel_hangcheck.c
index 3053a706a561..e5eaa06fe74d 100644
--- a/drivers/gpu/drm/i915/gt/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/intel_hangcheck.c
@@ -256,6 +256,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
struct intel_engine_cs *engine;
enum intel_engine_id id;
unsigned int hung = 0, stuck = 0, wedged = 0;
+ intel_wakeref_t wakeref;
if (!i915_modparams.enable_hangcheck)
return;
@@ -266,6 +267,10 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
if (i915_terminally_wedged(dev_priv))
return;
+ wakeref = intel_runtime_pm_get_if_in_use(dev_priv);
+ if (!wakeref)
+ return;
+
/* As enabling the GPU requires fairly extensive mmio access,
* periodically arm the mmio checker to see if we are triggering
* any invalid access.
@@ -313,6 +318,8 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
if (hung)
hangcheck_declare_hang(dev_priv, hung, stuck);
+ intel_runtime_pm_put(dev_priv, wakeref);
+
/* Reset timer in case GPU hangs without another request being added */
i915_queue_hangcheck(dev_priv);
}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 921cf30763e3..049ab7745505 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -2033,8 +2033,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
}
seq_printf(m, "RPS enabled? %d\n", rps->enabled);
- seq_printf(m, "GPU busy? %s [%d requests]\n",
- yesno(dev_priv->gt.awake), dev_priv->gt.active_requests);
+ seq_printf(m, "GPU busy? %s\n", yesno(dev_priv->gt.awake));
seq_printf(m, "Boosts outstanding? %d\n",
atomic_read(&rps->num_waiters));
seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive));
@@ -2053,9 +2052,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts));
- if (INTEL_GEN(dev_priv) >= 6 &&
- rps->enabled &&
- dev_priv->gt.active_requests) {
+ if (INTEL_GEN(dev_priv) >= 6 && rps->enabled && dev_priv->gt.awake) {
u32 rpup, rpupei;
u32 rpdown, rpdownei;
@@ -3085,8 +3082,6 @@ static int i915_engine_info(struct seq_file *m, void *unused)
wakeref = intel_runtime_pm_get(dev_priv);
seq_printf(m, "GT awake? %s\n", yesno(dev_priv->gt.awake));
- seq_printf(m, "Global active requests: %d\n",
- dev_priv->gt.active_requests);
seq_printf(m, "CS timestamp frequency: %u kHz\n",
RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz);
@@ -3932,8 +3927,7 @@ i915_drop_caches_set(void *data, u64 val)
if (val & DROP_IDLE) {
do {
- if (READ_ONCE(i915->gt.active_requests))
- flush_delayed_work(&i915->gem.retire_work);
+ flush_delayed_work(&i915->gem.retire_work);
drain_delayed_work(&i915->gem.idle_work);
} while (READ_ONCE(i915->gt.awake));
}
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 0aaaa8a281c2..f6c8042fe4da 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2013,10 +2013,10 @@ struct drm_i915_private {
struct list_head hwsp_free_list;
} timelines;
- intel_engine_mask_t active_engines;
struct list_head active_rings;
struct list_head closed_vma;
- u32 active_requests;
+
+ struct intel_wakeref_count wakeref;
/**
* Is the GPU currently considered idle, or busy executing
@@ -2027,12 +2027,16 @@ struct drm_i915_private {
*/
intel_wakeref_t awake;
+ struct blocking_notifier_head pm_notifications;
+
ktime_t last_init_time;
struct i915_vma *scratch;
} gt;
struct {
+ struct notifier_block pm_notifier;
+
/**
* We leave the user IRQ off as much as possible,
* but this means that requests will finish and never
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 48e7cbaf4e09..e2d11d9a063e 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -39,6 +39,7 @@
#include <linux/dma-buf.h>
#include <linux/mman.h>
+#include "gt/intel_gt_pm.h"
#include "gt/intel_mocs.h"
#include "gt/intel_reset.h"
#include "gt/intel_workarounds.h"
@@ -2910,9 +2911,6 @@ wait_for_timelines(struct drm_i915_private *i915,
struct i915_gt_timelines *gt = &i915->gt.timelines;
struct i915_timeline *tl;
- if (!READ_ONCE(i915->gt.active_requests))
- return timeout;
-
mutex_lock(>->mutex);
list_for_each_entry(tl, >->active_list, link) {
struct i915_request *rq;
@@ -4758,6 +4756,8 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv)
{
int err;
+ intel_gt_pm_init(dev_priv);
+
INIT_LIST_HEAD(&dev_priv->gt.active_rings);
INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 9074eb1e843f..67f8a4a807a0 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -75,9 +75,6 @@ struct drm_i915_private;
#define I915_GEM_IDLE_TIMEOUT (HZ / 5)
-void i915_gem_park(struct drm_i915_private *i915);
-void i915_gem_unpark(struct drm_i915_private *i915);
-
static inline void __tasklet_disable_sync_once(struct tasklet_struct *t)
{
if (!atomic_fetch_inc(&t->count))
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 060f5903544a..fa92b4ab821f 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -36,15 +36,8 @@ I915_SELFTEST_DECLARE(static struct igt_evict_ctl {
bool fail_if_busy:1;
} igt_evict_ctl;)
-static bool ggtt_is_idle(struct drm_i915_private *i915)
-{
- return !i915->gt.active_requests;
-}
-
static int ggtt_flush(struct drm_i915_private *i915)
{
- int err;
-
/*
* Not everything in the GGTT is tracked via vma (otherwise we
* could evict as required with minimal stalling) so we are forced
@@ -52,19 +45,10 @@ static int ggtt_flush(struct drm_i915_private *i915)
* the hopes that we can then remove contexts and the like only
* bound by their active reference.
*/
- err = i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines);
- if (err)
- return err;
-
- err = i915_gem_wait_for_idle(i915,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
- MAX_SCHEDULE_TIMEOUT);
- if (err)
- return err;
-
- GEM_BUG_ON(!ggtt_is_idle(i915));
- return 0;
+ return i915_gem_wait_for_idle(i915,
+ I915_WAIT_INTERRUPTIBLE |
+ I915_WAIT_LOCKED,
+ MAX_SCHEDULE_TIMEOUT);
}
static bool
@@ -222,24 +206,17 @@ i915_gem_evict_something(struct i915_address_space *vm,
* us a termination condition, when the last retired context is
* the kernel's there is no more we can evict.
*/
- if (!ggtt_is_idle(dev_priv)) {
- if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy))
- return -EBUSY;
+ if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy))
+ return -EBUSY;
- ret = ggtt_flush(dev_priv);
- if (ret)
- return ret;
+ ret = ggtt_flush(dev_priv);
+ if (ret)
+ return ret;
- cond_resched();
- goto search_again;
- }
+ cond_resched();
- /*
- * If we still have pending pageflip completions, drop
- * back to userspace to give our workqueues time to
- * acquire our locks and unpin the old scanouts.
- */
- return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC;
+ flags &= ~PIN_NONBLOCK;
+ goto search_again;
found:
/* drm_mm doesn't allow any other other operations while
diff --git a/drivers/gpu/drm/i915/i915_gem_pm.c b/drivers/gpu/drm/i915/i915_gem_pm.c
index 49480b7a6d25..973d7d6a49a7 100644
--- a/drivers/gpu/drm/i915/i915_gem_pm.c
+++ b/drivers/gpu/drm/i915/i915_gem_pm.c
@@ -4,22 +4,18 @@
* Copyright © 2019 Intel Corporation
*/
+#include "gt/intel_gt_pm.h"
+
#include "i915_drv.h"
#include "i915_gem_pm.h"
#include "i915_globals.h"
-static void __i915_gem_park(struct drm_i915_private *i915)
+static void i915_gem_park(struct drm_i915_private *i915)
{
- intel_wakeref_t wakeref;
-
- GEM_TRACE("\n");
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
lockdep_assert_held(&i915->drm.struct_mutex);
- GEM_BUG_ON(i915->gt.active_requests);
- GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
-
- if (!i915->gt.awake)
- return;
/*
* Be paranoid and flush a concurrent interrupt to make sure
@@ -34,202 +30,125 @@ static void __i915_gem_park(struct drm_i915_private *i915)
*/
synchronize_irq(i915->drm.irq);
- intel_engines_park(i915);
- i915_timelines_park(i915);
-
- i915_pmu_gt_parked(i915);
- i915_vma_parked(i915);
-
- wakeref = fetch_and_zero(&i915->gt.awake);
- GEM_BUG_ON(!wakeref);
+ for_each_engine(engine, i915, id) {
+ tasklet_kill(&engine->execlists.tasklet);
- if (INTEL_GEN(i915) >= 6)
- gen6_rps_idle(i915);
+ /*
+ * We are committed now to parking the engines, make sure there
+ * will be no more interrupts arriving later and the engines
+ * are truly idle.
+ */
+ if (wait_for(intel_engine_is_idle(engine), 10)) {
+ struct drm_printer p = drm_debug_printer(__func__);
- intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref);
-
- i915_globals_park();
-}
-
-static bool switch_to_kernel_context_sync(struct drm_i915_private *i915,
- unsigned long mask)
-{
- bool result = true;
-
- /*
- * Even if we fail to switch, give whatever is running a small chance
- * to save itself before we report the failure. Yes, this may be a
- * false positive due to e.g. ENOMEM, caveat emptor!
- */
- if (i915_gem_switch_to_kernel_context(i915, mask))
- result = false;
-
- if (i915_gem_wait_for_idle(i915,
- I915_WAIT_LOCKED |
- I915_WAIT_FOR_IDLE_BOOST,
- I915_GEM_IDLE_TIMEOUT))
- result = false;
-
- if (!result) {
- if (i915_modparams.reset) { /* XXX hide warning from gem_eio */
dev_err(i915->drm.dev,
- "Failed to idle engines, declaring wedged!\n");
- GEM_TRACE_DUMP();
+ "%s is not idle before parking\n",
+ engine->name);
+ intel_engine_dump(engine, &p, NULL);
}
- /* Forcibly cancel outstanding work and leave the gpu quiet. */
- i915_gem_set_wedged(i915);
+ i915_gem_batch_pool_fini(&engine->batch_pool);
}
- i915_retire_requests(i915); /* ensure we flush after wedging */
- return result;
+ i915_timelines_park(i915);
+ i915_vma_parked(i915);
+
+ i915_globals_park();
}
static void idle_work_handler(struct work_struct *work)
{
struct drm_i915_private *i915 =
container_of(work, typeof(*i915), gem.idle_work.work);
- bool rearm_hangcheck;
-
- if (!READ_ONCE(i915->gt.awake))
- return;
-
- if (READ_ONCE(i915->gt.active_requests))
- return;
-
- rearm_hangcheck =
- cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
if (!mutex_trylock(&i915->drm.struct_mutex)) {
/* Currently busy, come back later */
mod_delayed_work(i915->wq,
&i915->gem.idle_work,
msecs_to_jiffies(50));
- goto out_rearm;
+ return;
}
- /*
- * Flush out the last user context, leaving only the pinned
- * kernel context resident. Should anything unfortunate happen
- * while we are idle (such as the GPU being power cycled), no users
- * will be harmed.
- */
- if (!work_pending(&i915->gem.idle_work.work) &&
- !i915->gt.active_requests) {
- ++i915->gt.active_requests; /* don't requeue idle */
-
- switch_to_kernel_context_sync(i915, i915->gt.active_engines);
-
- if (!--i915->gt.active_requests) {
- __i915_gem_park(i915);
- rearm_hangcheck = false;
- }
- }
+ i915_gem_park(i915);
mutex_unlock(&i915->drm.struct_mutex);
-
-out_rearm:
- if (rearm_hangcheck) {
- GEM_BUG_ON(!i915->gt.awake);
- i915_queue_hangcheck(i915);
- }
}
static void retire_work_handler(struct work_struct *work)
{
- struct drm_i915_private *dev_priv =
- container_of(work, typeof(*dev_priv), gem.retire_work.work);
- struct drm_device *dev = &dev_priv->drm;
+ struct drm_i915_private *i915 =
+ container_of(work, typeof(*i915), gem.retire_work.work);
+ struct drm_device *dev = &i915->drm;
+ bool repeat = true;
/* Come back later if the device is busy... */
if (mutex_trylock(&dev->struct_mutex)) {
- i915_retire_requests(dev_priv);
+ repeat = i915_retire_requests(i915);
mutex_unlock(&dev->struct_mutex);
}
- /*
- * Keep the retire handler running until we are finally idle.
- * We do not need to do this test under locking as in the worst-case
- * we queue the retire worker once too often.
- */
- if (READ_ONCE(dev_priv->gt.awake))
- queue_delayed_work(dev_priv->wq,
- &dev_priv->gem.retire_work,
+ if (repeat)
+ queue_delayed_work(i915->wq,
+ &i915->gem.retire_work,
round_jiffies_up_relative(HZ));
}
-
-void i915_gem_park(struct drm_i915_private *i915)
+static int pm_notifier(struct notifier_block *nb,
+ unsigned long action,
+ void *data)
{
- GEM_TRACE("\n");
+ struct drm_i915_private *i915 =
+ container_of(nb, typeof(*i915), gem.pm_notifier);
- lockdep_assert_held(&i915->drm.struct_mutex);
- GEM_BUG_ON(i915->gt.active_requests);
+ switch (action) {
+ case INTEL_GT_UNPARK:
+ i915_globals_unpark();
+ queue_delayed_work(i915->wq,
+ &i915->gem.retire_work,
+ round_jiffies_up_relative(HZ));
+ break;
- if (!i915->gt.awake)
- return;
+ case INTEL_GT_PARK:
+ mod_delayed_work(i915->wq,
+ &i915->gem.idle_work,
+ msecs_to_jiffies(100));
+ break;
+ }
- /* Defer the actual call to __i915_gem_park() to prevent ping-pongs */
- mod_delayed_work(i915->wq, &i915->gem.idle_work, msecs_to_jiffies(100));
+ return NOTIFY_OK;
}
-void i915_gem_unpark(struct drm_i915_private *i915)
+static bool switch_to_kernel_context_sync(struct drm_i915_private *i915)
{
- GEM_TRACE("\n");
-
- lockdep_assert_held(&i915->drm.struct_mutex);
- GEM_BUG_ON(!i915->gt.active_requests);
- assert_rpm_wakelock_held(i915);
-
- if (i915->gt.awake)
- return;
-
- /*
- * It seems that the DMC likes to transition between the DC states a lot
- * when there are no connected displays (no active power domains) during
- * command submission.
- *
- * This activity has negative impact on the performance of the chip with
- * huge latencies observed in the interrupt handler and elsewhere.
- *
- * Work around it by grabbing a GT IRQ power domain whilst there is any
- * GT activity, preventing any DC state transitions.
- */
- i915->gt.awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
- GEM_BUG_ON(!i915->gt.awake);
-
- i915_globals_unpark();
-
- intel_enable_gt_powersave(i915);
- i915_update_gfx_val(i915);
- if (INTEL_GEN(i915) >= 6)
- gen6_rps_busy(i915);
- i915_pmu_gt_unparked(i915);
-
- intel_engines_unpark(i915);
+ bool result = true;
+ int flush = 1;
+
+ do {
+ if (i915_gem_wait_for_idle(i915,
+ I915_WAIT_LOCKED |
+ I915_WAIT_FOR_IDLE_BOOST,
+ I915_GEM_IDLE_TIMEOUT)) {
+ if (i915_modparams.reset) { /* XXX hide warning from gem_eio */
+ dev_err(i915->drm.dev,
+ "Failed to idle engines, declaring wedged!\n");
+ GEM_TRACE_DUMP();
+ }
+
+ /* Forcibly cancel outstanding work and leave the gpu quiet. */
+ i915_gem_set_wedged(i915);
+
+ result = false;
+ }
- i915_queue_hangcheck(i915);
+ i915_retire_requests(i915); /* ensure we flush after wedging */
+ } while (result && flush--);
- queue_delayed_work(i915->wq,
- &i915->gem.retire_work,
- round_jiffies_up_relative(HZ));
+ return result;
}
bool i915_gem_load_power_context(struct drm_i915_private *i915)
{
- /* Force loading the kernel context on all engines */
- if (!switch_to_kernel_context_sync(i915, ALL_ENGINES))
- return false;
-
- /*
- * Immediately park the GPU so that we enable powersaving and
- * treat it as idle. The next time we issue a request, we will
- * unpark and start using the engine->pinned_default_state, otherwise
- * it is in limbo and an early reset may fail.
- */
- __i915_gem_park(i915);
-
- return true;
+ return switch_to_kernel_context_sync(i915);
}
void i915_gem_suspend(struct drm_i915_private *i915)
@@ -253,7 +172,7 @@ void i915_gem_suspend(struct drm_i915_private *i915)
* state. Fortunately, the kernel_context is disposable and we do
* not rely on its state.
*/
- switch_to_kernel_context_sync(i915, i915->gt.active_engines);
+ switch_to_kernel_context_sync(i915);
mutex_unlock(&i915->drm.struct_mutex);
i915_reset_flush(i915);
@@ -363,4 +282,8 @@ void i915_gem_init__pm(struct drm_i915_private *i915)
{
INIT_DELAYED_WORK(&i915->gem.idle_work, idle_work_handler);
INIT_DELAYED_WORK(&i915->gem.retire_work, retire_work_handler);
+
+ i915->gem.pm_notifier.notifier_call = pm_notifier;
+ blocking_notifier_chain_register(&i915->gt.pm_notifications,
+ &i915->gem.pm_notifier);
}
diff --git a/drivers/gpu/drm/i915/i915_gem_pm.h b/drivers/gpu/drm/i915/i915_gem_pm.h
index 52f65e3f06b5..6f7d5d11ac3b 100644
--- a/drivers/gpu/drm/i915/i915_gem_pm.h
+++ b/drivers/gpu/drm/i915/i915_gem_pm.h
@@ -17,9 +17,6 @@ void i915_gem_init__pm(struct drm_i915_private *i915);
bool i915_gem_load_power_context(struct drm_i915_private *i915);
void i915_gem_resume(struct drm_i915_private *i915);
-void i915_gem_unpark(struct drm_i915_private *i915);
-void i915_gem_park(struct drm_i915_private *i915);
-
void i915_gem_idle_work_handler(struct work_struct *work);
void i915_gem_suspend(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index ab170affd266..d45972b3ddc2 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -428,6 +428,8 @@ void __i915_request_submit(struct i915_request *request)
/* Transfer from per-context onto the global per-engine timeline */
move_to_timeline(request, &engine->timeline);
+ engine->serial++;
+
trace_i915_request_execute(request);
}
@@ -1092,7 +1094,6 @@ void i915_request_add(struct i915_request *request)
list_add_tail(&request->ring_link, &ring->request_list);
if (list_is_first(&request->ring_link, &ring->request_list))
list_add(&ring->active_link, &request->i915->gt.active_rings);
- request->i915->gt.active_engines |= request->engine->mask;
request->emitted_jiffies = jiffies;
/*
@@ -1347,21 +1348,20 @@ long i915_request_wait(struct i915_request *rq,
return timeout;
}
-void i915_retire_requests(struct drm_i915_private *i915)
+bool i915_retire_requests(struct drm_i915_private *i915)
{
struct intel_ring *ring, *tmp;
lockdep_assert_held(&i915->drm.struct_mutex);
- if (!i915->gt.active_requests)
- return;
-
list_for_each_entry_safe(ring, tmp,
&i915->gt.active_rings, active_link) {
intel_ring_get(ring); /* last rq holds reference! */
ring_retire_requests(ring);
intel_ring_put(ring);
}
+
+ return !list_empty(&i915->gt.active_rings);
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 875be6f71412..c02c539ffd98 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -417,6 +417,6 @@ static inline void i915_request_mark_complete(struct i915_request *rq)
rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */
}
-void i915_retire_requests(struct drm_i915_private *i915);
+bool i915_retire_requests(struct drm_i915_private *i915);
#endif /* I915_REQUEST_H */
diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h
index 2c414453489c..71460aa9a357 100644
--- a/drivers/gpu/drm/i915/intel_wakeref.h
+++ b/drivers/gpu/drm/i915/intel_wakeref.h
@@ -22,7 +22,8 @@ struct intel_wakeref_count {
};
enum intel_wakeref_class {
- INTEL_WAKEREF_NONE = 0,
+ INTEL_WAKEREF_ENGINE = 0,
+ INTEL_WAKEREF_GT,
__INTEL_WAKEREF_NUM_CLASSES
};
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 9d646fa1b74e..0078a1520345 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -1633,8 +1633,7 @@ static int __igt_switch_to_kernel_context(struct drm_i915_private *i915,
}
}
- err = i915_gem_switch_to_kernel_context(i915,
- i915->gt.active_engines);
+ err = i915_gem_switch_to_kernel_context(i915, engines);
if (err)
return err;
@@ -1646,15 +1645,6 @@ static int __igt_switch_to_kernel_context(struct drm_i915_private *i915,
return err;
}
- if (i915->gt.active_requests) {
- pr_err("%d active requests remain after switching to kernel context, pass %d (%s) on %s engine%s\n",
- i915->gt.active_requests,
- pass, from_idle ? "idle" : "busy",
- __engine_name(i915, engines),
- is_power_of_2(engines) ? "" : "s");
- return -EINVAL;
- }
-
/* XXX Bonus points for proving we are the kernel context! */
mutex_unlock(&i915->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 12fc53c694a6..d8bdd1d53688 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -24,6 +24,7 @@
#include "../i915_selftest.h"
+#include "igt_flush_test.h"
#include "mock_gem_device.h"
#include "huge_gem_object.h"
@@ -505,17 +506,21 @@ static void disable_retire_worker(struct drm_i915_private *i915)
{
i915_gem_shrinker_unregister(i915);
- mutex_lock(&i915->drm.struct_mutex);
- if (!i915->gt.active_requests++) {
- intel_wakeref_t wakeref;
+ cancel_delayed_work_sync(&i915->gem.retire_work);
+ cancel_delayed_work_sync(&i915->gem.idle_work);
- with_intel_runtime_pm(i915, wakeref)
- i915_gem_unpark(i915);
- }
+ atomic_inc(&i915->gt.wakeref.count);
+}
+
+static void restore_retire_worker(struct drm_i915_private *i915)
+{
+ atomic_dec(&i915->gt.wakeref.count);
+
+ mutex_lock(&i915->drm.struct_mutex);
+ igt_flush_test(i915, I915_WAIT_LOCKED);
mutex_unlock(&i915->drm.struct_mutex);
- cancel_delayed_work_sync(&i915->gem.retire_work);
- cancel_delayed_work_sync(&i915->gem.idle_work);
+ i915_gem_shrinker_register(i915);
}
static int igt_mmap_offset_exhaustion(void *arg)
@@ -615,13 +620,7 @@ static int igt_mmap_offset_exhaustion(void *arg)
out:
drm_mm_remove_node(&resv);
out_park:
- mutex_lock(&i915->drm.struct_mutex);
- if (--i915->gt.active_requests)
- queue_delayed_work(i915->wq, &i915->gem.retire_work, 0);
- else
- queue_delayed_work(i915->wq, &i915->gem.idle_work, 0);
- mutex_unlock(&i915->drm.struct_mutex);
- i915_gem_shrinker_register(i915);
+ restore_retire_worker(i915);
return err;
err_obj:
i915_gem_object_put(obj);
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
index 94aee4071a66..263d73f66eb1 100644
--- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
@@ -14,7 +14,7 @@ int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
cond_resched();
if (flags & I915_WAIT_LOCKED &&
- i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines)) {
+ i915_gem_switch_to_kernel_context(i915, ALL_ENGINES)) {
pr_err("Failed to switch back to kernel context; declaring wedged\n");
i915_gem_set_wedged(i915);
}
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index fb677b4019a0..59f719846f81 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -45,7 +45,6 @@ void mock_device_flush(struct drm_i915_private *i915)
mock_engine_flush(engine);
i915_retire_requests(i915);
- GEM_BUG_ON(i915->gt.active_requests);
}
static void mock_device_release(struct drm_device *dev)
@@ -110,10 +109,6 @@ static void mock_retire_work_handler(struct work_struct *work)
static void mock_idle_work_handler(struct work_struct *work)
{
- struct drm_i915_private *i915 =
- container_of(work, typeof(*i915), gem.idle_work.work);
-
- i915->gt.active_engines = 0;
}
static int pm_domain_resume(struct device *dev)
--
2.20.1
More information about the Intel-gfx-trybot
mailing list