[PATCH 2/4] drm/i915/guc: Define CTB based TLB invalidation routines
Nirmoy Das
nirmoy.das at intel.com
Mon Sep 25 16:43:38 UTC 2023
Add routines to interface with GuC firmware for TLB invalidation.
Based on patches from Prathap Kumar Valsan.
Signed-off-by: Nirmoy Das <nirmoy.das at intel.com>
---
.../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h | 46 +++++++
drivers/gpu/drm/i915/gt/uc/intel_guc.c | 115 ++++++++++++++++++
drivers/gpu/drm/i915/gt/uc/intel_guc.h | 22 ++++
drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c | 20 ++-
drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h | 6 +
.../gpu/drm/i915/gt/uc/intel_guc_submission.c | 95 ++++++++++++++-
6 files changed, 299 insertions(+), 5 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h
index f359bef046e0..a19d5fd3ad58 100644
--- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h
@@ -138,6 +138,8 @@ enum intel_guc_action {
INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601,
INTEL_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507,
INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A,
+ INTEL_GUC_ACTION_TLB_INVALIDATION = 0x7000,
+ INTEL_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001,
INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002,
INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003,
INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004,
@@ -181,4 +183,48 @@ enum intel_guc_state_capture_event_status {
#define INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK 0x000000FF
+#define INTEL_GUC_TLB_INVAL_TYPE_MASK REG_GENMASK(7, 0)
+#define INTEL_GUC_TLB_INVAL_MODE_MASK REG_GENMASK(11, 8)
+/* Flush PPC or SMRO caches along with TLB invalidation request */
+#define INTEL_GUC_TLB_INVAL_FLUSH_CACHE REG_BIT(31)
+
+/**
+ * enum intel_guc_tlb_invalidation_type - type of TLB cache invalidation
+ *
+ * @INTEL_GUC_TLB_INVAL_FULL:
+ * Global TLB invalidation
+ * @INTEL_GUC_TLB_INVAL_GUC:
+ * Invalidate TLB on GuC itself
+ */
+enum intel_guc_tlb_invalidation_type {
+ INTEL_GUC_TLB_INVAL_FULL = 0x0,
+ INTEL_GUC_TLB_INVAL_GUC = 0x3,
+};
+
+/**
+ * enum intel_guc_tlb_inval_mode - define the mode for TLB cache invlidation
+ *
+ * @INTEL_GUC_TLB_INVAL_MODE_HEAVY: Heavy Invalidation Mode.
+ * The pipeline of the engine(s) for which the invalidation is targeted to is
+ * blocked, and all the in-flight transactions are guaranteed to be Globally
+ * Observed before completing the TLB invalidation
+ *
+ * @INTEL_GUC_TLB_INVAL_MODE_LITE: Light Invalidation Mode.
+ * TLBs of the targeted engine(s) are immediately invalidated.
+ * In-flight transactions are NOT guaranteed to be Globally Observed before
+ * completing TLB invalidation.
+ *
+ * Light Invalidation Mode is to be used only when
+ * it can be guaranteed (by SW) that the address translations remain invariant
+ * for the in-flight transactions across the TLB invalidation. In other words,
+ * this mode can be used when the TLB invalidation is intended to clear out the
+ * stale cached translations that are no longer in use. Light Invalidation Mode
+ * is much faster than the Heavy Invalidation Mode, as it does not wait for the
+ * in-flight transactions to be GOd.
+ */
+enum intel_guc_tlb_inval_mode {
+ INTEL_GUC_TLB_INVAL_MODE_HEAVY = 0x0,
+ INTEL_GUC_TLB_INVAL_MODE_LITE = 0x1,
+};
+
#endif /* _ABI_GUC_ACTIONS_ABI_H */
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
index 27df41c53b89..890feae899f3 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
@@ -881,6 +881,121 @@ int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value)
return __guc_self_cfg(guc, key, 2, value);
}
+static int guc_send_invalidate_tlb(struct intel_guc *guc, u32 type)
+{
+ struct intel_guc_tlb_wait _wq, *wq = &_wq;
+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
+ int err = 0;
+ u32 seqno;
+ u32 action[] = {
+ INTEL_GUC_ACTION_TLB_INVALIDATION,
+ 0,
+ REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_TYPE_MASK, type) |
+ REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_MODE_MASK,
+ INTEL_GUC_TLB_INVAL_MODE_HEAVY) |
+ INTEL_GUC_TLB_INVAL_FLUSH_CACHE,
+ };
+ u32 size = ARRAY_SIZE(action);
+
+ init_waitqueue_head(&_wq.wq);
+
+ if (xa_alloc_cyclic_irq(&guc->tlb_lookup, &seqno, wq,
+ xa_limit_32b, &guc->next_seqno,
+ GFP_ATOMIC | __GFP_NOWARN) < 0) {
+ /* Under severe memory pressure? Serialise TLB allocations */
+ xa_lock_irq(&guc->tlb_lookup);
+ wq = xa_load(&guc->tlb_lookup, guc->serial_slot);
+ wait_event_lock_irq(wq->wq,
+ !READ_ONCE(wq->status),
+ guc->tlb_lookup.xa_lock);
+ /*
+ * Update wq->status under lock to ensure only one waiter can
+ * issue the TLB invalidation command using the serial slot at a
+ * time. The condition is set to false before releasing the lock
+ * so that other caller continue to wait until woken up again.
+ */
+ wq->status = 1;
+ xa_unlock_irq(&guc->tlb_lookup);
+
+ seqno = guc->serial_slot;
+ }
+
+ action[1] = seqno;
+
+ add_wait_queue(&wq->wq, &wait);
+
+ err = intel_guc_send_busy_loop(guc, action, size, G2H_LEN_DW_INVALIDATE_TLB, true);
+ if (err) {
+ /*
+ * XXX: Failure of TLB invalidation is critical and would
+ * warrant a gt reset.
+ */
+ goto out;
+ }
+/*
+ * GuC has a timeout of 1ms for a TLB invalidation response from GAM. On a
+ * timeout GuC drops the request and has no mechanism to notify the host about
+ * the timeout. So keep a larger timeout that accounts for this individual
+ * timeout and max number of outstanding invalidation requests that can be
+ * queued in CT buffer.
+ */
+#define OUTSTANDING_GUC_TIMEOUT_PERIOD (HZ)
+ if (!wait_woken(&wait, TASK_UNINTERRUPTIBLE,
+ OUTSTANDING_GUC_TIMEOUT_PERIOD)) {
+ /*
+ * XXX: Failure of TLB invalidation is critical and would
+ * warrant a gt reset.
+ */
+ drm_err(&guc_to_gt(guc)->i915->drm,
+ "TLB invalidation response timed out for seqno %u\n", seqno);
+ err = -ETIME;
+ }
+out:
+ remove_wait_queue(&wq->wq, &wait);
+ if (seqno != guc->serial_slot)
+ xa_erase_irq(&guc->tlb_lookup, seqno);
+
+ return err;
+}
+
+/**
+ * intel_guc_invalidate_tlb_full() - Invalidate TLBs with GuC command
+ * @guc: the guc
+ *
+ * Use GuC to do a full TLB cache invalidation if supported.
+ *
+ * Return: 0 if successful, a negative errno code otherwise.
+ */
+int intel_guc_invalidate_tlb_full(struct intel_guc *guc)
+{
+ if (!INTEL_GUC_SUPPORTS_TLB_INVALIDATION(guc)) {
+ drm_err(&guc_to_gt(guc)->i915->drm,
+ "TLB full invalidation: Operation not supported in this platform!\n");
+ return 0;
+ }
+
+ return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_FULL);
+}
+
+/**
+ * intel_guc_invalidate_tlb_guc() - Invalidate TLBs of GuC
+ * @guc: the guc
+ *
+ * Use GuC to invalidate the TLB's of GuC itself.
+ *
+ * Return: 0 if successful, a negative errno code otherwise.
+ */
+int intel_guc_invalidate_tlb_guc(struct intel_guc *guc)
+{
+ if (!INTEL_GUC_SUPPORTS_TLB_INVALIDATION(guc)) {
+ drm_err(&guc_to_gt(guc)->i915->drm,
+ "GuC TLB invalidation: Operation not supported in this platform!\n");
+ return 0;
+ }
+
+ return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_GUC);
+}
+
/**
* intel_guc_load_status - dump information about GuC load status
* @guc: the GuC
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
index 6c392bad29c1..b5a1ab4145c9 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
@@ -79,6 +79,18 @@ struct intel_guc {
*/
atomic_t outstanding_submission_g2h;
+ /** @tlb_lookup: xarray to store all pending TLB invalidation requests */
+ struct xarray tlb_lookup;
+
+ /**
+ * @serial_slot: id to the initial waiter created in tlb_lookup,
+ * which is used only when failed to allocate new waiter.
+ */
+ u32 serial_slot;
+
+ /** @next_seqno: the next id (sequence no.) to allocate. */
+ u32 next_seqno;
+
/** @interrupts: pointers to GuC interrupt-managing functions. */
struct {
bool enabled;
@@ -296,6 +308,11 @@ struct intel_guc {
#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
#define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version)
+struct intel_guc_tlb_wait {
+ struct wait_queue_head wq;
+ u8 status;
+} __aligned(4);
+
static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
{
return container_of(log, struct intel_guc, log);
@@ -413,6 +430,9 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,
int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);
int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);
+int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
+int intel_guc_invalidate_tlb_full(struct intel_guc *guc);
+
static inline bool intel_guc_is_supported(struct intel_guc *guc)
{
return intel_uc_fw_is_supported(&guc->fw);
@@ -492,6 +512,8 @@ int intel_guc_error_capture_process_msg(struct intel_guc *guc,
const u32 *msg, u32 len);
int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action);
+int intel_guc_tlb_invalidation_done(struct intel_guc *guc, const u32 *hxg, u32 size);
+
struct intel_engine_cs *
intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
index 6e22af31513a..11486b556dac 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
@@ -1130,7 +1130,7 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *r
return 0;
}
-static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
+static bool ct_process_incoming_requests(struct intel_guc_ct *ct, struct list_head *incoming)
{
unsigned long flags;
struct ct_incoming_msg *request;
@@ -1138,11 +1138,11 @@ static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
int err;
spin_lock_irqsave(&ct->requests.lock, flags);
- request = list_first_entry_or_null(&ct->requests.incoming,
+ request = list_first_entry_or_null(incoming,
struct ct_incoming_msg, link);
if (request)
list_del(&request->link);
- done = !!list_empty(&ct->requests.incoming);
+ done = !!list_empty(incoming);
spin_unlock_irqrestore(&ct->requests.lock, flags);
if (!request)
@@ -1166,7 +1166,7 @@ static void ct_incoming_request_worker_func(struct work_struct *w)
bool done;
do {
- done = ct_process_incoming_requests(ct);
+ done = ct_process_incoming_requests(ct, &ct->requests.incoming);
} while (!done);
}
@@ -1186,14 +1186,26 @@ static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *requ
switch (action) {
case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
+ case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
g2h_release_space(ct, request->size);
}
+ /*
+ * Handle TLB invalidation response in interrupt context
+ * As TLB invalidation is needed to avoid leaking data, wait until
+ * TLB invalidation is completed before returning.
+ */
+ if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE) {
+ ct_free_msg(request);
+ return intel_guc_tlb_invalidation_done(ct_to_guc(ct), hxg, request->size);
+ }
+
spin_lock_irqsave(&ct->requests.lock, flags);
list_add_tail(&request->link, &ct->requests.incoming);
spin_unlock_irqrestore(&ct->requests.lock, flags);
queue_work(system_unbound_wq, &ct->requests.worker);
+
return 0;
}
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h
index b4d56eccfb1f..1141f151a54e 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h
@@ -22,6 +22,7 @@
/* Payload length only i.e. don't include G2H header length */
#define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET 2
#define G2H_LEN_DW_DEREGISTER_CONTEXT 1
+#define G2H_LEN_DW_INVALIDATE_TLB 1
#define GUC_CONTEXT_DISABLE 0
#define GUC_CONTEXT_ENABLE 1
@@ -498,4 +499,9 @@ enum intel_guc_recv_message {
INTEL_GUC_RECV_MSG_EXCEPTION = BIT(30),
};
+#define INTEL_GUC_SUPPORTS_TLB_INVALIDATION(__guc) \
+ ((intel_guc_ct_enabled(&(__guc)->ct)) && \
+ (intel_guc_submission_is_used(__guc)) && \
+ HAS_SELECTIVE_TLB_INVALIDATION(guc_to_gt(__guc)->i915))
+
#endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index cabdc645fcdd..40666bbc6d2f 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -1764,11 +1764,20 @@ static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t st
intel_context_put(parent);
}
+static void wake_up_tlb_invalidate(struct intel_guc_tlb_wait *wait)
+{
+ /* Barrier to ensure the store is observed by the woken thread */
+ smp_store_mb(wait->status, 0);
+ wake_up(&wait->wq);
+}
+
void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
{
+ struct intel_guc_tlb_wait *wait;
struct intel_context *ce;
unsigned long index;
unsigned long flags;
+ unsigned long i;
if (unlikely(!guc_submission_initialized(guc))) {
/* Reset called during driver load? GuC not yet initialised! */
@@ -1794,6 +1803,13 @@ void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stall
/* GuC is blown away, drop all references to contexts */
xa_destroy(&guc->context_lookup);
+
+ /*
+ * The full GT reset will have cleared the TLB caches and flushed the
+ * G2H message queue; we can release all the blocked waiters.
+ */
+ xa_for_each(&guc->tlb_lookup, i, wait)
+ wake_up_tlb_invalidate(wait);
}
static void guc_cancel_context_requests(struct intel_context *ce)
@@ -1916,6 +1932,41 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc)
static void destroyed_worker_func(struct work_struct *w);
static void reset_fail_worker_func(struct work_struct *w);
+static int init_tlb_lookup(struct intel_guc *guc)
+{
+ struct intel_guc_tlb_wait *wait;
+ int err;
+
+ xa_init_flags(&guc->tlb_lookup, XA_FLAGS_ALLOC);
+
+ wait = kzalloc(sizeof(*wait), GFP_KERNEL);
+ if (!wait)
+ return -ENOMEM;
+
+ init_waitqueue_head(&wait->wq);
+ err = xa_alloc_cyclic_irq(&guc->tlb_lookup, &guc->serial_slot, wait,
+ xa_limit_32b, &guc->next_seqno, GFP_KERNEL);
+ if (err == -ENOMEM) {
+ kfree(wait);
+ return err;
+ }
+
+ return 0;
+}
+
+static void fini_tlb_lookup(struct intel_guc *guc)
+{
+ struct intel_guc_tlb_wait *wait;
+
+ wait = xa_load(&guc->tlb_lookup, guc->serial_slot);
+ if (wait) {
+ GEM_BUG_ON(wait->status);
+ kfree(wait);
+ }
+
+ xa_destroy(&guc->tlb_lookup);
+}
+
/*
* Set up the memory resources to be shared with the GuC (via the GGTT)
* at firmware loading time.
@@ -1928,10 +1979,14 @@ int intel_guc_submission_init(struct intel_guc *guc)
if (guc->submission_initialized)
return 0;
+ ret = init_tlb_lookup(guc);
+ if (ret)
+ return ret;
+
if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 0, 0)) {
ret = guc_lrc_desc_pool_create_v69(guc);
if (ret)
- return ret;
+ goto tlb_fini;
}
guc->submission_state.guc_ids_bitmap =
@@ -1950,6 +2005,8 @@ int intel_guc_submission_init(struct intel_guc *guc)
destroy_pool:
guc_lrc_desc_pool_destroy_v69(guc);
+tlb_fini:
+ fini_tlb_lookup(guc);
return ret;
}
@@ -1962,6 +2019,7 @@ void intel_guc_submission_fini(struct intel_guc *guc)
guc_lrc_desc_pool_destroy_v69(guc);
i915_sched_engine_put(guc->sched_engine);
bitmap_free(guc->submission_state.guc_ids_bitmap);
+ fini_tlb_lookup(guc);
guc->submission_initialized = false;
}
@@ -4592,6 +4650,41 @@ g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)
return ce;
}
+static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno)
+{
+ struct intel_guc_tlb_wait *wait;
+ unsigned long flags;
+
+ xa_lock_irqsave(&guc->tlb_lookup, flags);
+ wait = xa_load(&guc->tlb_lookup, seqno);
+
+ /* We received a response after the waiting task did exit with a timeout */
+ if (unlikely(!wait))
+ drm_err(&guc_to_gt(guc)->i915->drm,
+ "Stale tlb invalidation response with seqno %d\n", seqno);
+
+ if (wait)
+ wake_up_tlb_invalidate(wait);
+
+ xa_unlock_irqrestore(&guc->tlb_lookup, flags);
+}
+
+int intel_guc_tlb_invalidation_done(struct intel_guc *guc, const u32 *hxg,
+ u32 size)
+{
+ u32 seqno, hxg_len, len;
+
+ hxg_len = size - GUC_CTB_MSG_MIN_LEN;
+ len = hxg_len - GUC_HXG_MSG_MIN_LEN;
+
+ if (unlikely(len < 1))
+ return -EPROTO;
+
+ seqno = hxg[GUC_HXG_MSG_MIN_LEN];
+ wait_wake_outstanding_tlb_g2h(guc, seqno);
+ return 0;
+}
+
int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
const u32 *msg,
u32 len)
--
2.41.0
More information about the Intel-gfx-trybot
mailing list