[PATCH 04/11] drm/xe: Add tlb_invalidation.seqno_lock
Matthew Brost
matthew.brost at intel.com
Sat Jul 6 00:02:45 UTC 2024
Stop abusing CT lock for GT TLB invalidation fence seqno, add a
dedicated lock.
Signed-off-by: Matthew Brost <matthew.brost at intel.com>
---
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 106 +++++++++++---------
drivers/gpu/drm/xe/xe_gt_types.h | 13 ++-
2 files changed, 71 insertions(+), 48 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
index 147840b66ba9..e37436c140df 100644
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
@@ -3,6 +3,8 @@
* Copyright © 2023 Intel Corporation
*/
+#include <drm/drm_managed.h>
+
#include "xe_gt_tlb_invalidation.h"
#include "abi/guc_actions_abi.h"
@@ -84,7 +86,8 @@ int xe_gt_tlb_invalidation_init(struct xe_gt *gt)
INIT_DELAYED_WORK(>->tlb_invalidation.fence_tdr,
xe_gt_tlb_fence_timeout);
- return 0;
+ return drmm_mutex_init(>_to_xe(gt)->drm,
+ >->tlb_invalidation.seqno_lock);
}
static void
@@ -118,7 +121,7 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)
* appear.
*/
- mutex_lock(>->uc.guc.ct.lock);
+ mutex_lock(>->tlb_invalidation.seqno_lock);
spin_lock_irq(>->tlb_invalidation.pending_lock);
cancel_delayed_work(>->tlb_invalidation.fence_tdr);
/*
@@ -138,7 +141,7 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)
>->tlb_invalidation.pending_fences, link)
invalidation_fence_signal(gt_to_xe(gt), fence);
spin_unlock_irq(>->tlb_invalidation.pending_lock);
- mutex_unlock(>->uc.guc.ct.lock);
+ mutex_unlock(>->tlb_invalidation.seqno_lock);
}
static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno)
@@ -159,11 +162,9 @@ static int send_tlb_invalidation(struct xe_guc *guc,
u32 *action, int len)
{
struct xe_gt *gt = guc_to_gt(guc);
- struct xe_device *xe = gt_to_xe(gt);
- int seqno;
- int ret;
xe_gt_assert(gt, fence);
+ lockdep_assert_held(>->tlb_invalidation.seqno_lock);
/*
* XXX: The seqno algorithm relies on TLB invalidation being processed
@@ -171,46 +172,39 @@ static int send_tlb_invalidation(struct xe_guc *guc,
* need to be updated.
*/
- mutex_lock(&guc->ct.lock);
+ action[1] = fence->seqno;
+ return xe_guc_ct_send(&guc->ct, action, len,
+ G2H_LEN_DW_TLB_INVALIDATE, 1);
+}
+
+static void xe_gt_tlb_invalidation_fence_prep(struct xe_gt *gt,
+ struct xe_gt_tlb_invalidation_fence *fence)
+{
+ struct xe_device *xe = gt_to_xe(gt);
+ int seqno;
+
+ lockdep_assert_held(>->tlb_invalidation.seqno_lock);
+
seqno = gt->tlb_invalidation.seqno;
fence->seqno = seqno;
+
trace_xe_gt_tlb_invalidation_fence_send(xe, fence);
- action[1] = seqno;
- ret = xe_guc_ct_send_locked(&guc->ct, action, len,
- G2H_LEN_DW_TLB_INVALIDATE, 1);
- if (!ret && fence) {
- spin_lock_irq(>->tlb_invalidation.pending_lock);
- /*
- * We haven't actually published the TLB fence as per
- * pending_fences, but in theory our seqno could have already
- * been written as we acquired the pending_lock. In such a case
- * we can just go ahead and signal the fence here.
- */
- if (tlb_invalidation_seqno_past(gt, seqno)) {
- __invalidation_fence_signal(xe, fence);
- } else {
- fence->invalidation_time = ktime_get();
- list_add_tail(&fence->link,
- >->tlb_invalidation.pending_fences);
-
- if (list_is_singular(>->tlb_invalidation.pending_fences))
- queue_delayed_work(system_wq,
- >->tlb_invalidation.fence_tdr,
- tlb_timeout_jiffies(gt));
- }
- spin_unlock_irq(>->tlb_invalidation.pending_lock);
- } else if (ret < 0 && fence) {
- __invalidation_fence_signal(xe, fence);
- }
- if (!ret) {
- gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) %
- TLB_INVALIDATION_SEQNO_MAX;
- if (!gt->tlb_invalidation.seqno)
- gt->tlb_invalidation.seqno = 1;
- }
- mutex_unlock(&guc->ct.lock);
- return ret;
+ spin_lock_irq(>->tlb_invalidation.pending_lock);
+ fence->invalidation_time = ktime_get();
+ list_add_tail(&fence->link,
+ >->tlb_invalidation.pending_fences);
+
+ if (list_is_singular(>->tlb_invalidation.pending_fences))
+ queue_delayed_work(system_wq,
+ >->tlb_invalidation.fence_tdr,
+ tlb_timeout_jiffies(gt));
+ spin_unlock_irq(>->tlb_invalidation.pending_lock);
+
+ gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) %
+ TLB_INVALIDATION_SEQNO_MAX;
+ if (!gt->tlb_invalidation.seqno)
+ gt->tlb_invalidation.seqno = 1;
}
#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
@@ -236,9 +230,20 @@ static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt,
0, /* seqno, replaced in send_tlb_invalidation */
MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
};
+ int ret;
+
+ mutex_lock(>->tlb_invalidation.seqno_lock);
+
+ xe_gt_tlb_invalidation_fence_prep(gt, fence);
+
+ ret = send_tlb_invalidation(>->uc.guc, fence, action,
+ ARRAY_SIZE(action));
+ if (ret < 0)
+ invalidation_fence_signal(gt_to_xe(gt), fence);
- return send_tlb_invalidation(>->uc.guc, fence, action,
- ARRAY_SIZE(action));
+ mutex_unlock(>->tlb_invalidation.seqno_lock);
+
+ return ret;
}
/**
@@ -310,6 +315,7 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
#define MAX_TLB_INVALIDATION_LEN 7
u32 action[MAX_TLB_INVALIDATION_LEN];
int len = 0;
+ int ret;
xe_gt_assert(gt, fence);
@@ -370,7 +376,17 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
- return send_tlb_invalidation(>->uc.guc, fence, action, len);
+ mutex_lock(>->tlb_invalidation.seqno_lock);
+
+ xe_gt_tlb_invalidation_fence_prep(gt, fence);
+
+ ret = send_tlb_invalidation(>->uc.guc, fence, action, len);
+ if (ret < 0)
+ invalidation_fence_signal(xe, fence);
+
+ mutex_unlock(>->tlb_invalidation.seqno_lock);
+
+ return ret;
}
/**
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index d190a66514c0..fb280f058cdc 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -169,17 +169,24 @@ struct xe_gt {
/** @tlb_invalidation: TLB invalidation state */
struct {
- /** @tlb_invalidation.seqno: TLB invalidation seqno, protected by CT lock */
+ /** @tlb_invalidation.seqno_lock: TLB invalidation seqno lock */
+ struct mutex seqno_lock;
+ /**
+ * @tlb_invalidation.seqno: TLB invalidation seqno, protected
+ * by @tlb_invalidation.seqno_lock
+ */
#define TLB_INVALIDATION_SEQNO_MAX 0x100000
int seqno;
/**
* @tlb_invalidation.seqno_recv: last received TLB invalidation seqno,
- * protected by CT lock
+ * protected by @tlb_invalidation.seqno_lock (send) and
+ * @tlb_invalidation.pending_lock (send, recv)
*/
int seqno_recv;
/**
* @tlb_invalidation.pending_fences: list of pending fences waiting TLB
- * invaliations, protected by CT lock
+ * invaliations, protected by @tlb_invalidation.seqno_lock
+ * (send) and @tlb_invalidation.pending_lock (send, recv)
*/
struct list_head pending_fences;
/**
--
2.34.1
More information about the Intel-xe
mailing list