[PATCH 5/5] drm/xe: Split TLB invalidation code in frontend and backend
Matthew Brost
matthew.brost at intel.com
Wed Jul 23 20:18:03 UTC 2025
On Wed, Jul 23, 2025 at 12:17:49PM -0700, Matthew Brost wrote:
> On Wed, Jul 23, 2025 at 06:22:22PM +0000, stuartsummers wrote:
> > From: Matthew Brost <matthew.brost at intel.com>
> >
> > The frontend exposes an API to the driver to send invalidations, handles
> > sequence number assignment, synchronization (fences), and provides a
> > timeout mechanism. The backend issues the actual invalidation to the
> > hardware (or firmware).
> >
> > The new layering easily allows issuing TLB invalidations to different
> > hardware or firmware interfaces.
> >
> > Normalize some naming while here too.
> >
> > Signed-off-by: Matthew Brost <matthew.brost at intel.com>
> > Signed-off-by: Stuart Summers <stuart.summers at intel.com>
> > ---
> > drivers/gpu/drm/xe/Makefile | 1 +
> > drivers/gpu/drm/xe/xe_guc_ct.c | 2 +-
> > drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 263 +++++++++++++
> > drivers/gpu/drm/xe/xe_guc_tlb_inval.h | 19 +
> > drivers/gpu/drm/xe/xe_tlb_inval.c | 495 +++++++-----------------
> > drivers/gpu/drm/xe/xe_tlb_inval.h | 14 +-
> > drivers/gpu/drm/xe/xe_tlb_inval_types.h | 77 +++-
> > 7 files changed, 505 insertions(+), 366 deletions(-)
> > create mode 100644 drivers/gpu/drm/xe/xe_guc_tlb_inval.c
> > create mode 100644 drivers/gpu/drm/xe/xe_guc_tlb_inval.h
> >
> > diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
> > index 332b2057cc00..8a2f836b3ab2 100644
> > --- a/drivers/gpu/drm/xe/Makefile
> > +++ b/drivers/gpu/drm/xe/Makefile
> > @@ -75,6 +75,7 @@ xe-y += xe_bb.o \
> > xe_guc_log.o \
> > xe_guc_pc.o \
> > xe_guc_submit.o \
> > + xe_guc_tlb_inval.o \
> > xe_heci_gsc.o \
> > xe_huc.o \
> > xe_hw_engine.o \
> > diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
> > index 2ef86c0ae8b4..90ebda5b3790 100644
> > --- a/drivers/gpu/drm/xe/xe_guc_ct.c
> > +++ b/drivers/gpu/drm/xe/xe_guc_ct.c
> > @@ -30,9 +30,9 @@
> > #include "xe_guc_log.h"
> > #include "xe_guc_relay.h"
> > #include "xe_guc_submit.h"
> > +#include "xe_guc_tlb_inval.h"
> > #include "xe_map.h"
> > #include "xe_pm.h"
> > -#include "xe_tlb_inval.h"
> > #include "xe_trace_guc.h"
> >
> > static void receive_g2h(struct xe_guc_ct *ct);
> > diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
> > new file mode 100644
> > index 000000000000..27d7dc938cb1
> > --- /dev/null
> > +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
> > @@ -0,0 +1,263 @@
> > +// SPDX-License-Identifier: MIT
> > +/*
> > + * Copyright © 2025 Intel Corporation
> > + */
> > +
> > +#include "abi/guc_actions_abi.h"
> > +
> > +#include "xe_device.h"
> > +#include "xe_gt_stats.h"
> > +#include "xe_gt_types.h"
> > +#include "xe_guc.h"
> > +#include "xe_guc_ct.h"
> > +#include "xe_guc_tlb_inval.h"
> > +#include "xe_force_wake.h"
> > +#include "xe_mmio.h"
> > +#include "xe_tlb_inval.h"
> > +
> > +#include "regs/xe_guc_regs.h"
> > +
> > +/*
> > + * XXX: The seqno algorithm relies on TLB invalidation being processed in order
> > + * which they currently are by the GuC, if that changes the algorithm will need
> > + * to be updated.
> > + */
> > +
> > +static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len)
> > +{
> > + struct xe_gt *gt = guc_to_gt(guc);
> > +
> > + lockdep_assert_held(&guc->ct.lock);
> > + xe_gt_assert(gt, action[1]); /* Seqno */
> > +
> > + xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1);
> > + return xe_guc_ct_send(&guc->ct, action, len,
> > + G2H_LEN_DW_TLB_INVALIDATE, 1);
>
> As written, you’d need xe_guc_ct_send_locked here—but you actually
> don’t. More on that below.
>
> > +}
> > +
> > +#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
> > + XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
> > + XE_GUC_TLB_INVAL_FLUSH_CACHE)
> > +
> > +static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > + u32 action[] = {
> > + XE_GUC_ACTION_TLB_INVALIDATION_ALL,
> > + seqno,
> > + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
> > + };
> > +
> > + return send_tlb_inval(guc, action, ARRAY_SIZE(action));
> > +}
> > +
> > +static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > + struct xe_gt *gt = guc_to_gt(guc);
> > + struct xe_device *xe = guc_to_xe(guc);
> > +
> > + lockdep_assert_held(&guc->ct.lock);
> > +
> > + /*
> > + * Returning -ECANCELED in this function is squashed at the caller and
> > + * signals waiters.
> > + */
> > +
> > + if (xe_guc_ct_enabled(&guc->ct) && guc->submission_state.enabled) {
> > + u32 action[] = {
> > + XE_GUC_ACTION_TLB_INVALIDATION,
> > + seqno,
> > + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
> > + };
> > +
> > + return send_tlb_inval(guc, action, ARRAY_SIZE(action));
> > + } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
> > + struct xe_mmio *mmio = >->mmio;
> > + unsigned int fw_ref;
> > +
> > + if (IS_SRIOV_VF(xe))
> > + return -ECANCELED;
> > +
> > + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
> > + if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
> > + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
> > + PVC_GUC_TLB_INV_DESC1_INVALIDATE);
> > + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0,
> > + PVC_GUC_TLB_INV_DESC0_VALID);
> > + } else {
> > + xe_mmio_write32(mmio, GUC_TLB_INV_CR,
> > + GUC_TLB_INV_CR_INVALIDATE);
> > + }
> > + xe_force_wake_put(gt_to_fw(gt), fw_ref);
> > + }
> > +
> > + return -ECANCELED;
> > +}
> > +
> > +/*
> > + * Ensure that roundup_pow_of_two(length) doesn't overflow.
> > + * Note that roundup_pow_of_two() operates on unsigned long,
> > + * not on u64.
> > + */
> > +#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
> > +
> > +static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
> > + u64 start, u64 end, u32 asid)
> > +{
> > +#define MAX_TLB_INVALIDATION_LEN 7
> > + struct xe_guc *guc = tlb_inval->private;
> > + struct xe_gt *gt = guc_to_gt(guc);
> > + u32 action[MAX_TLB_INVALIDATION_LEN];
> > + u64 length = end - start;
> > + int len = 0;
> > +
> > + lockdep_assert_held(&guc->ct.lock);
> > +
> > + if (guc_to_xe(guc)->info.force_execlist)
> > + return -ECANCELED;
> > +
> > + action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
> > + action[len++] = seqno;
> > + if (!gt_to_xe(gt)->info.has_range_tlb_inval ||
> > + length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
> > + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
> > + } else {
> > + u64 orig_start = start;
> > + u64 align;
> > +
> > + if (length < SZ_4K)
> > + length = SZ_4K;
> > +
> > + /*
> > + * We need to invalidate a higher granularity if start address
> > + * is not aligned to length. When start is not aligned with
> > + * length we need to find the length large enough to create an
> > + * address mask covering the required range.
> > + */
> > + align = roundup_pow_of_two(length);
> > + start = ALIGN_DOWN(start, align);
> > + end = ALIGN(end, align);
> > + length = align;
> > + while (start + length < end) {
> > + length <<= 1;
> > + start = ALIGN_DOWN(orig_start, length);
> > + }
> > +
> > + /*
> > + * Minimum invalidation size for a 2MB page that the hardware
> > + * expects is 16MB
> > + */
> > + if (length >= SZ_2M) {
> > + length = max_t(u64, SZ_16M, length);
> > + start = ALIGN_DOWN(orig_start, length);
> > + }
> > +
> > + xe_gt_assert(gt, length >= SZ_4K);
> > + xe_gt_assert(gt, is_power_of_2(length));
> > + xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
> > + ilog2(SZ_2M) + 1)));
> > + xe_gt_assert(gt, IS_ALIGNED(start, length));
> > +
> > + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
> > + action[len++] = asid;
> > + action[len++] = lower_32_bits(start);
> > + action[len++] = upper_32_bits(start);
> > + action[len++] = ilog2(length) - ilog2(SZ_4K);
> > + }
> > +
> > + xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
> > +
> > + return send_tlb_inval(guc, action, len);
> > +}
> > +
> > +static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > +
> > + return xe_guc_ct_initialized(&guc->ct);
> > +}
> > +
> > +static void tlb_inval_flush(struct xe_tlb_inval *tlb_inval)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > +
> > + LNL_FLUSH_WORK(&guc->ct.g2h_worker);
> > +}
> > +
> > +static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > +
> > + /* this reflects what HW/GuC needs to process TLB inv request */
> > + const long hw_tlb_timeout = HZ / 4;
> > +
> > + /* this estimates actual delay caused by the CTB transport */
> > + long delay = xe_guc_ct_queue_proc_time_jiffies(&guc->ct);
> > +
> > + return hw_tlb_timeout + 2 * delay;
> > +}
> > +
> > +static void tlb_inval_lock(struct xe_tlb_inval *tlb_inval)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > +
> > + mutex_lock(&guc->ct.lock);
> > +}
> > +
> > +static void tlb_inval_unlock(struct xe_tlb_inval *tlb_inval)
> > +{
> > + struct xe_guc *guc = tlb_inval->private;
> > +
> > + mutex_unlock(&guc->ct.lock);
> > +}
> > +
> > +static const struct xe_tlb_inval_ops guc_tlb_inval_ops = {
> > + .all = send_tlb_inval_all,
> > + .ggtt = send_tlb_inval_ggtt,
> > + .ppgtt = send_tlb_inval_ppgtt,
> > + .initialized = tlb_inval_initialized,
> > + .flush = tlb_inval_flush,
> > + .timeout_delay = tlb_inval_timeout_delay,
> > + .lock = tlb_inval_lock,
> > + .unlock = tlb_inval_unlock,
> > +};
> > +
> > +/**
> > + * xe_guc_tlb_inval_init_early() - Init GuC TLB invalidation early
> > + * @guc: GuC object
> > + * @tlb_inval: TLB invalidation client
> > + *
> > + * Inititialize GuC TLB invalidation by setting back pointer in TLB invalidation
> > + * client to the GuC and setting GuC backend ops.
> > + */
> > +void xe_guc_tlb_inval_init_early(struct xe_guc *guc,
> > + struct xe_tlb_inval *tlb_inval)
> > +{
> > + tlb_inval->private = guc;
> > + tlb_inval->ops = &guc_tlb_inval_ops;
> > +}
> > +
> > +/**
> > + * xe_guc_tlb_inval_done_handler() - TLB invalidation done handler
> > + * @guc: guc
> > + * @msg: message indicating TLB invalidation done
> > + * @len: length of message
> > + *
> > + * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any
> > + * invalidation fences for seqno. Algorithm for this depends on seqno being
> > + * received in-order and asserts this assumption.
> > + *
> > + * Return: 0 on success, -EPROTO for malformed messages.
> > + */
> > +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
> > +{
> > + struct xe_gt *gt = guc_to_gt(guc);
> > +
> > + if (unlikely(len != 1))
> > + return -EPROTO;
> > +
> > + xe_tlb_inval_done_handler(>->tlb_inval, msg[0]);
> > +
> > + return 0;
> > +}
> > diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.h b/drivers/gpu/drm/xe/xe_guc_tlb_inval.h
> > new file mode 100644
> > index 000000000000..07d668b02e3d
> > --- /dev/null
> > +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.h
> > @@ -0,0 +1,19 @@
> > +/* SPDX-License-Identifier: MIT */
> > +/*
> > + * Copyright © 2025 Intel Corporation
> > + */
> > +
> > +#ifndef _XE_GUC_TLB_INVAL_H_
> > +#define _XE_GUC_TLB_INVAL_H_
> > +
> > +#include <linux/types.h>
> > +
> > +struct xe_guc;
> > +struct xe_tlb_inval;
> > +
> > +void xe_guc_tlb_inval_init_early(struct xe_guc *guc,
> > + struct xe_tlb_inval *tlb_inval);
> > +
> > +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
> > +
> > +#endif
> > diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c
> > index c795b78362bf..071c25fbdbac 100644
> > --- a/drivers/gpu/drm/xe/xe_tlb_inval.c
> > +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c
> > @@ -12,50 +12,45 @@
> > #include "xe_gt_printk.h"
> > #include "xe_guc.h"
> > #include "xe_guc_ct.h"
> > +#include "xe_guc_tlb_inval.h"
> > #include "xe_gt_stats.h"
> > #include "xe_tlb_inval.h"
> > #include "xe_mmio.h"
> > #include "xe_pm.h"
> > -#include "xe_sriov.h"
> > +#include "xe_tlb_inval.h"
> > #include "xe_trace.h"
> > -#include "regs/xe_guc_regs.h"
> > -
> > -#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS
> >
> > -/*
> > - * TLB inval depends on pending commands in the CT queue and then the real
> > - * invalidation time. Double up the time to process full CT queue
> > - * just to be on the safe side.
> > +/**
> > + * DOC: Xe TLB invalidation
> > + *
> > + * Xe TLB invalidation is implemented in two layers. The first is the frontend
> > + * API, which provides an interface for TLB invalidations to the driver code.
> > + * The frontend handles seqno assignment, synchronization (fences), and the
> > + * timeout mechanism. The frontend is implemented via an embedded structure
> > + * xe_tlb_inval that includes a set of ops hooking into the backend. The backend
> > + * interacts with the hardware (or firmware) to perform the actual invalidation.
> > */
> > -static long tlb_timeout_jiffies(struct xe_gt *gt)
> > -{
> > - /* this reflects what HW/GuC needs to process TLB inv request */
> > - const long hw_tlb_timeout = HZ / 4;
> >
> > - /* this estimates actual delay caused by the CTB transport */
> > - long delay = xe_guc_ct_queue_proc_time_jiffies(>->uc.guc.ct);
> > -
> > - return hw_tlb_timeout + 2 * delay;
> > -}
> > +#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS
> >
> > static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
> > {
> > - struct xe_gt *gt;
> > -
> > if (WARN_ON_ONCE(!fence->tlb_inval))
> > return;
> >
> > - gt = fence->tlb_inval->private;
> > - xe_pm_runtime_put(gt_to_xe(gt));
> > + xe_pm_runtime_put(fence->tlb_inval->xe);
> > fence->tlb_inval = NULL; /* fini() should be called once */
> > }
> >
> > static void
> > -__inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
> > +xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence)
> > {
> > bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);
> >
> > - trace_xe_tlb_inval_fence_signal(xe, fence);
> > + lockdep_assert_held(&fence->tlb_inval->pending_lock);
> > +
> > + list_del(&fence->link);
> > + trace_xe_tlb_inval_fence_signal(fence->tlb_inval->xe, fence);
> > xe_tlb_inval_fence_fini(fence);
> > dma_fence_signal(&fence->base);
> > if (!stack)
> > @@ -63,57 +58,50 @@ __inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
> > }
> >
> > static void
> > -inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
> > +xe_tlb_inval_fence_signal_unlocked(struct xe_tlb_inval_fence *fence)
> > {
> > - lockdep_assert_held(&fence->tlb_inval->pending_lock);
> > -
> > - list_del(&fence->link);
> > - __inval_fence_signal(xe, fence);
> > -}
> > + struct xe_tlb_inval *tlb_inval = fence->tlb_inval;
> >
> > -static void
> > -inval_fence_signal_unlocked(struct xe_device *xe,
> > - struct xe_tlb_inval_fence *fence)
> > -{
> > - spin_lock_irq(&fence->tlb_inval->pending_lock);
> > - inval_fence_signal(xe, fence);
> > - spin_unlock_irq(&fence->tlb_inval->pending_lock);
> > + spin_lock_irq(&tlb_inval->pending_lock);
> > + xe_tlb_inval_fence_signal(fence);
> > + spin_unlock_irq(&tlb_inval->pending_lock);
> > }
> >
> > -static void xe_gt_tlb_fence_timeout(struct work_struct *work)
> > +static void xe_tlb_inval_fence_timeout(struct work_struct *work)
> > {
> > - struct xe_gt *gt = container_of(work, struct xe_gt,
> > - tlb_inval.fence_tdr.work);
> > - struct xe_device *xe = gt_to_xe(gt);
> > + struct xe_tlb_inval *tlb_inval = container_of(work, struct xe_tlb_inval,
> > + fence_tdr.work);
> > + struct xe_device *xe = tlb_inval->xe;
> > struct xe_tlb_inval_fence *fence, *next;
> > + long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval);
> >
> > - LNL_FLUSH_WORK(>->uc.guc.ct.g2h_worker);
> > + tlb_inval->ops->flush(tlb_inval);
> >
> > - spin_lock_irq(>->tlb_inval.pending_lock);
> > + spin_lock_irq(&tlb_inval->pending_lock);
> > list_for_each_entry_safe(fence, next,
> > - >->tlb_inval.pending_fences, link) {
> > + &tlb_inval->pending_fences, link) {
> > s64 since_inval_ms = ktime_ms_delta(ktime_get(),
> > fence->inval_time);
> >
> > - if (msecs_to_jiffies(since_inval_ms) < tlb_timeout_jiffies(gt))
> > + if (msecs_to_jiffies(since_inval_ms) < timeout_delay)
> > break;
> >
> > trace_xe_tlb_inval_fence_timeout(xe, fence);
> > - xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",
> > - fence->seqno, gt->tlb_inval.seqno_recv);
> > + drm_err(&xe->drm,
> > + "TLB invalidation fence timeout, seqno=%d recv=%d",
> > + fence->seqno, tlb_inval->seqno_recv);
> >
> > fence->base.error = -ETIME;
> > - inval_fence_signal(xe, fence);
> > + xe_tlb_inval_fence_signal(fence);
> > }
> > - if (!list_empty(>->tlb_inval.pending_fences))
> > - queue_delayed_work(system_wq,
> > - >->tlb_inval.fence_tdr,
> > - tlb_timeout_jiffies(gt));
> > - spin_unlock_irq(>->tlb_inval.pending_lock);
> > + if (!list_empty(&tlb_inval->pending_fences))
> > + queue_delayed_work(system_wq, &tlb_inval->fence_tdr,
> > + timeout_delay);
> > + spin_unlock_irq(&tlb_inval->pending_lock);
> > }
> >
> > /**
> > - * xe_tlb_inval_init_early - Initialize TLB invalidation state
> > + * xe_gt_tlb_inval_init_early() - Initialize TLB invalidation state
> > * @gt: GT structure
> > *
> > * Initialize TLB invalidation state, purely software initialization, should
> > @@ -123,13 +111,12 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work)
> > */
> > int xe_gt_tlb_inval_init_early(struct xe_gt *gt)
> > {
> > - gt->tlb_inval.private = gt;
> > + gt->tlb_inval.xe = gt_to_xe(gt);
> > gt->tlb_inval.seqno = 1;
> > INIT_LIST_HEAD(>->tlb_inval.pending_fences);
> > spin_lock_init(>->tlb_inval.pending_lock);
> > spin_lock_init(>->tlb_inval.lock);
> > - INIT_DELAYED_WORK(>->tlb_inval.fence_tdr,
> > - xe_gt_tlb_fence_timeout);
> > + INIT_DELAYED_WORK(>->tlb_inval.fence_tdr, xe_tlb_inval_fence_timeout);
> >
> > gt->tlb_inval.job_wq =
> > drmm_alloc_ordered_workqueue(>_to_xe(gt)->drm, "gt-tbl-inval-job-wq",
> > @@ -137,60 +124,64 @@ int xe_gt_tlb_inval_init_early(struct xe_gt *gt)
> > if (IS_ERR(gt->tlb_inval.job_wq))
> > return PTR_ERR(gt->tlb_inval.job_wq);
> >
> > + /* XXX: Blindly setting up backend to GuC */
> > + xe_guc_tlb_inval_init_early(>->uc.guc, >->tlb_inval);
> > +
> > return 0;
> > }
> >
> > /**
> > - * xe_tlb_inval_reset - Initialize TLB invalidation reset
> > + * xe_tlb_inval_reset() - TLB invalidation reset
> > * @tlb_inval: TLB invalidation client
> > *
> > * Signal any pending invalidation fences, should be called during a GT reset
> > */
> > void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
> > {
> > - struct xe_gt *gt = tlb_inval->private;
> > struct xe_tlb_inval_fence *fence, *next;
> > int pending_seqno;
> >
> > /*
> > - * we can get here before the CTs are even initialized if we're wedging
> > - * very early, in which case there are not going to be any pending
> > - * fences so we can bail immediately.
> > + * we can get here before the backends are even initialized if we're
> > + * wedging very early, in which case there are not going to be any
> > + * pendind fences so we can bail immediately.
> > */
> > - if (!xe_guc_ct_initialized(>->uc.guc.ct))
> > + if (!tlb_inval->ops->initialized(tlb_inval))
> > return;
> >
> > /*
> > - * CT channel is already disabled at this point. No new TLB requests can
> > + * Backend is already disabled at this point. No new TLB requests can
> > * appear.
> > */
> >
> > - mutex_lock(>->uc.guc.ct.lock);
> > - spin_lock_irq(>->tlb_inval.pending_lock);
> > - cancel_delayed_work(>->tlb_inval.fence_tdr);
> > + tlb_inval->ops->lock(tlb_inval);
>
> I think you want a dedicated lock embedded in struct xe_tlb_inval,
> rather than reaching into the backend to grab one.
>
> This will deadlock as written: G2H TLB inval messages are sometimes
> processed while holding ct->lock (non-fast path, unlikely) and sometimes
> without it (fast path, likely).
Ugh, I'm off today. Ignore the deadlock part, I was confusing myself...
I was thinking this was the function xe_tlb_inval_done_handler, it is
not. I still think xe_tlb_inval should its own lock but this patch
written should work with s/xe_guc_ct_send/xe_guc_ct_send_locked.
Matt
>
> I’d call this lock seqno_lock, since it protects exactly that—the order
> in which a seqno is assigned by the frontend and handed to the backend.
>
> Prime this lock for reclaim as well—do what primelockdep() does in
> xe_guc_ct.c—to make it clear that memory allocations are not allowed
> while the lock is held as TLB invalidations can be called from two
> reclaim paths:
>
> - MMU notifier callbacks
> - The dma-fence signaling path of VM binds that require a TLB
> invalidation
>
> > + spin_lock_irq(&tlb_inval->pending_lock);
> > + cancel_delayed_work(&tlb_inval->fence_tdr);
> > /*
> > * We might have various kworkers waiting for TLB flushes to complete
> > * which are not tracked with an explicit TLB fence, however at this
> > - * stage that will never happen since the CT is already disabled, so
> > - * make sure we signal them here under the assumption that we have
> > + * stage that will never happen since the backend is already disabled,
> > + * so make sure we signal them here under the assumption that we have
> > * completed a full GT reset.
> > */
> > - if (gt->tlb_inval.seqno == 1)
> > + if (tlb_inval->seqno == 1)
> > pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1;
> > else
> > - pending_seqno = gt->tlb_inval.seqno - 1;
> > - WRITE_ONCE(gt->tlb_inval.seqno_recv, pending_seqno);
> > + pending_seqno = tlb_inval->seqno - 1;
> > + WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno);
> >
> > list_for_each_entry_safe(fence, next,
> > - >->tlb_inval.pending_fences, link)
> > - inval_fence_signal(gt_to_xe(gt), fence);
> > - spin_unlock_irq(>->tlb_inval.pending_lock);
> > - mutex_unlock(>->uc.guc.ct.lock);
> > + &tlb_inval->pending_fences, link)
> > + xe_tlb_inval_fence_signal(fence);
> > + spin_unlock_irq(&tlb_inval->pending_lock);
> > + tlb_inval->ops->unlock(tlb_inval);
> > }
> >
> > -static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno)
> > +static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno)
> > {
> > - int seqno_recv = READ_ONCE(gt->tlb_inval.seqno_recv);
> > + int seqno_recv = READ_ONCE(tlb_inval->seqno_recv);
> > +
> > + lockdep_assert_held(&tlb_inval->pending_lock);
> >
> > if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2))
> > return false;
> > @@ -201,44 +192,20 @@ static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno)
> > return seqno_recv >= seqno;
> > }
> >
> > -static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len)
> > -{
> > - struct xe_gt *gt = guc_to_gt(guc);
> > -
> > - xe_gt_assert(gt, action[1]); /* Seqno */
> > - lockdep_assert_held(&guc->ct.lock);
> > -
> > - /*
> > - * XXX: The seqno algorithm relies on TLB invalidation being processed
> > - * in order which they currently are, if that changes the algorithm will
> > - * need to be updated.
> > - */
> > -
> > - xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1);
> > -
> > - return xe_guc_ct_send(&guc->ct, action, len,
> > - G2H_LEN_DW_TLB_INVALIDATE, 1);
> > -}
> > -
> > static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence)
> > {
> > struct xe_tlb_inval *tlb_inval = fence->tlb_inval;
> > - struct xe_gt *gt = tlb_inval->private;
> > - struct xe_device *xe = gt_to_xe(gt);
> > -
> > - lockdep_assert_held(>->uc.guc.ct.lock);
> >
> > fence->seqno = tlb_inval->seqno;
> > - trace_xe_tlb_inval_fence_send(xe, fence);
> > + trace_xe_tlb_inval_fence_send(tlb_inval->xe, fence);
> >
> > spin_lock_irq(&tlb_inval->pending_lock);
> > fence->inval_time = ktime_get();
> > list_add_tail(&fence->link, &tlb_inval->pending_fences);
> >
> > if (list_is_singular(&tlb_inval->pending_fences))
> > - queue_delayed_work(system_wq,
> > - &tlb_inval->fence_tdr,
> > - tlb_timeout_jiffies(gt));
> > + queue_delayed_work(system_wq, &tlb_inval->fence_tdr,
> > + tlb_inval->ops->timeout_delay(tlb_inval));
> > spin_unlock_irq(&tlb_inval->pending_lock);
> >
> > tlb_inval->seqno = (tlb_inval->seqno + 1) %
> > @@ -247,202 +214,63 @@ static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence)
> > tlb_inval->seqno = 1;
> > }
> >
> > -#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
> > - XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
> > - XE_GUC_TLB_INVAL_FLUSH_CACHE)
> > -
> > -static int send_tlb_inval_ggtt(struct xe_gt *gt, int seqno)
> > -{
> > - u32 action[] = {
> > - XE_GUC_ACTION_TLB_INVALIDATION,
> > - seqno,
> > - MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
> > - };
> > -
> > - return send_tlb_inval(>->uc.guc, action, ARRAY_SIZE(action));
> > -}
> > -
> > -static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
> > - struct xe_tlb_inval_fence *fence)
> > -{
> > - u32 action[] = {
> > - XE_GUC_ACTION_TLB_INVALIDATION_ALL,
> > - 0, /* seqno, replaced in send_tlb_inval */
> > - MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
> > - };
> > - struct xe_gt *gt = tlb_inval->private;
> > -
> > - xe_gt_assert(gt, fence);
> > -
> > - return send_tlb_inval(>->uc.guc, action, ARRAY_SIZE(action));
> > -}
> > +#define xe_tlb_inval_issue(__tlb_inval, __fence, op, args...) \
> > +({ \
> > + int __ret; \
> > + \
> > + xe_assert((__tlb_inval)->xe, (__tlb_inval)->ops); \
> > + xe_assert((__tlb_inval)->xe, (__fence)); \
> > + \
> > + (__tlb_inval)->ops->lock((__tlb_inval)); \
> > + xe_tlb_inval_fence_prep((__fence)); \
> > + __ret = op((__tlb_inval), (__fence)->seqno, ##args); \
> > + if (__ret < 0) \
> > + xe_tlb_inval_fence_signal_unlocked((__fence)); \
> > + (__tlb_inval)->ops->unlock((__tlb_inval)); \
> > + \
> > + __ret == -ECANCELED ? 0 : __ret; \
> > +})
> >
> > /**
> > - * xe_gt_tlb_invalidation_all - Invalidate all TLBs across PF and all VFs.
> > - * @gt: the &xe_gt structure
> > - * @fence: the &xe_tlb_inval_fence to be signaled on completion
> > + * xe_tlb_inval_all() - Issue a TLB invalidation for all TLBs
> > + * @tlb_inval: TLB invalidation client
> > + * @fence: invalidation fence which will be signal on TLB invalidation
> > + * completion
> > *
> > - * Send a request to invalidate all TLBs across PF and all VFs.
> > + * Issue a TLB invalidation for all TLBs. Completion of TLB is asynchronous and
> > + * caller can use the invalidation fence to wait for completion.
> > *
> > * Return: 0 on success, negative error code on error
> > */
> > int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
> > struct xe_tlb_inval_fence *fence)
> > {
> > - struct xe_gt *gt = tlb_inval->private;
> > - int err;
> > -
> > - err = send_tlb_inval_all(tlb_inval, fence);
> > - if (err)
> > - xe_gt_err(gt, "TLB invalidation request failed (%pe)", ERR_PTR(err));
> > -
> > - return err;
> > -}
> > -
> > -/*
> > - * Ensure that roundup_pow_of_two(length) doesn't overflow.
> > - * Note that roundup_pow_of_two() operates on unsigned long,
> > - * not on u64.
> > - */
> > -#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
> > -
> > -static int send_tlb_inval_ppgtt(struct xe_gt *gt, u64 start, u64 end,
> > - u32 asid, int seqno)
> > -{
> > -#define MAX_TLB_INVALIDATION_LEN 7
> > - u32 action[MAX_TLB_INVALIDATION_LEN];
> > - u64 length = end - start;
> > - int len = 0;
> > -
> > - action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
> > - action[len++] = seqno;
> > - if (!gt_to_xe(gt)->info.has_range_tlb_inval ||
> > - length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
> > - action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
> > - } else {
> > - u64 orig_start = start;
> > - u64 align;
> > -
> > - if (length < SZ_4K)
> > - length = SZ_4K;
> > -
> > - /*
> > - * We need to invalidate a higher granularity if start address
> > - * is not aligned to length. When start is not aligned with
> > - * length we need to find the length large enough to create an
> > - * address mask covering the required range.
> > - */
> > - align = roundup_pow_of_two(length);
> > - start = ALIGN_DOWN(start, align);
> > - end = ALIGN(end, align);
> > - length = align;
> > - while (start + length < end) {
> > - length <<= 1;
> > - start = ALIGN_DOWN(orig_start, length);
> > - }
> > -
> > - /*
> > - * Minimum invalidation size for a 2MB page that the hardware
> > - * expects is 16MB
> > - */
> > - if (length >= SZ_2M) {
> > - length = max_t(u64, SZ_16M, length);
> > - start = ALIGN_DOWN(orig_start, length);
> > - }
> > -
> > - xe_gt_assert(gt, length >= SZ_4K);
> > - xe_gt_assert(gt, is_power_of_2(length));
> > - xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
> > - ilog2(SZ_2M) + 1)));
> > - xe_gt_assert(gt, IS_ALIGNED(start, length));
> > -
> > - action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
> > - action[len++] = asid;
> > - action[len++] = lower_32_bits(start);
> > - action[len++] = upper_32_bits(start);
> > - action[len++] = ilog2(length) - ilog2(SZ_4K);
> > - }
> > -
> > - xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
> > -
> > - return send_tlb_inval(>->uc.guc, action, len);
> > -}
> > -
> > -static int __xe_tlb_inval_ggtt(struct xe_gt *gt,
> > - struct xe_tlb_inval_fence *fence)
> > -{
> > - int ret;
> > -
> > - mutex_lock(>->uc.guc.ct.lock);
> > -
> > - xe_tlb_inval_fence_prep(fence);
> > -
> > - ret = send_tlb_inval_ggtt(gt, fence->seqno);
> > - if (ret < 0)
> > - inval_fence_signal_unlocked(gt_to_xe(gt), fence);
> > -
> > - mutex_unlock(>->uc.guc.ct.lock);
> > -
> > - /*
> > - * -ECANCELED indicates the CT is stopped for a GT reset. TLB caches
> > - * should be nuked on a GT reset so this error can be ignored.
> > - */
> > - if (ret == -ECANCELED)
> > - return 0;
> > -
> > - return ret;
> > + return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->all);
> > }
> >
> > /**
> > - * xe_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT
> > + * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT
> > * @tlb_inval: TLB invalidation client
> > *
> > - * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is
> > - * synchronous.
> > + * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and
> > + * caller can use the invalidation fence to wait for completion.
> > *
> > * Return: 0 on success, negative error code on error
> > */
> > int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
> > {
> > - struct xe_gt *gt = tlb_inval->private;
> > - struct xe_device *xe = gt_to_xe(gt);
> > - unsigned int fw_ref;
> > -
> > - if (xe_guc_ct_enabled(>->uc.guc.ct) &&
> > - gt->uc.guc.submission_state.enabled) {
> > - struct xe_tlb_inval_fence fence;
> > - int ret;
> > -
> > - xe_tlb_inval_fence_init(tlb_inval, &fence, true);
> > - ret = __xe_tlb_inval_ggtt(gt, &fence);
> > - if (ret)
> > - return ret;
> > -
> > - xe_tlb_inval_fence_wait(&fence);
> > - } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
> > - struct xe_mmio *mmio = >->mmio;
> > -
> > - if (IS_SRIOV_VF(xe))
> > - return 0;
> > -
> > - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
> > - if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
> > - xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
> > - PVC_GUC_TLB_INV_DESC1_INVALIDATE);
> > - xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0,
> > - PVC_GUC_TLB_INV_DESC0_VALID);
> > - } else {
> > - xe_mmio_write32(mmio, GUC_TLB_INV_CR,
> > - GUC_TLB_INV_CR_INVALIDATE);
> > - }
> > - xe_force_wake_put(gt_to_fw(gt), fw_ref);
> > - }
> > + struct xe_tlb_inval_fence fence, *fence_ptr = &fence;
> > + int ret;
> >
> > - return 0;
> > + xe_tlb_inval_fence_init(tlb_inval, fence_ptr, true);
> > + ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, tlb_inval->ops->ggtt);
> > + xe_tlb_inval_fence_wait(fence_ptr);
> > +
> > + return ret;
> > }
> >
> > /**
> > - * xe_tlb_inval_range - Issue a TLB invalidation on this GT for an address range
> > + * xe_tlb_inval_range() - Issue a TLB invalidation for an address range
> > * @tlb_inval: TLB invalidation client
> > * @fence: invalidation fence which will be signal on TLB invalidation
> > * completion
> > @@ -460,33 +288,12 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
> > struct xe_tlb_inval_fence *fence, u64 start, u64 end,
> > u32 asid)
> > {
> > - struct xe_gt *gt = tlb_inval->private;
> > - struct xe_device *xe = gt_to_xe(gt);
> > - int ret;
> > -
> > - xe_gt_assert(gt, fence);
> > -
> > - /* Execlists not supported */
> > - if (xe->info.force_execlist) {
> > - __inval_fence_signal(xe, fence);
> > - return 0;
> > - }
> > -
> > - mutex_lock(>->uc.guc.ct.lock);
> > -
> > - xe_tlb_inval_fence_prep(fence);
> > -
> > - ret = send_tlb_inval_ppgtt(gt, start, end, asid, fence->seqno);
> > - if (ret < 0)
> > - inval_fence_signal_unlocked(xe, fence);
> > -
> > - mutex_unlock(>->uc.guc.ct.lock);
> > -
> > - return ret;
> > + return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
> > + start, end, asid);
> > }
> >
> > /**
> > - * xe_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM
> > + * xe_tlb_inval_vm() - Issue a TLB invalidation for a VM
> > * @tlb_inval: TLB invalidation client
> > * @vm: VM to invalidate
> > *
> > @@ -496,27 +303,22 @@ void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm)
> > {
> > struct xe_tlb_inval_fence fence;
> > u64 range = 1ull << vm->xe->info.va_bits;
> > - int ret;
> >
> > xe_tlb_inval_fence_init(tlb_inval, &fence, true);
> > -
> > - ret = xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid);
> > - if (ret < 0)
> > - return;
> > -
> > + xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid);
> > xe_tlb_inval_fence_wait(&fence);
> > }
> >
> > /**
> > - * xe_tlb_inval_done_handler - TLB invalidation done handler
> > - * @gt: gt
> > + * xe_tlb_inval_done_handler() - TLB invalidation done handler
> > + * @tlb_inval: TLB invalidation client
> > * @seqno: seqno of invalidation that is done
> > *
> > * Update recv seqno, signal any TLB invalidation fences, and restart TDR
>
> I'd mention that is function is safe be called from any context (i.e.,
> process, atomic, and hardirq contexts are allowed).
>
> We might need to convert tlb_inval.pending_lock to a raw_spinlock_t for
> PREEMPT_RT enablement. Same for the GuC fast_lock. AFAIK we haven’t had
> any complaints, so maybe I’m just overthinking it, but also perhaps not.
>
> > */
> > -static void xe_tlb_inval_done_handler(struct xe_gt *gt, int seqno)
> > +void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno)
> > {
> > - struct xe_device *xe = gt_to_xe(gt);
> > + struct xe_device *xe = tlb_inval->xe;
> > struct xe_tlb_inval_fence *fence, *next;
> > unsigned long flags;
> >
> > @@ -535,77 +337,53 @@ static void xe_tlb_inval_done_handler(struct xe_gt *gt, int seqno)
> > * officially process the CT message like if racing against
> > * process_g2h_msg().
> > */
> > - spin_lock_irqsave(>->tlb_inval.pending_lock, flags);
> > - if (tlb_inval_seqno_past(gt, seqno)) {
> > - spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags);
> > + spin_lock_irqsave(&tlb_inval->pending_lock, flags);
> > + if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) {
> > + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
> > return;
> > }
> >
> > - WRITE_ONCE(gt->tlb_inval.seqno_recv, seqno);
> > + WRITE_ONCE(tlb_inval->seqno_recv, seqno);
> >
> > list_for_each_entry_safe(fence, next,
> > - >->tlb_inval.pending_fences, link) {
> > + &tlb_inval->pending_fences, link) {
> > trace_xe_tlb_inval_fence_recv(xe, fence);
> >
> > - if (!tlb_inval_seqno_past(gt, fence->seqno))
> > + if (!xe_tlb_inval_seqno_past(tlb_inval, fence->seqno))
> > break;
> >
> > - inval_fence_signal(xe, fence);
> > + xe_tlb_inval_fence_signal(fence);
> > }
> >
> > - if (!list_empty(>->tlb_inval.pending_fences))
> > + if (!list_empty(&tlb_inval->pending_fences))
> > mod_delayed_work(system_wq,
> > - >->tlb_inval.fence_tdr,
> > - tlb_timeout_jiffies(gt));
> > + &tlb_inval->fence_tdr,
> > + tlb_inval->ops->timeout_delay(tlb_inval));
> > else
> > - cancel_delayed_work(>->tlb_inval.fence_tdr);
> > + cancel_delayed_work(&tlb_inval->fence_tdr);
> >
> > - spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags);
> > -}
> > -
> > -/**
> > - * xe_guc_tlb_inval_done_handler - TLB invalidation done handler
> > - * @guc: guc
> > - * @msg: message indicating TLB invalidation done
> > - * @len: length of message
> > - *
> > - * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any
> > - * invalidation fences for seqno. Algorithm for this depends on seqno being
> > - * received in-order and asserts this assumption.
> > - *
> > - * Return: 0 on success, -EPROTO for malformed messages.
> > - */
> > -int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
> > -{
> > - struct xe_gt *gt = guc_to_gt(guc);
> > -
> > - if (unlikely(len != 1))
> > - return -EPROTO;
> > -
> > - xe_tlb_inval_done_handler(gt, msg[0]);
> > -
> > - return 0;
> > + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
> > }
> >
> > static const char *
> > -inval_fence_get_driver_name(struct dma_fence *dma_fence)
> > +xe_inval_fence_get_driver_name(struct dma_fence *dma_fence)
> > {
> > return "xe";
> > }
> >
> > static const char *
> > -inval_fence_get_timeline_name(struct dma_fence *dma_fence)
> > +xe_inval_fence_get_timeline_name(struct dma_fence *dma_fence)
> > {
> > - return "inval_fence";
> > + return "tlb_inval_fence";
> > }
> >
> > static const struct dma_fence_ops inval_fence_ops = {
> > - .get_driver_name = inval_fence_get_driver_name,
> > - .get_timeline_name = inval_fence_get_timeline_name,
> > + .get_driver_name = xe_inval_fence_get_driver_name,
> > + .get_timeline_name = xe_inval_fence_get_timeline_name,
> > };
> >
> > /**
> > - * xe_tlb_inval_fence_init - Initialize TLB invalidation fence
> > + * xe_tlb_inval_fence_init() - Initialize TLB invalidation fence
> > * @tlb_inval: TLB invalidation client
> > * @fence: TLB invalidation fence to initialize
> > * @stack: fence is stack variable
> > @@ -618,15 +396,12 @@ void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
> > struct xe_tlb_inval_fence *fence,
> > bool stack)
> > {
> > - struct xe_gt *gt = tlb_inval->private;
> > -
> > - xe_pm_runtime_get_noresume(gt_to_xe(gt));
> > + xe_pm_runtime_get_noresume(tlb_inval->xe);
> >
> > - spin_lock_irq(>->tlb_inval.lock);
> > - dma_fence_init(&fence->base, &inval_fence_ops,
> > - >->tlb_inval.lock,
> > + spin_lock_irq(&tlb_inval->lock);
> > + dma_fence_init(&fence->base, &inval_fence_ops, &tlb_inval->lock,
> > dma_fence_context_alloc(1), 1);
> > - spin_unlock_irq(>->tlb_inval.lock);
> > + spin_unlock_irq(&tlb_inval->lock);
>
> While here, 'fence_lock' is probably a better name.
>
> Matt
>
> > INIT_LIST_HEAD(&fence->link);
> > if (stack)
> > set_bit(FENCE_STACK_BIT, &fence->base.flags);
> > diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h
> > index 7adee3f8c551..cdeafc8d4391 100644
> > --- a/drivers/gpu/drm/xe/xe_tlb_inval.h
> > +++ b/drivers/gpu/drm/xe/xe_tlb_inval.h
> > @@ -18,24 +18,30 @@ struct xe_vma;
> > int xe_gt_tlb_inval_init_early(struct xe_gt *gt);
> >
> > void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval);
> > -int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval);
> > -void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm);
> > int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
> > struct xe_tlb_inval_fence *fence);
> > +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval);
> > +void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm);
> > int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
> > struct xe_tlb_inval_fence *fence,
> > u64 start, u64 end, u32 asid);
> > -int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
> >
> > void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
> > struct xe_tlb_inval_fence *fence,
> > bool stack);
> > -void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence);
> >
> > +/**
> > + * xe_tlb_inval_fence_wait() - TLB invalidiation fence wait
> > + * @fence: TLB invalidation fence to wait on
> > + *
> > + * Wait on a TLB invalidiation fence until it signals, non interruptable
> > + */
> > static inline void
> > xe_tlb_inval_fence_wait(struct xe_tlb_inval_fence *fence)
> > {
> > dma_fence_wait(&fence->base, false);
> > }
> >
> > +void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno);
> > +
> > #endif /* _XE_TLB_INVAL_ */
> > diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
> > index 05b6adc929bb..c1ad96d24fc8 100644
> > --- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h
> > +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
> > @@ -9,10 +9,85 @@
> > #include <linux/workqueue.h>
> > #include <linux/dma-fence.h>
> >
> > -/** struct xe_tlb_inval - TLB invalidation client */
> > +struct xe_tlb_inval;
> > +
> > +/** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */
> > +struct xe_tlb_inval_ops {
> > + /**
> > + * @all: Invalidate all TLBs
> > + * @tlb_inval: TLB invalidation client
> > + * @seqno: Seqno of TLB invalidation
> > + *
> > + * Return 0 on success, -ECANCELED if backend is mid-reset, error on
> > + * failure
> > + */
> > + int (*all)(struct xe_tlb_inval *tlb_inval, u32 seqno);
> > +
> > + /**
> > + * @ggtt: Invalidate global translation TLBs
> > + * @tlb_inval: TLB invalidation client
> > + * @seqno: Seqno of TLB invalidation
> > + *
> > + * Return 0 on success, -ECANCELED if backend is mid-reset, error on
> > + * failure
> > + */
> > + int (*ggtt)(struct xe_tlb_inval *tlb_inval, u32 seqno);
> > +
> > + /**
> > + * @ppttt: Invalidate per-process translation TLBs
> > + * @tlb_inval: TLB invalidation client
> > + * @seqno: Seqno of TLB invalidation
> > + * @start: Start address
> > + * @end: End address
> > + * @asid: Address space ID
> > + *
> > + * Return 0 on success, -ECANCELED if backend is mid-reset, error on
> > + * failure
> > + */
> > + int (*ppgtt)(struct xe_tlb_inval *tlb_inval, u32 seqno, u64 start,
> > + u64 end, u32 asid);
> > +
> > + /**
> > + * @initialized: Backend is initialized
> > + * @tlb_inval: TLB invalidation client
> > + *
> > + * Return: True if back is initialized, False otherwise
> > + */
> > + bool (*initialized)(struct xe_tlb_inval *tlb_inval);
> > +
> > + /**
> > + * @flush: Flush pending TLB invalidations
> > + * @tlb_inval: TLB invalidation client
> > + */
> > + void (*flush)(struct xe_tlb_inval *tlb_inval);
> > +
> > + /**
> > + * @timeout_delay: Timeout delay for TLB invalidation
> > + * @tlb_inval: TLB invalidation client
> > + *
> > + * Return: Timeout delay for TLB invalidation in jiffies
> > + */
> > + long (*timeout_delay)(struct xe_tlb_inval *tlb_inval);
> > +
> > + /**
> > + * @lock: Lock resources protecting the backend seqno management
> > + */
> > + void (*lock)(struct xe_tlb_inval *tlb_inval);
> > +
> > + /**
> > + * @unlock: Lock resources protecting the backend seqno management
> > + */
> > + void (*unlock)(struct xe_tlb_inval *tlb_inval);
> > +};
> > +
> > +/** struct xe_tlb_inval - TLB invalidation client (frontend) */
> > struct xe_tlb_inval {
> > /** @private: Backend private pointer */
> > void *private;
> > + /** @xe: Pointer to Xe device */
> > + struct xe_device *xe;
> > + /** @ops: TLB invalidation ops */
> > + const struct xe_tlb_inval_ops *ops;
> > /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */
> > #define TLB_INVALIDATION_SEQNO_MAX 0x100000
> > int seqno;
> > --
> > 2.34.1
> >
More information about the Intel-xe
mailing list