[RFC PATCH 8/8] drm/xe: Allow mapping DMA-BUF buffer objects connected through IAF
Maarten Lankhorst
dev at lankhorst.se
Fri Jun 13 13:45:27 UTC 2025
IAF allows multiple xe devices to import VRAM from each other.
It's mapped as local VRAM, but outside the local device range.
To support this, Xe is changed to allow a special case of
same-implementation importing with direction = DMA_NONE.
If this DMA direction is used, the physical VRAM addresses are exported,
and can be imported into a xe_vm with the device memory bit set.
In order to prevent bugs, we ensure that this is only allowed for
xe <-> xe imports/exports, and only when the VRAM flag is set.
If any of the steps required fails for IAF-import, the default
non-IAF import path is attempted.
Signed-off-by: Maarten Lankhorst <dev at lankhorst.se>
---
drivers/gpu/drm/xe/xe_bo.c | 85 ++++++++++++++++++++++++----
drivers/gpu/drm/xe/xe_bo.h | 2 +
drivers/gpu/drm/xe/xe_dma_buf.c | 27 ++++++++-
drivers/gpu/drm/xe/xe_dma_buf.h | 1 +
drivers/gpu/drm/xe/xe_ggtt.c | 4 +-
drivers/gpu/drm/xe/xe_iaf.c | 42 ++++++++++++++
drivers/gpu/drm/xe/xe_iaf.h | 16 ++++++
drivers/gpu/drm/xe/xe_pt.c | 4 ++
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 23 +++++---
drivers/gpu/drm/xe/xe_vm.c | 2 +-
10 files changed, 184 insertions(+), 22 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 4e39188a021ab..13f0c685a012f 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -24,6 +24,7 @@
#include "xe_drm_client.h"
#include "xe_ggtt.h"
#include "xe_gt.h"
+#include "xe_iaf.h"
#include "xe_map.h"
#include "xe_migrate.h"
#include "xe_pm.h"
@@ -82,6 +83,11 @@ bool mem_type_is_vram(u32 mem_type)
return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;
}
+bool xe_bo_is_iaf(struct xe_bo *bo)
+{
+ return bo->flags & XE_BO_FLAG_IAF;
+}
+
static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)
{
return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);
@@ -656,6 +662,68 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
return ret;
}
+static void bo_unmap_dma_buf(struct xe_bo *bo)
+{
+ enum dma_data_direction dir = DMA_BIDIRECTIONAL;
+ struct dma_buf_attachment *attach = bo->ttm.base.import_attach;
+ struct dma_buf *dmabuf;
+
+ if (xe_bo_is_iaf(bo))
+ dir = DMA_NONE;
+
+ dma_buf_unmap_attachment(attach, bo->ttm.sg, dir);
+ bo->ttm.sg = NULL;
+
+ if (xe_bo_is_iaf(bo)) {
+ /* Cleanup IAF bindings */
+ dmabuf = attach->dmabuf;
+ xe_iaf_mapping_put(xe_bo_device(gem_to_xe_bo(dmabuf->priv))->iaf);
+ xe_iaf_mapping_put(xe_bo_device(bo)->iaf);
+ bo->flags &= ~XE_BO_FLAG_IAF;
+ }
+}
+
+static struct sg_table *bo_map_dma_buf(struct xe_bo *bo)
+{
+ struct dma_buf_attachment *attach = bo->ttm.base.import_attach;
+ struct dma_buf *dmabuf = attach->dmabuf;
+ struct sg_table *sg;
+
+ if (is_xe_dma_buf(dmabuf)) {
+ struct xe_device *src = xe_bo_device(gem_to_xe_bo(dmabuf->priv));
+ struct xe_device *dst = xe_bo_device(bo);
+ int err;
+
+ if (!xe_iaf_fabrics_connected(dst->iaf, src->iaf))
+ goto no_iaf;
+
+ err = xe_iaf_mapping_get(dst->iaf);
+ if (err) {
+ drm_warn(&dst->drm, "Getting DST IAF mapping ref failed with %pe", ERR_PTR(err));
+ goto no_iaf;
+ }
+
+ err = xe_iaf_mapping_get(src->iaf);
+ if (err) {
+ drm_warn(&src->drm, "Getting SRC IAF mapping ref failed with %pe", ERR_PTR(err));
+ goto err_put_dst;
+ }
+
+ sg = dma_buf_map_attachment(attach, DMA_NONE);
+ if (!IS_ERR(sg)) {
+ bo->flags |= XE_BO_FLAG_IAF;
+ return sg;
+ }
+
+ xe_iaf_mapping_put(src->iaf);
+err_put_dst:
+ xe_iaf_mapping_put(dst->iaf);
+ }
+
+no_iaf:
+ return dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+}
+
/*
* The dma-buf map_attachment() / unmap_attachment() is hooked up here.
* Note that unmapping the attachment is deferred to the next
@@ -665,9 +733,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
* backing store out. Should that assumption not hold, then we will be able
* to unconditionally call unmap_attachment() when moving out to system.
*/
-static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
- struct ttm_resource *new_res)
+static int xe_bo_move_dmabuf(struct xe_bo *bo, struct ttm_resource *new_res)
{
+ struct ttm_buffer_object *ttm_bo = &bo->ttm;
struct dma_buf_attachment *attach = ttm_bo->base.import_attach;
struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt,
ttm);
@@ -682,19 +750,16 @@ static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
ttm_bo->sg) {
dma_resv_wait_timeout(ttm_bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
false, MAX_SCHEDULE_TIMEOUT);
- dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
- ttm_bo->sg = NULL;
+ bo_unmap_dma_buf(bo);
}
if (new_res->mem_type == XE_PL_SYSTEM)
goto out;
- if (ttm_bo->sg) {
- dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
- ttm_bo->sg = NULL;
- }
+ if (ttm_bo->sg)
+ bo_unmap_dma_buf(bo);
- sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+ sg = bo_map_dma_buf(bo);
if (IS_ERR(sg))
return PTR_ERR(sg);
@@ -797,7 +862,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
if (ttm_bo->type == ttm_bo_type_sg) {
ret = xe_bo_move_notify(bo, ctx);
if (!ret)
- ret = xe_bo_move_dmabuf(ttm_bo, new_mem);
+ ret = xe_bo_move_dmabuf(bo, new_mem);
return ret;
}
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index ecb5fe3b1c1dc..210ee49d6a04d 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -44,6 +44,7 @@
#define XE_BO_FLAG_GGTT2 BIT(22)
#define XE_BO_FLAG_GGTT3 BIT(23)
#define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24)
+#define XE_BO_FLAG_IAF BIT(25)
/* this one is trigger internally only */
#define XE_BO_FLAG_INTERNAL_TEST BIT(30)
@@ -262,6 +263,7 @@ void xe_bo_vunmap(struct xe_bo *bo);
int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size);
bool mem_type_is_vram(u32 mem_type);
+bool xe_bo_is_iaf(struct xe_bo *bo);
bool xe_bo_is_vram(struct xe_bo *bo);
bool xe_bo_is_stolen(struct xe_bo *bo);
bool xe_bo_is_stolen_devmem(struct xe_bo *bo);
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 346f857f38374..2e8aca5d572ae 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -86,6 +86,8 @@ static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
xe_bo_unpin_external(bo);
}
+static const struct dma_buf_attach_ops xe_dma_buf_attach_ops;
+
static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
enum dma_data_direction dir)
{
@@ -94,12 +96,21 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
struct xe_bo *bo = gem_to_xe_bo(obj);
struct sg_table *sgt;
int r = 0;
+ bool iaf = false;
+
+ if (dir == DMA_NONE) {
+ if (attach->importer_ops != &xe_dma_buf_attach_ops)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ iaf = true;
+ }
- if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
+ if (!attach->peer2peer && !iaf &&
+ !xe_bo_can_migrate(bo, XE_PL_TT))
return ERR_PTR(-EOPNOTSUPP);
if (!xe_bo_is_pinned(bo)) {
- if (!attach->peer2peer)
+ if (!attach->peer2peer && !iaf)
r = xe_bo_migrate(bo, XE_PL_TT);
else
r = xe_bo_validate(bo, NULL, false);
@@ -109,6 +120,10 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
switch (bo->ttm.resource->mem_type) {
case XE_PL_TT:
+ /* IAF only available for VRAM */
+ if (iaf)
+ return ERR_PTR(-EOPNOTSUPP);
+
sgt = drm_prime_pages_to_sg(obj->dev,
bo->ttm.ttm->pages,
bo->ttm.ttm->num_pages);
@@ -146,7 +161,8 @@ static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
enum dma_data_direction dir)
{
if (sg_page(sgt->sgl)) {
- dma_unmap_sgtable(attach->dev, sgt, dir, 0);
+ if (dir != DMA_NONE)
+ dma_unmap_sgtable(attach->dev, sgt, dir, 0);
sg_free_table(sgt);
kfree(sgt);
} else {
@@ -187,6 +203,11 @@ static const struct dma_buf_ops xe_dmabuf_ops = {
.vunmap = drm_gem_dmabuf_vunmap,
};
+bool is_xe_dma_buf(struct dma_buf *dma_buf)
+{
+ return dma_buf->ops == &xe_dmabuf_ops;
+}
+
struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
{
struct xe_bo *bo = gem_to_xe_bo(obj);
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.h b/drivers/gpu/drm/xe/xe_dma_buf.h
index 861dd28a862c7..80d12c624b921 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.h
+++ b/drivers/gpu/drm/xe/xe_dma_buf.h
@@ -11,5 +11,6 @@
struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags);
struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
struct dma_buf *dma_buf);
+bool is_xe_dma_buf(struct dma_buf *dma_buf);
#endif
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index b8e1b44452e4d..c6ff18deb9db1 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -122,7 +122,9 @@ static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
{
u64 pte = XE_PAGE_PRESENT;
- if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
+ if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo) ||
+ /* It's likely legal to map IAF to GGTT, but it should be impossible? */
+ drm_WARN_ON(bo->ttm.base.dev, xe_bo_is_iaf(bo)))
pte |= XE_GGTT_PTE_DM;
return pte;
diff --git a/drivers/gpu/drm/xe/xe_iaf.c b/drivers/gpu/drm/xe/xe_iaf.c
index 431d20fea5369..af2a8d045b593 100644
--- a/drivers/gpu/drm/xe/xe_iaf.c
+++ b/drivers/gpu/drm/xe/xe_iaf.c
@@ -416,3 +416,45 @@ struct query_info *xe_iaf_connectivity_query(struct xe_iaf *iaf, u32 fabric_id)
return iaf->ops->connectivity_query(iaf->handle, fabric_id);
}
+
+bool xe_iaf_fabrics_connected(struct xe_iaf *src, struct xe_iaf *dst)
+{
+ struct query_info *qi;
+ bool connected;
+
+ if (!src || !dst || !src->ops || !dst->ops)
+ return false;
+
+ qi = xe_iaf_connectivity_query(src, dst->fabric_id);
+ if (IS_ERR(qi))
+ return false;
+
+ if (WARN_ON_ONCE(!qi))
+ return true;
+
+ connected = true;
+ for (int i = 0, n = qi->src_cnt * qi->dst_cnt; i < n; i++)
+ if (!qi->sd2sd[i].bandwidth) {
+ connected = false;
+ break;
+ }
+
+ kfree(qi);
+ return connected;
+}
+
+int xe_iaf_mapping_get(struct xe_iaf *iaf)
+{
+ if (!iaf || !iaf->ops)
+ return -ENODEV;
+
+ return iaf->ops->parent_event(iaf->handle, IAF_PARENT_MAPPING_GET);
+}
+
+void xe_iaf_mapping_put(struct xe_iaf *iaf)
+{
+ if (!iaf || !iaf->ops)
+ return;
+
+ iaf->ops->parent_event(iaf->handle, IAF_PARENT_MAPPING_PUT);
+}
diff --git a/drivers/gpu/drm/xe/xe_iaf.h b/drivers/gpu/drm/xe/xe_iaf.h
index df6b8f9f2bc5f..b8e75640e8967 100644
--- a/drivers/gpu/drm/xe/xe_iaf.h
+++ b/drivers/gpu/drm/xe/xe_iaf.h
@@ -38,6 +38,9 @@ int xe_iaf_init(struct xe_device *xe);
int xe_iaf_init_aux(struct xe_device *xe);
u64 xe_iaf_dpa_base(struct xe_device *xe);
struct query_info *xe_iaf_connectivity_query(struct xe_iaf *iaf, u32 fabric_id);
+bool xe_iaf_fabrics_connected(struct xe_iaf *src, struct xe_iaf *dst);
+int xe_iaf_mapping_get(struct xe_iaf *iaf);
+void xe_iaf_mapping_put(struct xe_iaf *iaf);
#else
@@ -61,6 +64,19 @@ struct query_info *xe_iaf_connectivity_query(struct xe_iaf *iaf, u32 fabric_id)
return ERR_PTR(-ENODEV);
}
+static inline int xe_iaf_mapping_get(struct xe_iaf *iaf)
+{
+ return -ENODEV;
+}
+
+static inline void xe_iaf_mapping_put(struct xe_iaf *iaf)
+{}
+
+static inline bool xe_iaf_fabrics_connected(struct xe_iaf *src, struct xe_iaf *dst)
+{
+ return false;
+}
+
#endif
#endif
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index f39d5cc9f411e..2231882b2bc45 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -764,6 +764,10 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
else
xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
xe_vma_size(vma), &curs);
+
+ /* When IAF is used, the bo requires the DM flag */
+ if (xe_bo_is_iaf(bo))
+ curs.mem_type = XE_PL_VRAM0;
} else if (!range) {
curs.size = xe_vma_size(vma);
}
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 9e375a40aee90..94a572693e139 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -396,11 +396,16 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
size_t size = min_t(u64, cursor.size, SZ_2G);
dma_addr_t addr;
- addr = dma_map_resource(dev, phys, size, dir,
- DMA_ATTR_SKIP_CPU_SYNC);
- r = dma_mapping_error(dev, addr);
- if (r)
- goto error_unmap;
+ if (valid_dma_direction(dir)) {
+ addr = dma_map_resource(dev, phys, size, dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ r = dma_mapping_error(dev, addr);
+ if (r)
+ goto error_unmap;
+ } else {
+ /* Only want the SG table for fabric */
+ addr = cursor.start + tile->mem.vram.dpa_base;
+ }
sg_set_page(sg, NULL, size, 0);
sg_dma_address(sg) = addr;
@@ -413,7 +418,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
error_unmap:
for_each_sgtable_sg((*sgt), sg, i) {
- if (!sg->length)
+ if (!sg->length || !valid_dma_direction(dir))
continue;
dma_unmap_resource(dev, sg->dma_address,
@@ -433,10 +438,14 @@ void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,
struct scatterlist *sg;
int i;
- for_each_sgtable_sg(sgt, sg, i)
+ for_each_sgtable_sg(sgt, sg, i) {
+ if (!valid_dma_direction(dir))
+ continue;
+
dma_unmap_resource(dev, sg->dma_address,
sg->length, dir,
DMA_ATTR_SKIP_CPU_SYNC);
+ }
sg_free_table(sgt);
kfree(sgt);
}
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 18f967ce1f1a6..f3dd38c95deb5 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1534,7 +1534,7 @@ static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
pte |= pte_encode_pat_index(pat_index, pt_level);
pte |= pte_encode_ps(pt_level);
- if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
+ if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo) || xe_bo_is_iaf(bo))
pte |= XE_PPGTT_PTE_DM;
return pte;
--
2.45.2
More information about the dri-devel
mailing list