[Intel-xe] [PATCH v2 1/3] drm/xe: Introduce XeLink device
Ruhl, Michael J
michael.j.ruhl at intel.com
Tue Nov 28 22:05:58 UTC 2023
One comment...
>-----Original Message-----
>From: Kershner, David <david.kershner at intel.com>
>Sent: Monday, November 27, 2023 4:01 PM
>To: Kershner, David <david.kershner at intel.com>; intel-
>xe at lists.freedesktop.org; Ruhl, Michael J <michael.j.ruhl at intel.com>; Fleck,
>John <john.fleck at intel.com>; De Marchi, Lucas <lucas.demarchi at intel.com>;
>Vivi, Rodrigo <rodrigo.vivi at intel.com>; Roper, Matthew D
><matthew.d.roper at intel.com>
>Subject: [PATCH v2 1/3] drm/xe: Introduce XeLink device
>
>Add XeLink support for PVC devices.
>
>Introduce the initial platform data structure that will be shared
>between the Xe driver and the XeLink driver.
>
>If the device is PVC, and there is an XeLink device present, add
>and initialize an Auxbus device with the appropriate resources.
>
>Add the XeLink information register.
>
>Add package address register for defining the Device Physical Address
>space (DPA).
>
>Add initial interrupt support.
>
>The package address register defines the device physical
>address (DPA) space for the given device.
>
>Using the "unique" xa_array index, generate a DPA address
>for the device and program the package register appropriately.
>
>Update the LMEM offset to use the DPA address.
>
>Signed-off-by: Michael J. Ruhl <michael.j.ruhl at intel.com>
>Signed-off-by: David Kershner <david.kershner at intel.com>
>---
> drivers/gpu/drm/xe/Makefile | 1 +
> drivers/gpu/drm/xe/regs/xe_gt_regs.h | 30 ++
> drivers/gpu/drm/xe/xe_device.c | 13 +-
> drivers/gpu/drm/xe/xe_device_types.h | 25 ++
> drivers/gpu/drm/xe/xe_gt_types.h | 2 +
> drivers/gpu/drm/xe/xe_irq.c | 22 ++
> drivers/gpu/drm/xe/xe_link.c | 449 +++++++++++++++++++++++++++
> drivers/gpu/drm/xe/xe_link.h | 36 +++
> drivers/gpu/drm/xe/xe_mmio.c | 3 +-
> drivers/gpu/drm/xe/xe_pci.c | 2 +
> drivers/gpu/drm/xe/xe_pci_types.h | 1 +
> include/drm/xelink_platform.h | 140 +++++++++
> 12 files changed, 721 insertions(+), 3 deletions(-)
> create mode 100644 drivers/gpu/drm/xe/xe_link.c
> create mode 100644 drivers/gpu/drm/xe/xe_link.h
> create mode 100644 include/drm/xelink_platform.h
<snip>
>+/**
>+ * init_resource - Create the resource array, and apply the appropriate data
>+ * @xe: valid xe instance
>+ * @res_cnt: pointer to return number of allocated resources
>+ *
>+ * First resource [0] is for the IRQ(s). Each device gets 1 IRQ. Subsequent
>+ * resources describe the IO memory space for the device(s).
>+ *
>+ * Make sure to set the gt->xelink_irq value.
>+ *
>+ * Return:
>+ * * res - Initialized resource array
>+ * * NULL - Allocaction failure
>+ */
>+static struct resource *init_resource(struct xe_device *xe,
>+ u32 *res_cnt)
>+{
>+ struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
>+ struct xe_gt *gt;
>+ struct resource *res_base, *res;
>+ u32 cnt = xe->info.tile_count * 2;
>+ unsigned int i;
>+
>+ /* Each sd gets one resource for IRQ and one for MEM */
>+ res_base = kcalloc(cnt, sizeof(*res_base), GFP_KERNEL);
>+ if (!res_base)
>+ return NULL;
>+
>+ res = res_base;
>+ for_each_gt(gt, xe, i) {
>+ res->start = xe->xelink.irq_base + i;
>+ res->end = xe->xelink.irq_base + i;
>+ res->flags = IORESOURCE_IRQ;
>+ res++;
>+
>+ res->start = pci_resource_start(pdev, GTTMMADR_BAR) +
This should be tile->vram->io_start?
M
>CD_BASE_OFFSET +
>+ i * gt_to_tile(gt)->mmio.size;
>+ res->end = res->start + CD_BAR_SIZE - 1;
>+ res->flags = IORESOURCE_MEM;
>+ drm_info(&xe->drm, "XeLink: mem_resource = %pR\n", res);
>+ res++;
>+ gt->xelink_irq = xe->xelink.irq_base + i;
>+ }
>+
>+ *res_cnt = cnt;
>+ return res_base;
>+}
>+
>+/**
>+ * xelink_irq_mask - Null callback. Masking/unmasking happens in the parent
>+ * driver
>+ * @d: Valid irq_data information
>+ */
>+static void xelink_irq_mask(struct irq_data *d)
>+{
>+}
>+
>+static void xelink_irq_unmask(struct irq_data *d)
>+{
>+}
>+
>+static struct irq_chip xelink_irq_chip = {
>+ .name = "xelink_irq_chip",
>+ .irq_mask = xelink_irq_mask,
>+ .irq_unmask = xelink_irq_unmask,
>+};
>+
>+/**
>+ * init_irq_desc - Allocate IRQ descriptors to use for the xelink
>+ * @xe: Valid xe instance
>+ *
>+ * Allocate the required IRQ descriptor(s) and initialize the
>+ * appropriate state.
>+ *
>+ * Return:
>+ * * 0 - Success
>+ * * errno - Error that occurred
>+ */
>+static int init_irq_desc(struct xe_device *xe)
>+{
>+ unsigned int num_subdevs = xe->info.tile_count;
>+ int err;
>+ int irq;
>+ int irq_base;
>+
>+ irq_base = irq_alloc_descs(-1, 0, num_subdevs, 0);
>+ if (irq_base < 0) {
>+ err = irq_base;
>+ goto cleanup;
>+ }
>+
>+ err = 0;
>+ for (irq = irq_base; !err && irq < irq_base + num_subdevs; irq++) {
>+ irq_set_chip_and_handler_name(irq, &xelink_irq_chip,
>+ handle_simple_irq,
>+ "xelink_irq_handler");
>+ err = irq_set_chip_data(irq, xe);
>+ }
>+
>+ if (err) {
>+ irq_free_descs(irq_base, num_subdevs);
>+ goto cleanup;
>+ }
>+
>+ drm_info(&xe->drm, "XeLink: IRQ base: %d cnt: %d\n", irq_base,
>+ num_subdevs);
>+
>+ xe->xelink.irq_base = irq_base;
>+
>+ return 0;
>+
>+cleanup:
>+ xe->xelink.index = err;
>+ drm_err(&xe->drm, "XeLink: Failed to allocate IRQ data: %d\n", err);
>+ return err;
>+}
>+
>+/**
>+ * xe_link_init_early - Set the XeLink info to the defaults
>+ * @xe: valid xe instance
>+ *
>+ * index is set to ENODEV to show that, by default, there is no device.
>+ * If any of the initialization steps fail, it will be set to the appropriate
>+ * errno value.
>+ */
>+void xe_link_init_early(struct xe_device *xe)
>+{
>+ xe->xelink.ops = &default_ops;
>+ xe->xelink.index = -ENODEV;
>+}
>+
>+/**
>+ * xe_link_init_mmio - check if XeLink is available via MMIO
>+ * @xe: valid xe instance
>+ *
>+ * Read the relevant regs to check for XeLink availability and get the socket id
>+ */
>+void xe_link_init_mmio(struct xe_device *xe)
>+{
>+ u32 xelink_info;
>+
>+ if (!HAS_XELINK(xe))
>+ return;
>+
>+ xelink_info = xe_mmio_read32(xe_device_get_root_tile(xe)-
>>primary_gt,
>+ PUNIT_MMIO_CR_POC_STRAPS);
>+
>+ xe->xelink.socket_id = REG_FIELD_GET(SOCKET_ID_MASK, xelink_info);
>+
>+ if (REG_FIELD_GET(CD_ALIVE, xelink_info)) {
>+ drm_info(&xe->drm, "XeLink available\n");
>+ xe->xelink.present = true;
>+ }
>+}
>+
>+/**
>+ * xe_link_init - Allocate device index and complete initial HW setup
>+ * @xe: valid device instance
>+ *
>+ * NOTE: index is zero inited. If the XeLink is not present, or an error occurs
>+ * during setup, this must be 0 for the range registers.
>+ *
>+ */
>+void xe_link_init(struct xe_device *xe)
>+{
>+ struct xe_gt *gt;
>+ static u32 last_id;
>+ unsigned int i;
>+ u32 index = 0;
>+ u32 range;
>+ u32 base;
>+ int err;
>+
>+ if (!HAS_XELINK(xe))
>+ return;
>+
>+ if (!xe->xelink.present)
>+ goto set_range;
>+
>+ err = init_irq_desc(xe);
>+ if (err) {
>+ xe->xelink.index = err;
>+ goto set_range;
>+ }
>+
>+ /*
>+ * Try the socket id first. Systems with this feature, will
>+ * get a deterministic value. If not, try with the cyclic.
>+ */
>+ err = xa_insert(&intel_fdevs, xe->xelink.socket_id, xe,
>+ GFP_KERNEL);
>+ if (!err)
>+ index = xe->xelink.socket_id;
>+
>+ /* socket_id is not available */
>+ if (err == -EBUSY) {
>+ /*
>+ * NOTE: index is only updated on success i.e. >= 0
>+ * < 0 err, 0 ok, > 0 wrapped
>+ */
>+ err = xa_alloc_cyclic(&intel_fdevs, &index, xe,
>+ XA_LIMIT(0, MAX_DEVICE_COUNT - 1),
>+ &last_id, GFP_KERNEL);
>+ }
>+ if (err < 0) {
>+ index = 0;
>+ xe->xelink.index = err;
>+ drm_err(&xe->drm,
>+ "XeLink: Failed to allocate index: %d\n",
>+ err);
>+ irq_free_descs(xe->xelink.irq_base,
>+ xe->info.tile_count);
>+ goto set_range;
>+ }
>+ xe->xelink.index = index;
>+ xe->xelink.dpa = (u64)index * MAX_DPA_SIZE * SZ_1G;
>+ drm_info(&xe->drm, "XeLink: [dpa 0x%llx-0x%llx\n", xe->xelink.dpa,
>+ ((u64)index + 1) * MAX_DPA_SIZE * SZ_1G - 1);
>+
>+ /*
>+ * Set range has to be done for all devices that support device
>+ * address space, regardless of presence or error.
>+ */
>+set_range:
>+ /* Set GAM address range registers */
>+ range = index * MAX_DPA_SIZE << PKG_ADDR_RANGE_BASE_SHIFT;
>+ range |= MAX_DPA_SIZE << PKG_ADDR_RANGE_RANGE_SHIFT;
>+ range |= PKG_ADDR_RANGE_ENABLE;
>+
>+ /* set SGunit address range register */
>+ base = index * MAX_DPA_SIZE << PKG_ADDR_BASE_BASE_SHIFT;
>+ base |= MAX_DPA_SIZE << PKG_ADDR_BASE_RANGE_SHIFT;
>+ base |= PKG_ADDR_BASE_ENABLE;
>+
>+ /* Needs to be set for each gt */
>+ for_each_gt(gt, xe, i) {
>+ xe_mmio_write32(gt, PKG_ADDR_RANGE, range);
>+ xe_mmio_write32(gt, PKG_ADDR_BASE, base);
>+ }
>+}
>+
>+static void xe_link_release_dev(struct device *dev)
>+{
>+ struct auxiliary_device *aux = to_auxiliary_dev(dev);
>+ struct xelink_pdata *pd = container_of(aux, struct xelink_pdata,
>aux_dev);
>+
>+ kfree(pd->resources);
>+ pd->resources = NULL;
>+
>+ kfree(pd);
>+}
>+
>+/**
>+ * xe_link_init_aux - Initialize resources and add auxiliary bus interface
>+ * @xe: valid xe instance
>+ *
>+ */
>+void xe_link_init_aux(struct xe_device *xe)
>+{
>+ struct device *dev = &to_pci_dev(xe->drm.dev)->dev;
>+ struct resource *res = NULL;
>+ struct xelink_pdata *pd;
>+ int err = -ENOMEM;
>+ u32 res_cnt;
>+
>+ if (!xe->xelink.present)
>+ return;
>+
>+ if (xe->xelink.index < 0) {
>+ err = xe->xelink.index;
>+ goto fail;
>+ }
>+
>+ pd = init_pd(xe);
>+ if (!pd)
>+ goto cleanup;
>+
>+ res = init_resource(xe, &res_cnt);
>+ if (!res)
>+ goto cleanup;
>+
>+ pd->resources = res;
>+ pd->num_resources = res_cnt;
>+
>+ pd->aux_dev.name = "xelink";
>+ pd->aux_dev.id = pd->index;
>+ pd->aux_dev.dev.parent = dev;
>+ pd->aux_dev.dev.release = xe_link_release_dev;
>+
>+ err = auxiliary_device_init(&pd->aux_dev);
>+ if (err)
>+ goto cleanup;
>+
>+ err = auxiliary_device_add(&pd->aux_dev);
>+ if (err) {
>+ auxiliary_device_uninit(&pd->aux_dev);
>+ goto cleanup;
>+ }
>+
>+ xe->xelink.pd = pd;
>+
>+ return;
>+
>+cleanup:
>+ xa_erase(&intel_fdevs, xe->xelink.index);
>+ irq_free_descs(xe->xelink.irq_base, xe->info.tile_count);
>+ kfree(res);
>+ kfree(pd);
>+ xe->xelink.index = err;
>+fail:
>+ drm_err(&xe->drm, "XeLink: Failed to initialize err: %d\n", err);
>+}
>+
>+void xe_link_remove(struct xe_device *xe)
>+{
>+ if (xe->xelink.index < 0)
>+ return;
>+
>+ auxiliary_device_delete(&xe->xelink.pd->aux_dev);
>+ auxiliary_device_uninit(&xe->xelink.pd->aux_dev);
>+ xa_erase(&intel_fdevs, xe->xelink.index);
>+ irq_free_descs(xe->xelink.irq_base, xe->info.tile_count);
>+
>+ xe->xelink.ops = &default_ops;
>+}
>diff --git a/drivers/gpu/drm/xe/xe_link.h b/drivers/gpu/drm/xe/xe_link.h
>new file mode 100644
>index 000000000000..2d3642166aed
>--- /dev/null
>+++ b/drivers/gpu/drm/xe/xe_link.h
>@@ -0,0 +1,36 @@
>+/* SPDX-License-Identifier: MIT */
>+/*
>+ * Copyright © 2019 - 2023 Intel Corporation
>+ */
>+
>+#ifndef _XE_LINK_H_
>+#define _XE_LINK_H_
>+
>+/*
>+ * Define the maximum number of devices instances based on the amount of
>+ * FID space.
>+ *
>+ * XARRAY limits are "inclusive", but using this value as a range check
>+ * outside of xarray, makes the exclusive upper bound a little easier to
>+ * deal with.
>+ *
>+ * I.e.:
>+ * [0 - 256)
>+ *
>+ * Less than HW supports, but more than will be currently possible.
>+ *
>+ */
>+#define MAX_DEVICE_COUNT 256
>+
>+/* Fixed Device Physical Address (DPA) size for a device/package (in GB) */
>+#define MAX_DPA_SIZE 128
>+
>+struct xe_device;
>+
>+void xe_link_init_early(struct xe_device *xe);
>+void xe_link_init_mmio(struct xe_device *xe);
>+void xe_link_init(struct xe_device *xe);
>+void xe_link_init_aux(struct xe_device *xe);
>+void xe_link_remove(struct xe_device *xe);
>+
>+#endif
>diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c
>index d0a36600e52b..6668cd3ad94d 100644
>--- a/drivers/gpu/drm/xe/xe_mmio.c
>+++ b/drivers/gpu/drm/xe/xe_mmio.c
>@@ -173,8 +173,7 @@ static int xe_determine_lmem_bar_size(struct
>xe_device *xe)
> if (!xe->mem.vram.io_size)
> return -EIO;
>
>- /* XXX: Need to change when xe link code is ready */
>- xe->mem.vram.dpa_base = 0;
>+ xe->mem.vram.dpa_base = xe->xelink.dpa;
>
> /* set up a map to the total memory area. */
> xe->mem.vram.mapping = ioremap_wc(xe->mem.vram.io_start, xe-
>>mem.vram.io_size);
>diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
>index cb762c11dd0f..9079faf4c3fb 100644
>--- a/drivers/gpu/drm/xe/xe_pci.c
>+++ b/drivers/gpu/drm/xe/xe_pci.c
>@@ -143,6 +143,7 @@ static const struct xe_graphics_desc graphics_xehpc = {
>
> .has_asid = 1,
> .has_flat_ccs = 0,
>+ .has_xelink = 1,
> .supports_usm = 1,
> };
>
>@@ -590,6 +591,7 @@ static int xe_info_init(struct xe_device *xe,
> xe->info.vm_max_level = graphics_desc->vm_max_level;
> xe->info.supports_usm = graphics_desc->supports_usm;
> xe->info.has_asid = graphics_desc->has_asid;
>+ xe->info.has_xelink = graphics_desc->has_xelink;
> xe->info.has_flat_ccs = graphics_desc->has_flat_ccs;
> xe->info.has_range_tlb_invalidation = graphics_desc-
>>has_range_tlb_invalidation;
>
>diff --git a/drivers/gpu/drm/xe/xe_pci_types.h
>b/drivers/gpu/drm/xe/xe_pci_types.h
>index dd3546ba6f90..5264f2e5163c 100644
>--- a/drivers/gpu/drm/xe/xe_pci_types.h
>+++ b/drivers/gpu/drm/xe/xe_pci_types.h
>@@ -25,6 +25,7 @@ struct xe_graphics_desc {
> u8 max_remote_tiles:2;
>
> u8 has_asid:1;
>+ u8 has_xelink:1;
> u8 has_flat_ccs:1;
> u8 has_range_tlb_invalidation:1;
> u8 supports_usm:1;
>diff --git a/include/drm/xelink_platform.h b/include/drm/xelink_platform.h
>new file mode 100644
>index 000000000000..81f33a38097a
>--- /dev/null
>+++ b/include/drm/xelink_platform.h
>@@ -0,0 +1,140 @@
>+/* SPDX-License-Identifier: MIT */
>+/*
>+ * Copyright(c) 2019 - 2023 Intel Corporation.
>+ */
>+
>+#ifndef __XELINK_PLATFORM_H
>+#define __XELINK_PLATFORM_H
>+
>+#define XELINK_VERSION 1
>+
>+#include <linux/auxiliary_bus.h>
>+#include <linux/types.h>
>+
>+/**
>+ * enum product_type - Product type identifying the parent
>+ * @XELINK_UNDEFINED: no product type hints
>+ * @XELINK_PONTEVECCHIO: parent is a PVC
>+ * @XELINK_PRODUCTS: end of the list
>+ *
>+ */
>+enum product_type {
>+ XELINK_UNDEFINED,
>+ XELINK_PONTEVECCHIO,
>+ XELINK_PRODUCTS
>+};
>+
>+/**
>+ * enum xelink_dev_event - events generated to the parent device
>+ * @XELINK_DEV_REMOVE: Xelink device was removed
>+ * @XELINK_DEV_ERROR: An error occurred
>+ * @XELINK_DEV_EVENTS: end of list
>+ *
>+ * Connectivity events, possible errors, etc.
>+ */
>+enum xelink_dev_event {
>+ XELINK_DEV_REMOVE,
>+ XELINK_DEV_EVENTS
>+};
>+
>+/**
>+ * enum xelink_parent_event - Events generated by the parent device
>+ * @XELINK_PARENT_PCIE_ERR: Parent had a PCI error
>+ * @XELINK_PARENT_MAPPING_GET: Notify XeLink of buffer mapping
>+ * @XELINK_PARENT_MAPPING_PUT: Notify XeLink of buffer unmapping
>+ *
>+ * These are examples.
>+ */
>+enum xelink_parent_event {
>+ XELINK_PARENT_PCIE_ERR,
>+ XELINK_PARENT_MAPPING_GET,
>+ XELINK_PARENT_MAPPING_PUT,
>+};
>+
>+/**
>+ * struct sd2sd_info - Subdevice to subdevice connectivity info
>+ * @bandwidth: in Gbps units not factoring in width or quality degredation
>+ * @latency: in 1/10 hops units
>+ */
>+struct sd2sd_info {
>+ u16 bandwidth;
>+ u16 latency;
>+};
>+
>+/**
>+ * struct query_info - connectivity query response information
>+ * @src_cnt: Requester subdevice count
>+ * @dst_cnt: Destination path count
>+ * @sd2sd: array of src/dst bandwidth/latency information
>+ *
>+ * Query info will be a variably sized data structure allocated by the
>+ * XeLink driver. The access will be indexed by
>+ * (src_index * dst_cnt) + dst_index
>+ *
>+ * The caller will need to free the buffer when done.
>+ */
>+struct query_info {
>+ u8 src_cnt;
>+ u8 dst_cnt;
>+ struct sd2sd_info sd2sd[];
>+};
>+
>+/**
>+ * struct xelink_ops - Communication path from parent to XeLink instance
>+ * @connectivity_query: Query a device for xelink_id connectivity
>+ * @parent_event: Any events needed by the XeLink device
>+ *
>+ * connectivity_query() returns:
>+ * a populated query_info on success,
>+ * an ERR_PTR() on failure
>+ *
>+ */
>+struct xelink_ops {
>+ struct query_info *(*connectivity_query)(void *handle, u32 xelink_id);
>+ int (*parent_event)(void *handle, enum xelink_parent_event event);
>+};
>+
>+struct dpa_space {
>+ u32 pkg_offset;
>+ u16 pkg_size;
>+};
>+
>+/**
>+ * struct xelink_pdata - Platform specific data that needs to be shared
>+ * @version: PSD version information
>+ * @parent: Handle to use when calling the parent device
>+ * @product: a product hint for any necessary special case requirements
>+ * @index: unique device index. This will be part of the device name
>+ * @dpa: Device physical address offset and size
>+ * @sd_cnt: parent subdevice count
>+ * @socket_id: device socket information
>+ * @slot: PCI/CXL slot number
>+ * @aux_dev: Auxiliary bus device
>+ * @resources: Array of resources (Assigned by Xe, the IRQ/MEM for the
>device)
>+ * @num_resources: number of resources in resources array
>+ * @register_dev: Register a XeLink instance and ops with the parent device
>+ * @unregister_dev: Unregister a XeLink instance from the parent device
>+ * @dev_event: Notify parent that an event has occurred
>+ */
>+struct xelink_pdata {
>+ u16 version;
>+ void *parent;
>+ enum product_type product;
>+ u16 index;
>+ struct dpa_space dpa;
>+ u8 sd_cnt;
>+ u8 socket_id;
>+ u8 slot;
>+
>+ struct auxiliary_device aux_dev;
>+ struct resource *resources;
>+ u32 num_resources;
>+
>+ int (*register_dev)(void *parent, void *handle, u32 xelink_id,
>+ const struct xelink_ops *ops);
>+ void (*unregister_dev)(void *parent, const void *handle);
>+ int (*dev_event)(void *parent, void *handle,
>+ enum xelink_dev_event event, void *event_data);
>+};
>+
>+#endif /* __XELINK_PLATFORM_H */
>--
>2.38.1
More information about the Intel-xe
mailing list