[PATCH i-g-t 2/2] lib/xe_mmio: Introduce Xe MMIO lib

[Piotr Piórkowski]

Kamil Konieczny <kamil.konieczny at linux.intel.com> wrote on pon [2024-mar-25 18:36:06 +0100]:
> Hi Piórkowski,,
> On 2024-03-25 at 16:57:09 +0100, Piórkowski, Piotr wrote:
> > From: Piotr Piórkowski <piotr.piorkowski at intel.com>
> > 
> > Currently in IGT we have a library intel_mmio for simple MMIO operations
> > on intel GPU devices, but it is limited only to accessing registers,
> > has a lot of legacy code related to the i915 and offers no support for
> > multi tile.
> > Let's reuse the memory mapping from the previous library and add separate
> > helpers, dedicated to Xe, for registers and GGTT access that support multi
> > tile.
> > 
> > Signed-off-by: Piotr Piórkowski <piotr.piorkowski at intel.com>
> > ---
> >  lib/meson.build   |   3 +-
> >  lib/xe/xe_mmio.c  | 207 ++++++++++++++++++++++++++++++++++++++++++++++
> >  lib/xe/xe_mmio.h  |  37 +++++++++
> >  lib/xe/xe_query.c |  19 +++++
> >  lib/xe/xe_query.h |   1 +
> >  5 files changed, 266 insertions(+), 1 deletion(-)
> >  create mode 100644 lib/xe/xe_mmio.c
> >  create mode 100644 lib/xe/xe_mmio.h
> > 
> > diff --git a/lib/meson.build b/lib/meson.build
> > index 934bac5c6..2f9666846 100644
> > --- a/lib/meson.build
> > +++ b/lib/meson.build
> > @@ -110,6 +110,7 @@ lib_sources = [
> >  	'igt_dsc.c',
> >  	'xe/xe_gt.c',
> >  	'xe/xe_ioctl.c',
> > +	'xe/xe_mmio.c',
> >  	'xe/xe_query.c',
> >  	'xe/xe_spin.c',
> >  	'xe/xe_util.c',
> > @@ -162,7 +163,7 @@ if libdrm_amdgpu.found()
> >  		lib_sources +=[ 'amdgpu/amd_dispatch.c',]
> >  	else
> >  		warning('libdrm <= 2.4.99 found, amdgpu_cs_query_reset_state2 not applicable')
> > -	endif	
> > +	endif
> >  endif
> >  
> >  if libunwind.found()
> > diff --git a/lib/xe/xe_mmio.c b/lib/xe/xe_mmio.c
> > new file mode 100644
> > index 000000000..bd4227f25
> > --- /dev/null
> > +++ b/lib/xe/xe_mmio.c
> > @@ -0,0 +1,207 @@
> > +// SPDX-License-Identifier: MIT
> > +/*
> > + * Copyright(c) 2024 Intel Corporation. All rights reserved.
> > + */
> > +
> > +#include "igt_device.h"
> > +
> > +#include "xe/xe_mmio.h"
> > +#include "xe/xe_query.h"
> > +
> > +/**
> > + * xe_mmio_vf_access_init:
> > + * @pf_fd: xe device file descriptor
> > + * @vf_id: PCI virtual function number (0 if native or PF itself)
> > + * @mmio: xe mmio structure for IO operations
> > + *
> > + * This initializes the xe mmio structure, and maps the MMIO BAR owned by
> > + * the specified virtual function associated with @pf_fd.
> > + */
> > +void xe_mmio_vf_access_init(int pf_fd, int vf_id, struct xe_mmio *mmio)
> > +{
> > +	struct pci_device *pci_dev = __igt_device_get_pci_device(pf_fd, vf_id);
> > +
> > +	igt_assert_f(pci_dev, "No PCI device found for VF%u\n", vf_id);
> > +
> > +	intel_mmio_use_pci_bar(&mmio->intel_mmio, pci_dev);
> > +
> > +	igt_assert(!mmio->intel_mmio.igt_mmio);
> > +
> > +	mmio->fd = pf_fd;
> > +	mmio->intel_mmio.safe = false;
> > +	mmio->intel_mmio.pci_device_id = pci_dev->device_id;
> > +}
> > +
> > +/**
> > + * xe_mmio_access_init:
> > + * @pf_fd: xe device file descriptor
> > + * @mmio: xe mmio structure for IO operations
> > + *
> > + * This initializes the xe mmio structure, and maps MMIO BAR for @pf_fd device.
> > + */
> > +void xe_mmio_access_init(int pf_fd, struct xe_mmio *mmio)
> > +{
> > +	xe_mmio_vf_access_init(pf_fd, 0, mmio);
> > +}
> > +
> > +/**
> > + * xe_mmio_access_fini:
> > + * @mmio: xe mmio structure for IO operations
> > + *
> > + * Clean up the mmio access helper initialized with
> > + * xe_mmio_access_init()/xe_mmio_vf_access_init().
> > + */
> > +void xe_mmio_access_fini(struct xe_mmio *mmio)
> > +{
> > +	mmio->intel_mmio.pci_device_id = 0;
> > +	intel_mmio_unmap_pci_bar(&mmio->intel_mmio);
> > +	igt_pci_system_cleanup();
> > +}
> > +
> > +/**
> > + * xe_mmio_read32:
> > + * @mmio: xe mmio structure for IO operations
> > + * @reg: mmio register offset
> > + *
> > + * 32-bit read of the register at @offset.
> > + *
> > + * Returns:
> > + * The value read from the register.
> > + */
> > +uint32_t xe_mmio_read32(struct xe_mmio *mmio, uint32_t reg)
> > +{
> > +	return ioread32(mmio->intel_mmio.igt_mmio, reg);
> > +}
> > +
> > +/**
> > + * xe_mmio_read64:
> > + * @mmio: xe mmio structure for IO operations
> > + * @reg: mmio register offset
> > + *
> > + * 64-bit read of the register at @offset.
> > + *
> > + * Returns:
> > + * The value read from the register.
> > + */
> > +uint64_t xe_mmio_read64(struct xe_mmio *mmio, uint32_t reg)
> > +{
> > +	return ioread64(mmio->intel_mmio.igt_mmio, reg);
> > +}
> > +
> > +/**
> > + * xe_mmio_write32:
> > + * @mmio: xe mmio structure for IO operations
> > + * @reg: mmio register offset
> > + * @val: value to write
> > + *
> > + * 32-bit write to the register at @offset.
> > + */
> > +void xe_mmio_write32(struct xe_mmio *mmio, uint32_t reg, uint32_t val)
> > +{
> > +	return iowrite32(mmio->intel_mmio.igt_mmio, reg, val);
> > +}
> > +
> > +/**
> > + * xe_mmio_write64:
> > + * @mmio: xe mmio structure for IO operations
> > + * @reg: mmio register offset
> > + * @val: value to write
> > + *
> > + * 64-bit write to the register at @offset.
> > + */
> > +void xe_mmio_write64(struct xe_mmio *mmio, uint32_t reg, uint64_t val)
> > +{
> > +	return iowrite64(mmio->intel_mmio.igt_mmio, reg, val);
> > +}
> > +
> > +/**
> > + * xe_mmio_gt_read32:
> > + * @mmio: xe mmio structure for IO operations
> > + * @gt: gt id
> > + * @reg: mmio register offset in tile to which @gt belongs
> > + *
> > + * 32-bit read of the register at @offset in tile to which @gt belongs.
> > + *
> > + * Returns:
> > + * The value read from the register.
> > + */
> > +uint32_t xe_mmio_gt_read32(struct xe_mmio *mmio, int gt, uint32_t reg)
> > +{
> > +	return xe_mmio_read32(mmio, reg + (TILE_MMIO_SIZE * xe_gt_get_tile_id(mmio->fd, gt)));
> > +}
> > +
> > +/**
> > + * xe_mmio_gt_read64:
> > + * @mmio: xe mmio structure for IO operations
> > + * @gt: gt id
> > + * @reg: mmio register offset in tile to which @gt belongs
> > + *
> > + * 64-bit read of the register at @offset in tile to which @gt belongs.
> > + *
> > + * Returns:
> > + * The value read from the register.
> > + */
> > +uint64_t xe_mmio_gt_read64(struct xe_mmio *mmio, int gt, uint32_t reg)
> > +{
> > +	return xe_mmio_read64(mmio, reg + (TILE_MMIO_SIZE * xe_gt_get_tile_id(mmio->fd, gt)));
> > +}
> > +
> > +/**
> > + * xe_mmio_gt_write32:
> > + * @mmio: xe mmio structure for IO operations
> > + * @gt: gt id
> > + * @reg: mmio register offset
> > + * @val: value to write
> > + *
> > + * 32-bit write to the register at @offset in tile to which @gt belongs.
> > + */
> > +void xe_mmio_gt_write32(struct xe_mmio *mmio, int gt, uint32_t reg, uint32_t val)
> > +{
> > +	return xe_mmio_write32(mmio, reg + (TILE_MMIO_SIZE * xe_gt_get_tile_id(mmio->fd, gt)),
> > +			       val);
> > +}
> > +
> > +/**
> > + * xe_mmio_gt_write64:
> > + * @mmio: xe mmio structure for IO operations
> > + * @gt: gt id
> > + * @reg: mmio register offset
> > + * @val: value to write
> > + *
> > + * 64-bit write to the register at @offset in tile to which @gt belongs.
> > + */
> > +void xe_mmio_gt_write64(struct xe_mmio *mmio, int gt, uint32_t reg, uint64_t val)
> > +{
> > +	return xe_mmio_write64(mmio, reg + (TILE_MMIO_SIZE * xe_gt_get_tile_id(mmio->fd, gt)),
> > +			       val);
> > +}
> > +
> > +/**
> > + * xe_mmio_ggtt_read:
> > + * @mmio: xe mmio structure for IO operations
> > + * @gt: gt id
> > + * @offset: PTE offset from the beginning of GGTT, in tile to which @gt belongs
> > + *
> > + * Read of GGTT PTE at GGTT @offset in tile to which @gt belongs.
> > + *
> > + * Returns:
> > + * The value read from the register.
> > + */
> > +xe_ggtt_pte_t xe_mmio_ggtt_read(struct xe_mmio *mmio, int gt, uint32_t offset)
> > +{
> > +	return xe_mmio_gt_read64(mmio, gt, offset + GGTT_OFFSET_IN_TILE);
> > +}
> > +
> > +/**
> > + * xe_mmio_ggtt_write:
> > + * @mmio: xe mmio structure for IO operations
> > + * @gt: gt id
> > + * @offset: PTE offset from the beginning of GGTT, in tile to which @gt belongs
> > + * @pte: PTE value to write
> > + *
> > + * Write PTE value at GGTT @offset in tile to which @gt belongs.
> > + */
> > +void xe_mmio_ggtt_write(struct xe_mmio *mmio, int gt, uint32_t offset, xe_ggtt_pte_t pte)
> > +{
> > +	return xe_mmio_gt_write64(mmio, gt, offset + GGTT_OFFSET_IN_TILE, pte);
> > +}
> > diff --git a/lib/xe/xe_mmio.h b/lib/xe/xe_mmio.h
> > new file mode 100644
> > index 000000000..ed45040bd
> > --- /dev/null
> > +++ b/lib/xe/xe_mmio.h
> > @@ -0,0 +1,37 @@
> > +/* SPDX-License-Identifier: MIT */
> > +/*
> > + * Copyright(c) 2024 Intel Corporation. All rights reserved.
> > + */
> 
> Add here standard ifdef/define and at end of file endif:
> 
> #ifndef IGT_XE_MMIO_H
> #define IGT_XE_MMIO_H

Oh thanks for catching that.
I'll fix it right away!

- Piotr
> 
> Regards,
> Kamil
> 
> > +
> > +#include "lib/intel_io.h"
> > +#include "lib/igt_sizes.h"
> > +
> > +#define TILE_MMIO_SIZE		SZ_16M
> > +#define GGTT_OFFSET_IN_TILE	SZ_8M
> > +
> > +typedef uint64_t xe_ggtt_pte_t;
> > +
> > +struct xe_mmio {
> > +	int fd;
> > +	unsigned int vf_id;
> > +	struct intel_mmio_data intel_mmio;
> > +};
> > +
> > +void xe_mmio_vf_access_init(int pf_fd, int vf_id, struct xe_mmio *mmio);
> > +void xe_mmio_access_init(int pf_fd, struct xe_mmio *mmio);
> > +void xe_mmio_access_fini(struct xe_mmio *mmio);
> > +
> > +uint32_t xe_mmio_read32(struct xe_mmio *mmio, uint32_t reg);
> > +uint64_t xe_mmio_read64(struct xe_mmio *mmio, uint32_t reg);
> > +
> > +void xe_mmio_write32(struct xe_mmio *mmio, uint32_t reg, uint32_t val);
> > +void xe_mmio_write64(struct xe_mmio *mmio, uint32_t reg, uint64_t val);
> > +
> > +uint32_t xe_mmio_gt_read32(struct xe_mmio *mmio, int gt, uint32_t reg);
> > +uint64_t xe_mmio_gt_read64(struct xe_mmio *mmio, int gt, uint32_t reg);
> > +
> > +void xe_mmio_gt_write32(struct xe_mmio *mmio, int gt, uint32_t reg, uint32_t val);
> > +void xe_mmio_gt_write64(struct xe_mmio *mmio, int gt, uint32_t reg, uint64_t val);
> > +
> > +xe_ggtt_pte_t xe_mmio_ggtt_read(struct xe_mmio *mmio, int gt, uint32_t pte_offset);
> > +void xe_mmio_ggtt_write(struct xe_mmio *mmio, int gt, uint32_t pte_offset, xe_ggtt_pte_t pte);
> > diff --git a/lib/xe/xe_query.c b/lib/xe/xe_query.c
> > index 53a2b4386..c885e3a79 100644
> > --- a/lib/xe/xe_query.c
> > +++ b/lib/xe/xe_query.c
> > @@ -731,6 +731,25 @@ bool xe_is_media_gt(int fd, int gt)
> >  	return false;
> >  }
> >  
> > +/**
> > + * xe_gt_to_tile_id:
> > + * @fd: xe device fd
> > + * @gt: gt id
> > + *
> > + * Returns tile id for given @gt.
> > + */
> > +int xe_gt_get_tile_id(int fd, int gt)
> > +{
> > +	struct xe_device *xe_dev;
> > +
> > +	xe_dev = find_in_cache(fd);
> > +
> > +	igt_assert(xe_dev);
> > +	igt_assert(gt < xe_number_gt(fd));
> > +
> > +	return xe_dev->gt_list->gt_list[gt].tile_id;
> > +}
> > +
> >  igt_constructor
> >  {
> >  	xe_device_cache_init();
> > diff --git a/lib/xe/xe_query.h b/lib/xe/xe_query.h
> > index 82af2706d..b1b3a989e 100644
> > --- a/lib/xe/xe_query.h
> > +++ b/lib/xe/xe_query.h
> > @@ -99,6 +99,7 @@ const char *xe_engine_class_string(uint32_t engine_class);
> >  bool xe_has_engine_class(int fd, uint16_t engine_class);
> >  bool xe_has_media_gt(int fd);
> >  bool xe_is_media_gt(int fd, int gt);
> > +int xe_gt_get_tile_id(int fd, int gt);
> >  
> >  struct xe_device *xe_device_get(int fd);
> >  void xe_device_put(int fd);
> > -- 
> > 2.34.1
> > 

--