[Intel-xe] [PATCH 2/3] drm/xe: Prepare for running in different SR-IOV modes
Michal Wajdeczko
michal.wajdeczko at intel.com
Wed Nov 22 23:09:18 UTC 2023
On 22.11.2023 20:47, Matt Roper wrote:
> On Wed, Nov 15, 2023 at 08:38:03AM +0100, Michal Wajdeczko wrote:
>> We will be adding support for the SR-IOV and driver might be then
>> running, in addition to existing non-virtualized bare-metal mode,
>> also in Physical Function (PF) or Virtual Function (VF) mode.
>>
>> Since these additional modes require some changes to the driver,
>> define enum flag to represent different SR-IOV modes and add a
>> function where we will detect the actual mode in the runtime.
>>
>> We start with a forced bare-metal mode as it is sufficient to
>> enable basic functionality and ensures no impact to existing code.
>
> For platforms that support SRIOV, can you describe what the benefit is
> of running in bare-metal mode vs just running in PF mode with no VFs
> active? Will we wind up needing to do extra initialization in PF mode
> that could be counter-productive if someone didn't actually want to use
> any virtualization? Will the system have additional limitations in PF
> mode that we can avoid in bare-metal mode?
In general, there should be no major differences between running as PF
without VFs and bare-metal. However, on some platforms we simply can't
enable SR-IOV with other mega features (like on PVC it won't work with
XeLink enabled).
If for future platforms above won't hold, then explicit decision whether
PF functionality is needed should be done at deployment time using
modparam. This was shortly discussed in RFC [1].
While our preliminary SR-IOV support on i915 put some restriction on PF
in number of available GuC context IDs, the Xe implementation should not
have such problems.
Most of significant allocations or resources to support VFs will be done
either as part of explicit VFs provisioning or implicit auto
provisioning done when VFs are being enabled. Any other mandatory SR-IOV
specific allocations (for HW or sysfs) should be negligible.
Note that we will try to hide all PF code under CONFIG_PCI_IOV but, if
desired, and discussed in [1], we can also use XE specific config to
hide both PF and VF code independently.
>
> Aside from adding some explanation for that,
>
> Reviewed-by: Matt Roper <matthew.d.roper at intel.com>
Thanks,
Michal
[1] https://patchwork.freedesktop.org/series/126289/
>
>>
>> Signed-off-by: Michal Wajdeczko <michal.wajdeczko at intel.com>
>> ---
>> drivers/gpu/drm/xe/Makefile | 3 ++
>> drivers/gpu/drm/xe/xe_device_types.h | 7 ++++
>> drivers/gpu/drm/xe/xe_pci.c | 3 ++
>> drivers/gpu/drm/xe/xe_sriov.c | 55 ++++++++++++++++++++++++++++
>> drivers/gpu/drm/xe/xe_sriov.h | 42 +++++++++++++++++++++
>> drivers/gpu/drm/xe/xe_sriov_types.h | 28 ++++++++++++++
>> 6 files changed, 138 insertions(+)
>> create mode 100644 drivers/gpu/drm/xe/xe_sriov.c
>> create mode 100644 drivers/gpu/drm/xe/xe_sriov.h
>> create mode 100644 drivers/gpu/drm/xe/xe_sriov_types.h
>>
>> diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
>> index cb41b4fead08..fbba74d9a330 100644
>> --- a/drivers/gpu/drm/xe/Makefile
>> +++ b/drivers/gpu/drm/xe/Makefile
>> @@ -123,6 +123,9 @@ xe-y += xe_bb.o \
>> # graphics hardware monitoring (HWMON) support
>> xe-$(CONFIG_HWMON) += xe_hwmon.o
>>
>> +# graphics virtualization (SR-IOV) support
>> +xe-y += xe_sriov.o
>> +
>> # i915 Display compat #defines and #includes
>> subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \
>> -I$(srctree)/$(src)/display/ext \
>> diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
>> index f2ba4f746fa2..2712905c7a91 100644
>> --- a/drivers/gpu/drm/xe/xe_device_types.h
>> +++ b/drivers/gpu/drm/xe/xe_device_types.h
>> @@ -18,6 +18,7 @@
>> #include "xe_platform_types.h"
>> #include "xe_pt_types.h"
>> #include "xe_pmu.h"
>> +#include "xe_sriov_types.h"
>> #include "xe_step_types.h"
>>
>> #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
>> @@ -299,6 +300,12 @@ struct xe_device {
>> struct ttm_resource_manager sys_mgr;
>> } mem;
>>
>> + /** @sriov: device level virtualization data */
>> + struct {
>> + /** @sriov.__mode: SR-IOV mode (Don't access directly!) */
>> + enum xe_sriov_mode __mode;
>> + } sriov;
>> +
>> /** @usm: unified memory state */
>> struct {
>> /** @asid: convert a ASID to VM */
>> diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
>> index c1c8048bbc0f..3030b223e4a0 100644
>> --- a/drivers/gpu/drm/xe/xe_pci.c
>> +++ b/drivers/gpu/drm/xe/xe_pci.c
>> @@ -24,6 +24,7 @@
>> #include "xe_module.h"
>> #include "xe_pci_types.h"
>> #include "xe_pm.h"
>> +#include "xe_sriov.h"
>> #include "xe_step.h"
>>
>> enum toggle_d3cold {
>> @@ -706,6 +707,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>>
>> pci_set_master(pdev);
>>
>> + xe_sriov_probe_early(xe, desc->has_sriov);
>> +
>> err = xe_info_init(xe, desc, subplatform_desc);
>> if (err)
>> goto err_pci_disable;
>> diff --git a/drivers/gpu/drm/xe/xe_sriov.c b/drivers/gpu/drm/xe/xe_sriov.c
>> new file mode 100644
>> index 000000000000..42a0e0c917a0
>> --- /dev/null
>> +++ b/drivers/gpu/drm/xe/xe_sriov.c
>> @@ -0,0 +1,55 @@
>> +// SPDX-License-Identifier: MIT
>> +/*
>> + * Copyright © 2023 Intel Corporation
>> + */
>> +
>> +#include "xe_assert.h"
>> +#include "xe_sriov.h"
>> +
>> +/**
>> + * xe_sriov_mode_to_string - Convert enum value to string.
>> + * @mode: the &xe_sriov_mode to convert
>> + *
>> + * Returns: SR-IOV mode as a user friendly string.
>> + */
>> +const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode)
>> +{
>> + switch (mode) {
>> + case XE_SRIOV_MODE_NONE:
>> + return "none";
>> + case XE_SRIOV_MODE_PF:
>> + return "SR-IOV PF";
>> + case XE_SRIOV_MODE_VF:
>> + return "SR-IOV VF";
>> + default:
>> + return "<invalid>";
>> + }
>> +}
>> +
>> +/**
>> + * xe_sriov_probe_early - Probe a SR-IOV mode.
>> + * @xe: the &xe_device to probe mode on
>> + * @has_sriov: flag indicating hardware support for SR-IOV
>> + *
>> + * This function should be called only once and as soon as possible during
>> + * driver probe to detect whether we are running a SR-IOV Physical Function
>> + * (PF) or a Virtual Function (VF) device.
>> + *
>> + * SR-IOV PF mode detection is based on PCI @dev_is_pf() function.
>> + * SR-IOV VF mode detection is based on dedicated MMIO register read.
>> + */
>> +void xe_sriov_probe_early(struct xe_device *xe, bool has_sriov)
>> +{
>> + enum xe_sriov_mode mode = XE_SRIOV_MODE_NONE;
>> +
>> + /* TODO: replace with proper mode detection */
>> + xe_assert(xe, !has_sriov);
>> +
>> + xe_assert(xe, !xe->sriov.__mode);
>> + xe->sriov.__mode = mode;
>> + xe_assert(xe, xe->sriov.__mode);
>> +
>> + if (has_sriov)
>> + drm_info(&xe->drm, "Running in %s mode\n",
>> + xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
>> +}
>> diff --git a/drivers/gpu/drm/xe/xe_sriov.h b/drivers/gpu/drm/xe/xe_sriov.h
>> new file mode 100644
>> index 000000000000..5af73a3172b0
>> --- /dev/null
>> +++ b/drivers/gpu/drm/xe/xe_sriov.h
>> @@ -0,0 +1,42 @@
>> +/* SPDX-License-Identifier: MIT */
>> +/*
>> + * Copyright © 2023 Intel Corporation
>> + */
>> +
>> +#ifndef _XE_SRIOV_H_
>> +#define _XE_SRIOV_H_
>> +
>> +#include "xe_assert.h"
>> +#include "xe_device_types.h"
>> +#include "xe_sriov_types.h"
>> +
>> +const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode);
>> +
>> +void xe_sriov_probe_early(struct xe_device *xe, bool has_sriov);
>> +
>> +static inline enum xe_sriov_mode xe_device_sriov_mode(struct xe_device *xe)
>> +{
>> + xe_assert(xe, xe->sriov.__mode);
>> + return xe->sriov.__mode;
>> +}
>> +
>> +static inline bool xe_device_is_sriov_pf(struct xe_device *xe)
>> +{
>> + return xe_device_sriov_mode(xe) == XE_SRIOV_MODE_PF;
>> +}
>> +
>> +static inline bool xe_device_is_sriov_vf(struct xe_device *xe)
>> +{
>> + return xe_device_sriov_mode(xe) == XE_SRIOV_MODE_VF;
>> +}
>> +
>> +#ifdef CONFIG_PCI_IOV
>> +#define IS_SRIOV_PF(xe) xe_device_is_sriov_pf(xe)
>> +#else
>> +#define IS_SRIOV_PF(xe) (typecheck(struct xe_device *, (xe)) && false)
>> +#endif
>> +#define IS_SRIOV_VF(xe) xe_device_is_sriov_vf(xe)
>> +
>> +#define IS_SRIOV(xe) (IS_SRIOV_PF(xe) || IS_SRIOV_VF(xe))
>> +
>> +#endif
>> diff --git a/drivers/gpu/drm/xe/xe_sriov_types.h b/drivers/gpu/drm/xe/xe_sriov_types.h
>> new file mode 100644
>> index 000000000000..999a4311b98b
>> --- /dev/null
>> +++ b/drivers/gpu/drm/xe/xe_sriov_types.h
>> @@ -0,0 +1,28 @@
>> +/* SPDX-License-Identifier: MIT */
>> +/*
>> + * Copyright © 2023 Intel Corporation
>> + */
>> +
>> +#ifndef _XE_SRIOV_TYPES_H_
>> +#define _XE_SRIOV_TYPES_H_
>> +
>> +#include <linux/build_bug.h>
>> +
>> +/**
>> + * enum xe_sriov_mode - SR-IOV mode
>> + * @XE_SRIOV_MODE_NONE: bare-metal mode (non-virtualized)
>> + * @XE_SRIOV_MODE_PF: SR-IOV Physical Function (PF) mode
>> + * @XE_SRIOV_MODE_VF: SR-IOV Virtual Function (VF) mode
>> + */
>> +enum xe_sriov_mode {
>> + /*
>> + * Note: We don't use default enum value 0 to allow catch any too early
>> + * attempt of checking the SR-IOV mode prior to the actual mode probe.
>> + */
>> + XE_SRIOV_MODE_NONE = 1,
>> + XE_SRIOV_MODE_PF,
>> + XE_SRIOV_MODE_VF,
>> +};
>> +static_assert(XE_SRIOV_MODE_NONE);
>> +
>> +#endif
>> --
>> 2.25.1
>>
>
More information about the Intel-xe
mailing list