[PATCH 2/2] drm/xe/vf: Use only assigned GGTT region

Michał Winiarski michal.winiarski at intel.com
Mon May 27 07:51:27 UTC 2024 

On Fri, May 24, 2024 at 01:37:14PM +0200, Michal Wajdeczko wrote:
> Each VF is assigned a limited range of the GGTT address space.
> To ensure that the VF driver does not use GGTT allocations outside
> of the assigned region, explicitly reserve GGTT space below and
> above this region when initializing GGTT.
> 
> Signed-off-by: Michal Wajdeczko <michal.wajdeczko at intel.com>
> ---
>  drivers/gpu/drm/xe/xe_device_types.h |  3 +
>  drivers/gpu/drm/xe/xe_ggtt.c         | 14 ++++-
>  drivers/gpu/drm/xe/xe_gt_sriov_vf.c  | 93 ++++++++++++++++++++++++++++
>  drivers/gpu/drm/xe/xe_gt_sriov_vf.h  |  1 +
>  4 files changed, 110 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
> index 8e7048ff3ee5..dad7e9554c2c 100644
> --- a/drivers/gpu/drm/xe/xe_device_types.h
> +++ b/drivers/gpu/drm/xe/xe_device_types.h
> @@ -196,6 +196,9 @@ struct xe_tile {
>  		struct {
>  			/** @sriov.vf.memirq: Memory Based Interrupts. */
>  			struct xe_memirq memirq;
> +
> +			/** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */
> +			struct drm_mm_node ggtt_balloon[2];
>  		} vf;
>  	} sriov;
>  
> diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
> index 17e5066763db..b01a670fecb8 100644
> --- a/drivers/gpu/drm/xe/xe_ggtt.c
> +++ b/drivers/gpu/drm/xe/xe_ggtt.c
> @@ -20,6 +20,7 @@
>  #include "xe_device.h"
>  #include "xe_gt.h"
>  #include "xe_gt_printk.h"
> +#include "xe_gt_sriov_vf.h"
>  #include "xe_gt_tlb_invalidation.h"
>  #include "xe_map.h"
>  #include "xe_pm.h"
> @@ -141,6 +142,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
>  	struct xe_device *xe = tile_to_xe(ggtt->tile);
>  	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
>  	unsigned int gsm_size;
> +	int err;
>  
>  	if (IS_SRIOV_VF(xe))
>  		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
> @@ -194,7 +196,17 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
>  	mutex_init(&ggtt->lock);
>  	primelockdep(ggtt);
>  
> -	return drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
> +	err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
> +	if (err)
> +		return err;
> +
> +	if (IS_SRIOV_VF(xe)) {
> +		err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0));
> +		if (err)
> +			return err;
> +	}
> +
> +	return 0;
>  }
>  
>  static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> index 66069eb8c5be..5ef0c3d78e7c 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
> @@ -14,9 +14,12 @@
>  #include "abi/guc_klvs_abi.h"
>  #include "abi/guc_relay_actions_abi.h"
>  #include "regs/xe_gt_regs.h"
> +#include "regs/xe_gtt_defs.h"
> +#include "regs/xe_guc_regs.h"

It doesn't look like xe_guc_regs.h is used directly - please, drop it.

Other than that - LGTM.

Reviewed-by: Michał Winiarski <michal.winiarski at intel.com>

-Michał

>  #include "xe_assert.h"
>  #include "xe_device.h"
> +#include "xe_ggtt.h"
>  #include "xe_gt_sriov_printk.h"
>  #include "xe_gt_sriov_vf.h"
>  #include "xe_gt_sriov_vf_types.h"
> @@ -25,6 +28,8 @@
>  #include "xe_guc_relay.h"
>  #include "xe_mmio.h"
>  #include "xe_sriov.h"
> +#include "xe_uc_fw.h"
> +#include "xe_wopcm.h"
>  
>  #define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo)))
>  
> @@ -474,6 +479,94 @@ u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt)
>  	return gt->sriov.vf.self_config.num_ctxs;
>  }
>  
> +static int vf_balloon_ggtt(struct xe_gt *gt)
> +{
> +	struct xe_gt_sriov_vf_selfconfig *config = &gt->sriov.vf.self_config;
> +	struct xe_tile *tile = gt_to_tile(gt);
> +	struct xe_ggtt *ggtt = tile->mem.ggtt;
> +	struct xe_device *xe = gt_to_xe(gt);
> +	u64 start, end;
> +	int err;
> +
> +	xe_gt_assert(gt, IS_SRIOV_VF(xe));
> +	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
> +
> +	if (!config->ggtt_size)
> +		return -ENODATA;
> +
> +	/*
> +	 * VF can only use part of the GGTT as allocated by the PF:
> +	 *
> +	 *      WOPCM                                  GUC_GGTT_TOP
> +	 *      |<------------ Total GGTT size ------------------>|
> +	 *
> +	 *           VF GGTT base -->|<- size ->|
> +	 *
> +	 *      +--------------------+----------+-----------------+
> +	 *      |////////////////////|   block  |\\\\\\\\\\\\\\\\\|
> +	 *      +--------------------+----------+-----------------+
> +	 *
> +	 *      |<--- balloon[0] --->|<-- VF -->|<-- balloon[1] ->|
> +	 */
> +
> +	start = xe_wopcm_size(xe);
> +	end = config->ggtt_base;
> +	if (end != start) {
> +		err = xe_ggtt_balloon(ggtt, start, end, &tile->sriov.vf.ggtt_balloon[0]);
> +		if (err)
> +			goto failed;
> +	}
> +
> +	start = config->ggtt_base + config->ggtt_size;
> +	end = GUC_GGTT_TOP;
> +	if (end != start) {
> +		err = xe_ggtt_balloon(ggtt, start, end, &tile->sriov.vf.ggtt_balloon[1]);
> +		if (err)
> +			goto deballoon;
> +	}
> +
> +	return 0;
> +
> +deballoon:
> +	xe_ggtt_deballoon(ggtt, &tile->sriov.vf.ggtt_balloon[0]);
> +failed:
> +	return err;
> +}
> +
> +static void deballoon_ggtt(struct drm_device *drm, void *arg)
> +{
> +	struct xe_tile *tile = arg;
> +	struct xe_ggtt *ggtt = tile->mem.ggtt;
> +
> +	xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile)));
> +	xe_ggtt_deballoon(ggtt, &tile->sriov.vf.ggtt_balloon[1]);
> +	xe_ggtt_deballoon(ggtt, &tile->sriov.vf.ggtt_balloon[0]);
> +}
> +
> +/**
> + * xe_gt_sriov_vf_prepare_ggtt - Prepare a VF's GGTT configuration.
> + * @gt: the &xe_gt
> + *
> + * This function is for VF use only.
> + *
> + * Return: 0 on success or a negative error code on failure.
> + */
> +int xe_gt_sriov_vf_prepare_ggtt(struct xe_gt *gt)
> +{
> +	struct xe_tile *tile = gt_to_tile(gt);
> +	struct xe_device *xe = tile_to_xe(tile);
> +	int err;
> +
> +	if (xe_gt_is_media_type(gt))
> +		return 0;
> +
> +	err = vf_balloon_ggtt(gt);
> +	if (err)
> +		return err;
> +
> +	return drmm_add_action_or_reset(&xe->drm, deballoon_ggtt, tile);
> +}
> +
>  static int relay_action_handshake(struct xe_gt *gt, u32 *major, u32 *minor)
>  {
>  	u32 request[VF2PF_HANDSHAKE_REQUEST_MSG_LEN] = {
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> index 0391ed72a570..7a04bcaffe9f 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
> @@ -16,6 +16,7 @@ int xe_gt_sriov_vf_bootstrap(struct xe_gt *gt);
>  int xe_gt_sriov_vf_query_config(struct xe_gt *gt);
>  int xe_gt_sriov_vf_connect(struct xe_gt *gt);
>  int xe_gt_sriov_vf_query_runtime(struct xe_gt *gt);
> +int xe_gt_sriov_vf_prepare_ggtt(struct xe_gt *gt);
>  
>  u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt);
>  u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);
> -- 
> 2.43.0
>

More information about the Intel-xe mailing list