[PATCH] drm/amdgpu: SR-IOV data exchange between PF&VF

Deucher, Alexander Alexander.Deucher at amd.com
Tue Oct 10 15:21:18 UTC 2017


> -----Original Message-----
> From: amd-gfx [mailto:amd-gfx-bounces at lists.freedesktop.org] On Behalf
> Of Horace Chen
> Sent: Tuesday, October 10, 2017 3:52 AM
> To: amd-gfx at lists.freedesktop.org
> Cc: Chen, Horace
> Subject: [PATCH] drm/amdgpu: SR-IOV data exchange between PF&VF
> 
> SR-IOV need to exchange some data between PF&VF through shared VRAM
> 
> PF will copy some necessary firmware and information to the shared
> VRAM. It also requires some information from VF. PF will send a
> key through mailbox2 to help guest calculate checksum so that it can
> verify whether the data is correct.
> 
> So check the data on the specified offset of the shared VRAM, if the
> checksum is right, read values from it and write some VF information
> next to the data from PF.
> 
> Signed-off-by: Horace Chen <horace.chen at amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |   3 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   |  75 ++++++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   | 178
> +++++++++++++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c      |   6 +
>  4 files changed, 262 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index a964241..2275427 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -2343,6 +2343,9 @@ int amdgpu_device_init(struct amdgpu_device
> *adev,
>  	if (r)
>  		DRM_ERROR("ib ring test failed (%d).\n", r);
> 
> +	if (amdgpu_sriov_vf(adev))
> +		amdgpu_virt_init_data_exchange(adev);
> +
>  	amdgpu_fbdev_init(adev);
> 
>  	r = amdgpu_pm_sysfs_init(adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> index ab05121..6aed0c7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
> @@ -274,3 +274,78 @@ void amdgpu_virt_free_mm_table(struct
> amdgpu_device *adev)
>  			      (void *)&adev->virt.mm_table.cpu_addr);
>  	adev->virt.mm_table.gpu_addr = 0;
>  }
> +
> +
> +int amdgpu_virt_fw_reserve_get_checksum(void *obj,
> +					unsigned long obj_size,
> +					unsigned int key,
> +					unsigned int chksum)
> +{
> +	unsigned int ret = key;
> +	unsigned long i = 0;
> +	unsigned char *pos;
> +
> +	pos = (char *)obj;
> +	/* calculate checksum */
> +	for (i = 0; i < obj_size; ++i)
> +		ret += *(pos + i);
> +	/* minus the chksum itself */
> +	pos = (char *)&chksum;
> +	for (i = 0; i < sizeof(chksum); ++i)
> +		ret -= *(pos + i);
> +	return ret;
> +}
> +
> +void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
> +{
> +	uint32_t pf2vf_ver = 0;
> +	uint32_t pf2vf_size = 0;
> +	uint32_t checksum = 0;
> +	uint32_t checkval;
> +	char *str;
> +
> +	adev->virt.fw_reserve.p_pf2vf = NULL;
> +	adev->virt.fw_reserve.p_vf2pf = NULL;
> +
> +	if (adev->fw_vram_usage.va != NULL) {
> +		adev->virt.fw_reserve.p_pf2vf =
> +			(struct amdgim_pf2vf_info_header *)(
> +			adev->fw_vram_usage.va +
> AMDGIM_DATAEXCHANGE_OFFSET_KB);
> +		pf2vf_ver = adev->virt.fw_reserve.p_pf2vf->version;
> +		AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size,
> &pf2vf_size);
> +		AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum,
> &checksum);
> +
> +		/* pf2vf message must be in 4K */
> +		if (pf2vf_size > 0 && pf2vf_size < 4 << 10) {

Also, you may want to add parens around the 4 <<10 for clarity and I don't remember the operator precedence, but I think that may cause a problem.  Probably clearer to just use 4096 here.  

> +			checkval = amdgpu_virt_fw_reserve_get_checksum(
> +				adev->virt.fw_reserve.p_pf2vf, pf2vf_size,
> +				adev->virt.fw_reserve.checksum_key,
> checksum);
> +			if (checkval == checksum) {
> +				adev->virt.fw_reserve.p_vf2pf =
> +					((void *)adev-
> >virt.fw_reserve.p_pf2vf +
> +					pf2vf_size);
> +				memset((void *)adev-
> >virt.fw_reserve.p_vf2pf, 0,
> +					sizeof(amdgim_vf2pf_info));
> +				AMDGPU_FW_VRAM_VF2PF_WRITE(adev,
> header.version,
> +					AMDGPU_FW_VRAM_VF2PF_VER);
> +				AMDGPU_FW_VRAM_VF2PF_WRITE(adev,
> header.size,
> +					sizeof(amdgim_vf2pf_info));
> +				AMDGPU_FW_VRAM_VF2PF_READ(adev,
> driver_version,
> +					&str);
> +				if (THIS_MODULE->version != NULL)
> +					strcpy(str, THIS_MODULE->version);
> +				else
> +					strcpy(str, "N/A");
> +				AMDGPU_FW_VRAM_VF2PF_WRITE(adev,
> driver_cert,
> +					0);
> +				AMDGPU_FW_VRAM_VF2PF_WRITE(adev,
> checksum,
> +
> 	amdgpu_virt_fw_reserve_get_checksum(
> +					adev->virt.fw_reserve.p_vf2pf,
> +					pf2vf_size,
> +					adev-
> >virt.fw_reserve.checksum_key, 0));
> +			}
> +		}
> +	}
> +}
> +
> +
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> index e5fd0ff..a90530d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
> @@ -58,6 +58,179 @@ struct amdgpu_virt_ops {
>  	void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1,
> u32 data2, u32 data3);
>  };
> 
> +/*
> + * Firmware Reserve Frame buffer
> + */
> +struct amdgpu_virt_fw_reserve {
> +	struct amdgim_pf2vf_info_header *p_pf2vf;
> +	struct amdgim_vf2pf_info_header *p_vf2pf;
> +	unsigned int checksum_key;
> +};
> +/*
> + * Defination between PF and VF
> + * Structures forcibly aligned to 4 to keep the same style as PF.
> + */
> +#define AMDGIM_DATAEXCHANGE_OFFSET_KB           (64 * 1024)

This should be OFFSET_BYTES not KB.  Other than those things, looks good.

Alex

> +
> +#define AMDGIM_GET_STRUCTURE_RESERVED_SIZE(total, u8, u16, u32,
> u64) \
> +		(total - (((u8)+3) / 4 + ((u16)+1) / 2 + (u32) + (u64)*2))
> +
> +enum AMDGIM_FEATURE_FLAG {
> +	/* GIM supports feature of Error log collecting */
> +	AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1,
> +	/* GIM supports feature of loading uCodes */
> +	AMDGIM_FEATURE_GIM_LOAD_UCODES   = 0x2,
> +};
> +
> +struct amdgim_pf2vf_info_header {
> +	/* the total structure size in byte. */
> +	uint32_t size;
> +	/* version of this structure, written by the GIM */
> +	uint32_t version;
> +} __aligned(4);
> +struct  amdgim_pf2vf_info_v1 {
> +	/* header contains size and version */
> +	struct amdgim_pf2vf_info_header header;
> +	/* max_width * max_height */
> +	unsigned int uvd_enc_max_pixels_count;
> +	/* 16x16 pixels/sec, codec independent */
> +	unsigned int uvd_enc_max_bandwidth;
> +	/* max_width * max_height */
> +	unsigned int vce_enc_max_pixels_count;
> +	/* 16x16 pixels/sec, codec independent */
> +	unsigned int vce_enc_max_bandwidth;
> +	/* MEC FW position in kb from the start of visible frame buffer */
> +	unsigned int mecfw_kboffset;
> +	/* The features flags of the GIM driver supports. */
> +	unsigned int feature_flags;
> +	/* use private key from mailbox 2 to create chueksum */
> +	unsigned int checksum;
> +} __aligned(4);
> +
> +struct  amdgim_pf2vf_info_v2 {
> +	/* header contains size and version */
> +	struct amdgim_pf2vf_info_header header;
> +	/* use private key from mailbox 2 to create chueksum */
> +	unsigned int checksum;
> +	/* The features flags of the GIM driver supports. */
> +	unsigned int feature_flags;
> +	/* max_width * max_height */
> +	unsigned int uvd_enc_max_pixels_count;
> +	/* 16x16 pixels/sec, codec independent */
> +	unsigned int uvd_enc_max_bandwidth;
> +	/* max_width * max_height */
> +	unsigned int vce_enc_max_pixels_count;
> +	/* 16x16 pixels/sec, codec independent */
> +	unsigned int vce_enc_max_bandwidth;
> +	/* MEC FW position in kb from the start of VF visible frame buffer */
> +	unsigned int mecfw_kboffset;
> +	/* MEC FW size in KB */
> +	unsigned int mecfw_ksize;
> +	/* UVD FW position in kb from the start of VF visible frame buffer */
> +	unsigned int uvdfw_kboffset;
> +	/* UVD FW size in KB */
> +	unsigned int uvdfw_ksize;
> +	/* VCE FW position in kb from the start of VF visible frame buffer */
> +	unsigned int vcefw_kboffset;
> +	/* VCE FW size in KB */
> +	unsigned int vcefw_ksize;
> +	unsigned int
> reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (9 +
> sizeof(struct amdgim_pf2vf_info_header)/sizeof(uint32_t)), 3)];
> +} __aligned(4);
> +
> +
> +struct amdgim_vf2pf_info_header {
> +	/* the total structure size in byte. */
> +	uint32_t size;
> +	/*version of this structure, written by the guest */
> +	uint32_t version;
> +} __aligned(4);
> +
> +struct amdgim_vf2pf_info_v1 {
> +	/* header contains size and version */
> +	struct amdgim_vf2pf_info_header header;
> +	/* driver version */
> +	char driver_version[64];
> +	/* driver certification, 1=WHQL, 0=None */
> +	unsigned int driver_cert;
> +	/* guest OS type and version: need a define */
> +	unsigned int os_info;
> +	/* in the unit of 1M */
> +	unsigned int fb_usage;
> +	/* guest gfx engine usage percentage */
> +	unsigned int gfx_usage;
> +	/* guest gfx engine health percentage */
> +	unsigned int gfx_health;
> +	/* guest compute engine usage percentage */
> +	unsigned int compute_usage;
> +	/* guest compute engine health percentage */
> +	unsigned int compute_health;
> +	/* guest vce engine usage percentage. 0xffff means N/A. */
> +	unsigned int vce_enc_usage;
> +	/* guest vce engine health percentage. 0xffff means N/A. */
> +	unsigned int vce_enc_health;
> +	/* guest uvd engine usage percentage. 0xffff means N/A. */
> +	unsigned int uvd_enc_usage;
> +	/* guest uvd engine usage percentage. 0xffff means N/A. */
> +	unsigned int uvd_enc_health;
> +	unsigned int checksum;
> +} __aligned(4);
> +
> +struct amdgim_vf2pf_info_v2 {
> +	/* header contains size and version */
> +	struct amdgim_vf2pf_info_header header;
> +	unsigned int checksum;
> +	/* driver version */
> +	char driver_version[64];
> +	/* driver certification, 1=WHQL, 0=None */
> +	unsigned int driver_cert;
> +	/* guest OS type and version: need a define */
> +	unsigned int os_info;
> +	/* in the unit of 1M */
> +	unsigned int fb_usage;
> +	/* guest gfx engine usage percentage */
> +	unsigned int gfx_usage;
> +	/* guest gfx engine health percentage */
> +	unsigned int gfx_health;
> +	/* guest compute engine usage percentage */
> +	unsigned int compute_usage;
> +	/* guest compute engine health percentage */
> +	unsigned int compute_health;
> +	/* guest vce engine usage percentage. 0xffff means N/A. */
> +	unsigned int vce_enc_usage;
> +	/* guest vce engine health percentage. 0xffff means N/A. */
> +	unsigned int vce_enc_health;
> +	/* guest uvd engine usage percentage. 0xffff means N/A. */
> +	unsigned int uvd_enc_usage;
> +	/* guest uvd engine usage percentage. 0xffff means N/A. */
> +	unsigned int uvd_enc_health;
> +	unsigned int
> reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 64, 0, (12 +
> sizeof(struct amdgim_vf2pf_info_header)/sizeof(uint32_t)), 0)];
> +} __aligned(4);
> +
> +#define AMDGPU_FW_VRAM_VF2PF_VER 2
> +typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ;
> +
> +#define AMDGPU_FW_VRAM_VF2PF_WRITE(adev, field, val) \
> +	do { \
> +		((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)-
> >field = (val); \
> +	} while (0)
> +
> +#define AMDGPU_FW_VRAM_VF2PF_READ(adev, field, val) \
> +	do { \
> +		(*val) = ((amdgim_vf2pf_info *)adev-
> >virt.fw_reserve.p_vf2pf)->field; \
> +	} while (0)
> +
> +#define AMDGPU_FW_VRAM_PF2VF_READ(adev, field, val) \
> +	do { \
> +		if (!adev->virt.fw_reserve.p_pf2vf) \
> +			*(val) = 0; \
> +		else { \
> +			if (adev->virt.fw_reserve.p_pf2vf->version == 1) \
> +				*(val) = ((struct amdgim_pf2vf_info_v1
> *)adev->virt.fw_reserve.p_pf2vf)->field; \
> +			if (adev->virt.fw_reserve.p_pf2vf->version == 2) \
> +				*(val) = ((struct amdgim_pf2vf_info_v2
> *)adev->virt.fw_reserve.p_pf2vf)->field; \
> +		} \
> +	} while (0)
> +
>  /* GPU virtualization */
>  struct amdgpu_virt {
>  	uint32_t			caps;
> @@ -72,6 +245,7 @@ struct amdgpu_virt {
>  	struct amdgpu_mm_table		mm_table;
>  	const struct amdgpu_virt_ops	*ops;
>  	struct amdgpu_vf_error_buffer   vf_errors;
> +	struct amdgpu_virt_fw_reserve	fw_reserve;
>  };
> 
>  #define AMDGPU_CSA_SIZE    (8 * 1024)
> @@ -114,5 +288,9 @@ int amdgpu_virt_reset_gpu(struct amdgpu_device
> *adev);
>  int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct
> amdgpu_job *job);
>  int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
>  void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
> +int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long
> obj_size,
> +					unsigned int key,
> +					unsigned int chksum);
> +void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev);
> 
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
> b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
> index 2812d88..b4906d2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
> +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
> @@ -183,6 +183,12 @@ static int xgpu_ai_send_access_requests(struct
> amdgpu_device *adev,
>  			pr_err("Doesn't get READY_TO_ACCESS_GPU from
> pf, give up\n");
>  			return r;
>  		}
> +		/* Retrieve checksum from mailbox2 */
> +		if (req == IDH_REQ_GPU_INIT_ACCESS) {
> +			adev->virt.fw_reserve.checksum_key =
> +				RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO,
> 0,
> +
> 	mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2));
> +		}
>  	}
> 
>  	return 0;
> --
> 2.7.4
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx at lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


More information about the amd-gfx mailing list