[PATCH 3/4] drm/amdkfd: report pcie bandwidth as number of lanes

[Felix Kuehling]

On 2021-06-21 3:23 p.m., Jonathan Kim wrote:
> Similar to xGMI reporting the min/max bandwidth as the number of links
> between peers, PCIe will report the min/max bandwidth as the number of
> supported lanes.
>
> Signed-off-by: Jonathan Kim <jonathan.kim at amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 24 ++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  3 +++
>   drivers/gpu/drm/amd/amdkfd/kfd_crat.c      |  3 +++
>   3 files changed, 30 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index c84989eda8eb..99c662b70519 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -568,6 +568,30 @@ uint8_t amdgpu_amdkfd_get_xgmi_num_links(struct kgd_dev *dst, struct kgd_dev *sr
>   	return  (uint8_t)ret;
>   }
>   
> +uint32_t amdgpu_amdkfd_get_pcie_min_lanes(struct kgd_dev *dev)
> +{
> +	struct amdgpu_device *adev = (struct amdgpu_device *)dev;
> +	int min_lane_shift = ffs(adev->pm.pcie_mlw_mask >>
> +				CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
> +
> +	if (min_lane_shift < 0)
> +		return 0;
> +
> +	return 1UL << min_lane_shift;
> +}
> +
> +uint32_t amdgpu_amdkfd_get_pcie_max_lanes(struct kgd_dev *dev)
> +{
> +	struct amdgpu_device *adev = (struct amdgpu_device *)dev;
> +	int max_lane_shift = fls(adev->pm.pcie_mlw_mask >>
> +				CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
> +
> +	if (max_lane_shift < 0)
> +		return 0;
> +
> +	return 1UL << max_lane_shift;
> +}
> +
>   uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
>   {
>   	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index 20e4bfce62be..88322c72a43d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -31,6 +31,7 @@
>   #include <linux/workqueue.h>
>   #include <kgd_kfd_interface.h>
>   #include <drm/ttm/ttm_execbuf_util.h>
> +#include "amd_pcie.h"
>   #include "amdgpu_sync.h"
>   #include "amdgpu_vm.h"
>   
> @@ -227,6 +228,8 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
>   int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
>   uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
>   uint8_t amdgpu_amdkfd_get_xgmi_num_links(struct kgd_dev *dst, struct kgd_dev *src);
> +uint32_t amdgpu_amdkfd_get_pcie_min_lanes(struct kgd_dev *dev);
> +uint32_t amdgpu_amdkfd_get_pcie_max_lanes(struct kgd_dev *dev);
>   
>   /* Read user wptr from a specified user address space with page fault
>    * disabled. The memory must be pinned and mapped to the hardware when
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index 75047b77649b..f70d69035fe7 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -1036,6 +1036,7 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
>   			props->max_latency = iolink->maximum_latency;
>   			props->min_bandwidth = iolink->minimum_bandwidth;
>   			props->max_bandwidth = iolink->maximum_bandwidth;
> +
>   			props->rec_transfer_size =
>   					iolink->recommended_transfer_size;
>   
> @@ -1993,6 +1994,8 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
>   		sub_type_hdr->maximum_bandwidth = 1;
>   	} else {
>   		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
> +		sub_type_hdr->minimum_bandwidth = amdgpu_amdkfd_get_pcie_min_lanes(kdev->kgd);
> +		sub_type_hdr->maximum_bandwidth = amdgpu_amdkfd_get_pcie_max_lanes(kdev->kgd);

Similar to XGMI, I did not mean to directly report the number of lanes 
here. Instead, used it to calculate the actual bandwidth in MB/s.

Regards,
   Felix


>   	}
>   
>   	sub_type_hdr->proximity_domain_from = proximity_domain;