[PATCH 3/4] drm/amdkfd: report pcie bandwidth as number of lanes

[Kim, Jonathan]

[AMD Official Use Only]

Ping on series.
Note Patch 4 can be dropped.  Runtime doesn't require an extra flag to determine direct connections.

Thanks,

Jon

> -----Original Message-----
> From: Kim, Jonathan <jonathan.kim at amd.com>
> Sent: Monday, June 21, 2021 3:24 PM
> To: amd-gfx at lists.freedesktop.org
> Cc: Zhang, Hawking <Hawking.Zhang at amd.com>; Kuehling, Felix
> <Felix.Kuehling at amd.com>; Kim, Jonathan <Jonathan.Kim at amd.com>
> Subject: [PATCH 3/4] drm/amdkfd: report pcie bandwidth as number of lanes
>
> Similar to xGMI reporting the min/max bandwidth as the number of links
> between peers, PCIe will report the min/max bandwidth as the number of
> supported lanes.
>
> Signed-off-by: Jonathan Kim <jonathan.kim at amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 24
> ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  3 +++
>  drivers/gpu/drm/amd/amdkfd/kfd_crat.c      |  3 +++
>  3 files changed, 30 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index c84989eda8eb..99c662b70519 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -568,6 +568,30 @@ uint8_t
> amdgpu_amdkfd_get_xgmi_num_links(struct kgd_dev *dst, struct kgd_dev
> *sr
>       return  (uint8_t)ret;
>  }
>
> +uint32_t amdgpu_amdkfd_get_pcie_min_lanes(struct kgd_dev *dev) {
> +     struct amdgpu_device *adev = (struct amdgpu_device *)dev;
> +     int min_lane_shift = ffs(adev->pm.pcie_mlw_mask >>
> +                             CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
> +
> +     if (min_lane_shift < 0)
> +             return 0;
> +
> +     return 1UL << min_lane_shift;
> +}
> +
> +uint32_t amdgpu_amdkfd_get_pcie_max_lanes(struct kgd_dev *dev) {
> +     struct amdgpu_device *adev = (struct amdgpu_device *)dev;
> +     int max_lane_shift = fls(adev->pm.pcie_mlw_mask >>
> +                             CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
> +
> +     if (max_lane_shift < 0)
> +             return 0;
> +
> +     return 1UL << max_lane_shift;
> +}
> +
>  uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev
> *kgd)  {
>       struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff --git
> a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index 20e4bfce62be..88322c72a43d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -31,6 +31,7 @@
>  #include <linux/workqueue.h>
>  #include <kgd_kfd_interface.h>
>  #include <drm/ttm/ttm_execbuf_util.h>
> +#include "amd_pcie.h"
>  #include "amdgpu_sync.h"
>  #include "amdgpu_vm.h"
>
> @@ -227,6 +228,8 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct
> kgd_dev *kgd);  int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
> uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct
> kgd_dev *src);  uint8_t amdgpu_amdkfd_get_xgmi_num_links(struct kgd_dev
> *dst, struct kgd_dev *src);
> +uint32_t amdgpu_amdkfd_get_pcie_min_lanes(struct kgd_dev *dev);
> +uint32_t amdgpu_amdkfd_get_pcie_max_lanes(struct kgd_dev *dev);
>
>  /* Read user wptr from a specified user address space with page fault
>   * disabled. The memory must be pinned and mapped to the hardware
> when diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index 75047b77649b..f70d69035fe7 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -1036,6 +1036,7 @@ static int kfd_parse_subtype_iolink(struct
> crat_subtype_iolink *iolink,
>                       props->max_latency = iolink->maximum_latency;
>                       props->min_bandwidth = iolink-
> >minimum_bandwidth;
>                       props->max_bandwidth = iolink-
> >maximum_bandwidth;
> +
>                       props->rec_transfer_size =
>                                       iolink->recommended_transfer_size;
>
> @@ -1993,6 +1994,8 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int
> *avail_size,
>               sub_type_hdr->maximum_bandwidth = 1;
>       } else {
>               sub_type_hdr->io_interface_type =
> CRAT_IOLINK_TYPE_PCIEXPRESS;
> +             sub_type_hdr->minimum_bandwidth =
> amdgpu_amdkfd_get_pcie_min_lanes(kdev->kgd);
> +             sub_type_hdr->maximum_bandwidth =
> +amdgpu_amdkfd_get_pcie_max_lanes(kdev->kgd);
>       }
>
>       sub_type_hdr->proximity_domain_from = proximity_domain;
> --
> 2.25.1