[PATCH 1/7] drm/amdkfd: Add gws number to kfd topology node properties

[Kuehling, Felix]

On 2019-05-15 4:37 p.m., Zeng, Oak wrote:
> Add amdgpu_amdkfd interface to get num_gws and add num_gws
> to /sys/class/kfd/kfd/topology/nodes/x/properties. Only report
> num_gws if MEC FW support GWS barriers. Currently it is
> determined by a environment variable which will be replaced

It's not an environment variable, it's a module parameter. With that 
fixed, the patch is Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>

Patches 2, 3, 7 are also Reviewed-by: Felix Kuehling 
<Felix.Kuehling at amd.com>. I replied with comments on patches 4, 5, 6.

Regards,
   Felix


> with MEC FW version check when firmware is ready.
>
> Change-Id: Ie0d00fb20a37ef2856860dbecbe1ad0ca1ef09f7
> Signed-off-by: Oak Zeng <Oak.Zeng at amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c |  7 +++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    | 10 ++++++++++
>   drivers/gpu/drm/amd/amdkfd/kfd_priv.h      |  5 +++++
>   drivers/gpu/drm/amd/amdkfd/kfd_topology.c  |  5 +++++
>   drivers/gpu/drm/amd/amdkfd/kfd_topology.h  |  1 +
>   6 files changed, 29 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index 98326e3b..a4780d5 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -544,6 +544,13 @@ uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
>   	return adev->rmmio_remap.bus_addr;
>   }
>   
> +uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd)
> +{
> +	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
> +
> +	return adev->gds.gws_size;
> +}
> +
>   int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
>   				uint32_t vmid, uint64_t gpu_addr,
>   				uint32_t *ib_cmd, uint32_t ib_len)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f57f297..5700643 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -169,6 +169,7 @@ int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
>   uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
>   uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
>   uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd);
> +uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
>   uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
>   
>   #define read_user_wptr(mmptr, wptr, dst)				\
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index a334d3b..3a03c2b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -666,6 +666,16 @@ MODULE_PARM_DESC(noretry,
>   int halt_if_hws_hang;
>   module_param(halt_if_hws_hang, int, 0644);
>   MODULE_PARM_DESC(halt_if_hws_hang, "Halt if HWS hang is detected (0 = off (default), 1 = on)");
> +
> +/**
> + * DOC: hws_gws_support(bool)
> + * Whether HWS support gws barriers. Default value: false (not supported)
> + * This will be replaced with a MEC firmware version check once firmware
> + * is ready
> + */
> +bool hws_gws_support;
> +module_param(hws_gws_support, bool, 0444);
> +MODULE_PARM_DESC(hws_gws_support, "MEC FW support gws barriers (false = not supported (Default), true = supported)");
>   #endif
>   
>   /**
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 8f02d78..338fb07 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -160,6 +160,11 @@ extern int noretry;
>    */
>   extern int halt_if_hws_hang;
>   
> +/*
> + * Whether MEC FW support GWS barriers
> + */
> +extern bool hws_gws_support;
> +
>   enum cache_policy {
>   	cache_policy_coherent,
>   	cache_policy_noncoherent
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> index 2c06d6c..128c72c 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> @@ -454,6 +454,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
>   			dev->node_props.lds_size_in_kb);
>   	sysfs_show_32bit_prop(buffer, "gds_size_in_kb",
>   			dev->node_props.gds_size_in_kb);
> +	sysfs_show_32bit_prop(buffer, "num_gws",
> +			dev->node_props.num_gws);
>   	sysfs_show_32bit_prop(buffer, "wave_front_size",
>   			dev->node_props.wave_front_size);
>   	sysfs_show_32bit_prop(buffer, "array_count",
> @@ -1290,6 +1292,9 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
>   	dev->node_props.num_sdma_engines = gpu->device_info->num_sdma_engines;
>   	dev->node_props.num_sdma_xgmi_engines =
>   				gpu->device_info->num_xgmi_sdma_engines;
> +	dev->node_props.num_gws = (hws_gws_support &&
> +		dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
> +		amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0;
>   
>   	kfd_fill_mem_clk_max_info(dev);
>   	kfd_fill_iolink_non_crat_info(dev);
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
> index 949e885..276354a 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
> @@ -65,6 +65,7 @@ struct kfd_node_properties {
>   	uint32_t max_waves_per_simd;
>   	uint32_t lds_size_in_kb;
>   	uint32_t gds_size_in_kb;
> +	uint32_t num_gws;
>   	uint32_t wave_front_size;
>   	uint32_t array_count;
>   	uint32_t simd_arrays_per_engine;