[PATCH] drm/amdkfd: Fix reserved SDMA queues handling

[Felix Kuehling]

On 2023-06-07 11:27, Mukul Joshi wrote:
> This patch fixes a regression caused by a bad merge where
> the handling of reserved SDMA queues was accidentally removed.
> With the fix, the reserved SDMA queues are again correctly
> marked as unavailable for allocation.
>
> Fixes: c27842c84a848 ("drm/amdkfd: Update SDMA queue management for GFX9.4.3")
> Signed-off-by: Mukul Joshi <mukul.joshi at amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>


> ---
>   drivers/gpu/drm/amd/amdkfd/kfd_device.c             | 13 ++++++-------
>   .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c   | 10 +++++-----
>   drivers/gpu/drm/amd/amdkfd/kfd_priv.h               |  2 +-
>   3 files changed, 12 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> index 9fc9d32cb579..9d4abfd8b55e 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> @@ -106,20 +106,19 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
>   		kfd->device_info.num_sdma_queues_per_engine = 8;
>   	}
>   
> +	bitmap_zero(kfd->device_info.reserved_sdma_queues_bitmap, KFD_MAX_SDMA_QUEUES);
> +
>   	switch (sdma_version) {
>   	case IP_VERSION(6, 0, 0):
> +	case IP_VERSION(6, 0, 1):
>   	case IP_VERSION(6, 0, 2):
>   	case IP_VERSION(6, 0, 3):
>   		/* Reserve 1 for paging and 1 for gfx */
>   		kfd->device_info.num_reserved_sdma_queues_per_engine = 2;
>   		/* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */
> -		kfd->device_info.reserved_sdma_queues_bitmap = 0xFULL;
> -		break;
> -	case IP_VERSION(6, 0, 1):
> -		/* Reserve 1 for paging and 1 for gfx */
> -		kfd->device_info.num_reserved_sdma_queues_per_engine = 2;
> -		/* BIT(0)=engine-0 queue-0; BIT(1)=engine-0 queue-1; ... */
> -		kfd->device_info.reserved_sdma_queues_bitmap = 0x3ULL;
> +		bitmap_set(kfd->device_info.reserved_sdma_queues_bitmap, 0,
> +			   kfd->adev->sdma.num_instances *
> +			   kfd->device_info.num_reserved_sdma_queues_per_engine);
>   		break;
>   	default:
>   		break;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index 0c1be91a87c6..498ad7d4e7d9 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -123,11 +123,6 @@ unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
>   		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
>   }
>   
> -static inline uint64_t get_reserved_sdma_queues_bitmap(struct device_queue_manager *dqm)
> -{
> -	return dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap;
> -}
> -
>   static void init_sdma_bitmaps(struct device_queue_manager *dqm)
>   {
>   	bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
> @@ -135,6 +130,11 @@ static void init_sdma_bitmaps(struct device_queue_manager *dqm)
>   
>   	bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
>   	bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));
> +
> +	/* Mask out the reserved queues */
> +	bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap,
> +		      dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
> +		      KFD_MAX_SDMA_QUEUES);
>   }
>   
>   void program_sh_mem_settings(struct device_queue_manager *dqm,
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 023b17e0116b..7364a5d77c6e 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -239,7 +239,7 @@ struct kfd_device_info {
>   	uint32_t no_atomic_fw_version;
>   	unsigned int num_sdma_queues_per_engine;
>   	unsigned int num_reserved_sdma_queues_per_engine;
> -	uint64_t reserved_sdma_queues_bitmap;
> +	DECLARE_BITMAP(reserved_sdma_queues_bitmap, KFD_MAX_SDMA_QUEUES);
>   };
>   
>   unsigned int kfd_get_num_sdma_engines(struct kfd_node *kdev);