[RFC] Add KFD available memory ioctl thunk support

[Yat Sin, David]

> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces at lists.freedesktop.org> On Behalf Of Daniel
> Phillips
> Sent: Tuesday, March 22, 2022 1:29 PM
> To: amd-gfx at lists.freedesktop.org
> Cc: Kuehling, Felix <Felix.Kuehling at amd.com>
> Subject: [RFC] Add KFD available memory ioctl thunk support
> 
> Hi all,
> 
> This patch adds thunk support for the new KFD memory availability ioctl.
> 
> I am posting this patch inline with Thunderbird just for now, to establish the
> principle that I can post patches/code old school style without mangling
> whitespace. Please bear with me while I get that sorted out. I will switch to
> posting with git am  pretty soon.
> 
>  include/hsakmt.h          |   11 +++++++++++
>  include/linux/kfd_ioctl.h |   18 ++++++++++++++++--
>  src/memory.c              |   23 +++++++++++++++++++++++
>  3 files changed, 50 insertions(+), 2 deletions(-)
> 
> Regards,
> Daniel
> 
> diff --git a/include/hsakmt.h b/include/hsakmt.h index ff2d023..abc617f
> 100644
> --- a/include/hsakmt.h
> +++ b/include/hsakmt.h
> @@ -374,6 +374,17 @@ hsaKmtFreeMemory(
>      HSAuint64   SizeInBytes         //IN
>      );
> 
> +/**
> +  Inquires memory available for allocation as a memory buffer */
> +
> +HSAKMT_STATUS
> +HSAKMTAPI
> +hsaKmtAvailableMemory(
> +    HSAuint32 Node,
> +    HSAuint64 *AvailableBytes
> +    );
> +
>  /**
>    Registers with KFD a memory buffer that may be accessed by the GPU  */
> diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index
> 8a0ed49..abfa948 100644
> --- a/include/linux/kfd_ioctl.h
> +++ b/include/linux/kfd_ioctl.h
> @@ -34,9 +34,10 @@
>   * - 1.6 - Query clear flags in SVM get_attr API
>   * - 1.7 - Checkpoint Restore (CRIU) API
>   * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
> + * - 1.9 - Add available_memory ioctl
>   */
>  #define KFD_IOCTL_MAJOR_VERSION 1
> -#define KFD_IOCTL_MINOR_VERSION 8
> +#define KFD_IOCTL_MINOR_VERSION 9
> 
>  /*
>   * Debug revision change log
> @@ -769,6 +770,16 @@ struct kfd_ioctl_free_memory_of_gpu_args {
>  	__u64 handle;		/* to KFD */
>  };
> 
> +/* Inquire available memory with kfd_ioctl_get_available_memory
> + *
> + * @available: memory available for alloc  */ struct
> +kfd_ioctl_get_available_memory_args {
> +	__u64 available;	/* from KFD */
> +	__u32 gpu_id;		/* to KFD */
> +	__u32 pad;
> +};
> +
>  /* Map memory to one or more GPUs
>   *
>   * @handle:                memory handle returned by alloc
> @@ -1328,7 +1339,7 @@ struct kfd_ioctl_set_xnack_mode_args {
>  		AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
> 
>  #define AMDKFD_COMMAND_START		0x01
> -#define AMDKFD_COMMAND_END		0x23
> +#define AMDKFD_COMMAND_END		0x24
> 
>  /* non-upstream ioctls */
>  #define AMDKFD_IOC_IPC_IMPORT_HANDLE                                    \
> diff --git a/src/memory.c b/src/memory.c index 6d2a4f4..b2cd759 100644
> --- a/src/memory.c
> +++ b/src/memory.c
> @@ -199,6 +199,29 @@ HSAKMT_STATUS HSAKMTAPI
> hsaKmtFreeMemory(void *MemoryAddress,
>  	return fmm_release(MemoryAddress);
>  }
> 
> +HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node,
> HSAuint64
> +*AvailableBytes) {
> +	struct kfd_ioctl_get_available_memory_args args = {};
> +	HSAKMT_STATUS result;
> +
> +	CHECK_KFD_OPEN();
> +	CHECK_KFD_MINOR_VERSION(7);

I think this should check for minor version 9 to match #define KFD_IOCTL_MINOR_VERSION 9 in kfd_ioctl.h

Regards,
David


> +
> +	pr_debug("[%s] node %d\n", __func__, Node);
> +
> +	result = validate_nodeid(Node, &args.gpu_id);
> +	if (result != HSAKMT_STATUS_SUCCESS) {
> +		pr_err("[%s] invalid node ID: %d\n", __func__, Node);
> +		return result;
> +	}
> +
> +	if (kmtIoctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args))
> +		return HSAKMT_STATUS_ERROR;
> +
> +	*AvailableBytes = args.available;
> +	return HSAKMT_STATUS_SUCCESS;
> +}
> +
>  HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void
> *MemoryAddress,
>  					     HSAuint64 MemorySizeInBytes)
>  {