[RFC] Add KFD available memory ioctl

[Daniel Phillips]

Hi all,

This patch adds a new KFD ioctl to return the largest possible memory
size that can be allocated as a buffer object by the existing
kfd_ioctl_alloc_memory_of_gpu. It attempts to use exactly the same 
accept/reject criteria as kfd_ioctl_alloc_memory_of_gpu, so that 
allocating a new buffer object of the size returned by this new ioctl is 
guaranteed to succeed, barring races with other allocating tasks.

Thunk patches to expose and to test the new ioctl will follow separately.

This patch is still in testing, so I am not asking for merge yet. This
work is targeted for ROCm 5.2, however we may be getting a tad close to
feature freeze time for that.

Regards,
Daniel

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h       |    1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c |   14 ++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c         |   15 +++++++++++++++
include/uapi/linux/kfd_ioctl.h                   |   16 +++++++++++++---
4 files changed, 43 insertions(+), 3 deletions(-)

Signed-off-by: Daniel Phillips <daniel.phillips at amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4cb14c2fe53f..0a4c9187cba1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct 
amdgpu_device *adev,
 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
                     void *drm_priv);
 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
         struct amdgpu_device *adev, uint64_t va, uint64_t size,
         void *drm_priv, struct kgd_mem **mem,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c 
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index cd89d2e46852..f904117e4960 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1494,6 +1494,20 @@ int amdgpu_amdkfd_criu_resume(void *p)
     return ret;
 }

+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+    uint64_t reserved_for_pt =
+        ALIGN(ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size), PAGE_SIZE);
+    size_t available;
+
+    spin_lock(&kfd_mem_limit.mem_limit_lock);
+    available =
+        adev->gmc.real_vram_size -
+        adev->kfd.vram_used - reserved_for_pt;
+    spin_unlock(&kfd_mem_limit.mem_limit_lock);
+    return available;
+}
+
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
         struct amdgpu_device *adev, uint64_t va, uint64_t size,
         void *drm_priv, struct kgd_mem **mem,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c 
b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 59d3fe269e7c..a63683a854ee 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -961,6 +961,19 @@ bool kfd_dev_is_large_bar(struct kfd_dev *dev)
     return false;
 }

+static int kfd_ioctl_get_available_memory(struct file *filep,
+                     struct kfd_process *p, void *data)
+{
+    struct kfd_ioctl_get_available_memory_args *args = data;
+    struct kfd_dev *dev;
+
+    dev = kfd_device_by_id(args->gpu_id);
+    if (!dev)
+        return -EINVAL;
+    args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+    return 0;
+}
+
 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
                     struct kfd_process *p, void *data)
 {
@@ -2693,6 +2706,8 @@ static const struct amdkfd_ioctl_desc 
amdkfd_ioctls[] = {
     AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
             kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),

+    AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
+            kfd_ioctl_get_available_memory, 0),
 };

 #define AMDKFD_CORE_IOCTL_COUNT    ARRAY_SIZE(amdkfd_ioctls)
diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
index b40687bf1014..4ee3bcee12fd 100644
--- a/include/uapi/linux/kfd_ioctl.h
+++ b/include/uapi/linux/kfd_ioctl.h
@@ -33,9 +33,10 @@
  * - 1.5 - Add SVM API
  * - 1.6 - Query clear flags in SVM get_attr API
  * - 1.7 - Checkpoint Restore (CRIU) API
+ * - 1.8 - Add available memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 7
+#define KFD_IOCTL_MINOR_VERSION 8

 struct kfd_ioctl_get_version_args {
     __u32 major_version;    /* from KFD */
@@ -99,6 +100,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
     __u32 pad;
 };

+struct kfd_ioctl_get_available_memory_args {
+    __u64 available;    /* from KFD */
+    __u32 gpu_id;        /* to KFD */
+    __u32 pad;
+};
+
 /* For kfd_ioctl_set_memory_policy_args.default_policy and 
alternate_policy */
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -823,7 +830,10 @@ struct kfd_ioctl_set_xnack_mode_args {
 #define AMDKFD_IOC_CRIU_OP            \
         AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)

+#define AMDKFD_IOC_AVAILABLE_MEMORY            \
+        AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args)
+
 #define AMDKFD_COMMAND_START        0x01
-#define AMDKFD_COMMAND_END        0x23
+#define AMDKFD_COMMAND_END        0x24

-#endif
+        #endif