[PATCH v3 04/16] drm/amdkfd: CRIU Implement KFD dumper ioctl
David Yat Sin
david.yatsin at amd.com
Wed Sep 29 12:00:26 UTC 2021
From: Rajneesh Bhardwaj <rajneesh.bhardwaj at amd.com>
This adds support to discover the buffer objects that belong to a
process being checkpointed. The data corresponding to these buffer
objects is returned to user space plugin running under criu master
context which then stores this info to recreate these buffer objects
during a restore operation.
Signed-off-by: David Yat Sin <david.yatsin at amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj at amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 20 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 2 +
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 188 ++++++++++++++++++++++-
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 3 +-
4 files changed, 211 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index e2896ac2c9ce..5d557180cd49 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1160,6 +1160,26 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev,
return ttm_pool_free(&adev->mman.bdev.pool, ttm);
}
+/**
+ * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
+ * task
+ *
+ * @tbo: The ttm_buffer_object that contains the userptr
+ * @user_addr: The returned value
+ */
+int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
+ uint64_t *user_addr)
+{
+ struct amdgpu_ttm_tt *gtt;
+
+ if (!tbo->ttm)
+ return -EINVAL;
+
+ gtt = (void *)tbo->ttm;
+ *user_addr = gtt->userptr;
+ return 0;
+}
+
/**
* amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
* task
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index e69f3e8e06e5..a7c0e6372339 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -177,6 +177,8 @@ static inline bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
#endif
void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
+ uint64_t *user_addr);
int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,
uint64_t addr, uint32_t flags);
bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 1906ded40698..cc3d8fd1d26f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -42,6 +42,7 @@
#include "kfd_svm.h"
#include "amdgpu_amdkfd.h"
#include "kfd_smi_events.h"
+#include "amdgpu_object.h"
static long kfd_ioctl(struct file *, unsigned int, unsigned long);
static int kfd_open(struct inode *, struct file *);
@@ -1841,6 +1842,44 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
return -EPERM;
}
#endif
+static int criu_dump_process(struct kfd_process *p, struct kfd_ioctl_criu_dumper_args *args)
+{
+ int ret;
+ struct kfd_criu_process_bucket *process_bucket;
+ struct kfd_criu_process_priv_data *process_priv;
+
+ if (args->num_objects != 1) {
+ pr_err("Only 1 process supported\n");
+ return -EINVAL;
+ }
+
+ if (args->objects_size != sizeof(*process_bucket) + sizeof(*process_priv)) {
+ pr_err("Invalid objects size for process\n");
+ return -EINVAL;
+ }
+
+ process_bucket = kzalloc(args->objects_size, GFP_KERNEL);
+ if (!process_bucket)
+ return -ENOMEM;
+
+ /* Private data starts after process bucket */
+ process_priv = (void *)(process_bucket + 1);
+
+ process_priv->version = KFD_CRIU_PRIV_VERSION;
+
+ process_bucket->priv_data_offset = 0;
+ process_bucket->priv_data_size = sizeof(*process_priv);
+
+ ret = copy_to_user((void __user *)args->objects, process_bucket, args->objects_size);
+ if (ret) {
+ pr_err("Failed to copy process information to user\n");
+ ret = -EFAULT;
+ }
+
+ kfree(process_bucket);
+ return ret;
+}
+
uint64_t get_process_num_bos(struct kfd_process *p)
{
uint64_t num_of_bos = 0, i;
@@ -1861,10 +1900,157 @@ uint64_t get_process_num_bos(struct kfd_process *p)
return num_of_bos;
}
+static int criu_dump_bos(struct kfd_process *p, struct kfd_ioctl_criu_dumper_args *args)
+{
+ struct kfd_criu_bo_bucket *bo_buckets;
+ struct kfd_criu_bo_priv_data *bo_privs;
+ uint64_t num_bos;
+
+ int ret = 0, pdd_index, bo_index = 0, id;
+ void *mem;
+
+ num_bos = get_process_num_bos(p);
+
+ if (args->num_objects != num_bos) {
+ pr_err("Mismatch with number of BOs (current:%lld user:%lld)\n",
+ num_bos, args->num_objects);
+ return -EINVAL;
+ }
+
+ if (args->objects_size != args->num_objects * (sizeof(*bo_buckets) + sizeof(*bo_privs))) {
+ pr_err("Invalid objects size for BOs\n");
+ return -EINVAL;
+ }
+
+ bo_buckets = kvzalloc(args->objects_size, GFP_KERNEL);
+ if (!bo_buckets)
+ return -ENOMEM;
+
+ /* Private data for first BO starts after all bo_buckets */
+ bo_privs = (void *)(bo_buckets + args->num_objects);
+
+ for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
+ struct kfd_process_device *pdd = p->pdds[pdd_index];
+ struct amdgpu_bo *dumper_bo;
+ struct kgd_mem *kgd_mem;
+
+ idr_for_each_entry(&pdd->alloc_idr, mem, id) {
+ struct kfd_criu_bo_bucket *bo_bucket;
+ struct kfd_criu_bo_priv_data *bo_priv;
+
+ if (!mem) {
+ ret = -ENOMEM;
+ goto exit;
+ }
+
+ kgd_mem = (struct kgd_mem *)mem;
+ dumper_bo = kgd_mem->bo;
+
+ if ((uint64_t)kgd_mem->va <= pdd->gpuvm_base)
+ continue;
+
+ bo_bucket = &bo_buckets[bo_index];
+ bo_priv = &bo_privs[bo_index];
+
+ bo_bucket->addr = (uint64_t)kgd_mem->va;
+ bo_bucket->size = amdgpu_bo_size(dumper_bo);
+ bo_bucket->gpu_id = pdd->dev->id;
+ bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
+
+ bo_bucket->priv_data_offset = bo_index * sizeof(*bo_priv);
+ bo_bucket->priv_data_size = sizeof(*bo_priv);
+
+ bo_priv->idr_handle = id;
+ if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+ ret = amdgpu_ttm_tt_get_userptr(&dumper_bo->tbo,
+ &bo_priv->user_addr);
+ if (ret) {
+ pr_err("Failed to obtain user address for user-pointer bo\n");
+ goto exit;
+ }
+ }
+ if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
+ bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
+ KFD_MMAP_GPU_ID(pdd->dev->id);
+ else if (bo_bucket->alloc_flags &
+ KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
+ bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
+ KFD_MMAP_GPU_ID(pdd->dev->id);
+ else
+ bo_bucket->offset = amdgpu_bo_mmap_offset(dumper_bo);
+
+ pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
+ "gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
+ bo_bucket->size,
+ bo_bucket->addr,
+ bo_bucket->offset,
+ bo_bucket->gpu_id,
+ bo_bucket->alloc_flags,
+ bo_priv->idr_handle);
+ bo_index++;
+ }
+ }
+
+ ret = copy_to_user((void __user *)args->objects, bo_buckets, args->objects_size);
+ if (ret) {
+ pr_err("Failed to copy bo information to user\n");
+ ret = -EFAULT;
+ }
+
+exit:
+ kvfree(bo_buckets);
+ return ret;
+}
+
static int kfd_ioctl_criu_dumper(struct file *filep,
struct kfd_process *p, void *data)
{
- return 0;
+ struct kfd_ioctl_criu_dumper_args *args = data;
+ int ret;
+
+ pr_debug("CRIU dump type:%d\n", args->type);
+
+ if (!args->objects || !args->objects_size)
+ return -EINVAL;
+
+ if (args->objects_index_start != 0) {
+ /* Partial to be implemented later */
+ pr_err("Partial dumps not supported\n");
+ return -EINVAL;
+ }
+
+ mutex_lock(&p->mutex);
+
+ if (!kfd_has_process_device_data(p)) {
+ pr_err("No pdd for given process\n");
+ ret = -ENODEV;
+ goto err_unlock;
+ }
+
+ switch (args->type) {
+ case KFD_CRIU_OBJECT_TYPE_PROCESS:
+ ret = criu_dump_process(p, args);
+ break;
+ case KFD_CRIU_OBJECT_TYPE_BO:
+ ret = criu_dump_bos(p, args);
+ break;
+ case KFD_CRIU_OBJECT_TYPE_QUEUE:
+ case KFD_CRIU_OBJECT_TYPE_EVENT:
+ case KFD_CRIU_OBJECT_TYPE_DEVICE:
+ case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
+ default:
+ pr_err("Unsupported object type:%d\n", args->type);
+ ret = -EINVAL;
+ }
+
+err_unlock:
+ mutex_unlock(&p->mutex);
+ if (ret)
+ pr_err("Failed to dump CRIU type:%d ret:%d\n", args->type, ret);
+ else
+ pr_debug("CRIU dump type:%d ret:%d\n", args->type, ret);
+
+ return ret;
}
static int kfd_ioctl_criu_restorer(struct file *filep,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 914306209c9c..5940d594b420 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1042,7 +1042,8 @@ struct kfd_criu_device_priv_data {
};
struct kfd_criu_bo_priv_data {
- uint64_t reserved;
+ uint64_t user_addr;
+ uint32_t idr_handle;
};
struct kfd_criu_svm_range_priv_data {
--
2.17.1
More information about the amd-gfx
mailing list