[PATCH v2] drm/amdkfd: CRIU export dmabuf handles for GTT BOs

[Felix Kuehling]

Am 2022-03-08 um 16:08 schrieb David Yat Sin:
> Export dmabuf handles for GTT BOs so that their contents can be accessed
> using SDMA during checkpoint/restore.
>
> Signed-off-by: David Yat Sin <david.yatsin at amd.com>

Looks good to me. Please also post a link to the user mode change for this.

Note that the user mode code has not been merged upstream yet. I think 
this should be the final cleanup before the user mode CRIU plugin can be 
merged with the updated KFD version dependency.

Thanks,
   Felix


> ---
>   drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 12 ++++++++----
>   include/uapi/linux/kfd_ioctl.h           |  3 ++-
>   2 files changed, 10 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
> index 2c7d76e67ddb..e1e2362841f8 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
> @@ -1759,7 +1759,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
>   					goto exit;
>   				}
>   			}
> -			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
> +			if (bo_bucket->alloc_flags
> +			    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
>   				ret = criu_get_prime_handle(&dumper_bo->tbo.base,
>   						bo_bucket->alloc_flags &
>   						KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
> @@ -1812,7 +1813,8 @@ static int criu_checkpoint_bos(struct kfd_process *p,
>   
>   exit:
>   	while (ret && bo_index--) {
> -		if (bo_buckets[bo_index].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
> +		if (bo_buckets[bo_index].alloc_flags
> +		    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
>   			close_fd(bo_buckets[bo_index].dmabuf_fd);
>   	}
>   
> @@ -2211,7 +2213,8 @@ static int criu_restore_bo(struct kfd_process *p,
>   
>   	pr_debug("map memory was successful for the BO\n");
>   	/* create the dmabuf object and export the bo */
> -	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
> +	if (bo_bucket->alloc_flags
> +	    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
>   		ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR,
>   					    &bo_bucket->dmabuf_fd);
>   		if (ret)
> @@ -2281,7 +2284,8 @@ static int criu_restore_bos(struct kfd_process *p,
>   
>   exit:
>   	while (ret && i--) {
> -		if (bo_buckets[i].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
> +		if (bo_buckets[i].alloc_flags
> +		   & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
>   			close_fd(bo_buckets[i].dmabuf_fd);
>   	}
>   	kvfree(bo_buckets);
> diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
> index b40687bf1014..eb9ff85f8556 100644
> --- a/include/uapi/linux/kfd_ioctl.h
> +++ b/include/uapi/linux/kfd_ioctl.h
> @@ -33,9 +33,10 @@
>    * - 1.5 - Add SVM API
>    * - 1.6 - Query clear flags in SVM get_attr API
>    * - 1.7 - Checkpoint Restore (CRIU) API
> + * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
>    */
>   #define KFD_IOCTL_MAJOR_VERSION 1
> -#define KFD_IOCTL_MINOR_VERSION 7
> +#define KFD_IOCTL_MINOR_VERSION 8
>   
>   struct kfd_ioctl_get_version_args {
>   	__u32 major_version;	/* from KFD */