[RFC 10/29] nvkm/vgpu: introduce interfaces for NVIDIA vGPU VFIO module

[Zhi Wang]

NVIDIA vGPU VFIO module requires interfaces from the core driver support
to issue GSP RPCs, allocating FBMEM, create/destroy/reset vGPUs...

Implement interfaces to expose the core driver functions to
NVIDIA vGPU VFIO module.

Cc: Neo Jia <cjia at nvidia.com>
Cc: Surath Mitra <smitra at nvidia.com>
Signed-off-by: Zhi Wang <zhiw at nvidia.com>
---
 .../nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h  |  6 ++
 drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/Kbuild  |  1 +
 drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vfio.c  | 69 +++++++++++++++++++
 .../gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c  |  5 ++
 include/drm/nvkm_vgpu_mgr_vfio.h              | 24 +++++++
 5 files changed, 105 insertions(+)
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vfio.c
 create mode 100644 include/drm/nvkm_vgpu_mgr_vfio.h

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h b/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
index aaba6d9a88b4..5a856fa905f9 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
@@ -2,6 +2,8 @@
 #ifndef __NVKM_VGPU_MGR_H__
 #define __NVKM_VGPU_MGR_H__
 
+#include <drm/nvkm_vgpu_mgr_vfio.h>
+
 #define NVIDIA_MAX_VGPUS 2
 
 struct nvkm_vgpu_mgr {
@@ -17,6 +19,9 @@ struct nvkm_vgpu_mgr {
 	struct nvif_device_priv *dev_priv;
 
 	u64 vmmu_segment_size;
+
+	void *vfio_ops;
+	struct nvidia_vgpu_vfio_handle_data vfio_handle_data;
 };
 
 bool nvkm_vgpu_mgr_is_supported(struct nvkm_device *device);
@@ -25,5 +30,6 @@ int nvkm_vgpu_mgr_init(struct nvkm_device *device);
 void nvkm_vgpu_mgr_fini(struct nvkm_device *device);
 void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device,
 					void *info);
+void nvkm_vgpu_mgr_init_vfio_ops(struct nvkm_vgpu_mgr *vgpu_mgr);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/Kbuild b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/Kbuild
index 244e967d4edc..a62c10cb1446 100644
--- a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/Kbuild
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: MIT
 nvkm-y += nvkm/vgpu_mgr/vgpu_mgr.o
+nvkm-y += nvkm/vgpu_mgr/vfio.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vfio.c b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vfio.c
new file mode 100644
index 000000000000..e98c9e83ee60
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vfio.c
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: MIT */
+
+#include <core/device.h>
+
+#include <vgpu_mgr/vgpu_mgr.h>
+#include <drm/nvkm_vgpu_mgr_vfio.h>
+
+static bool vgpu_mgr_is_enabled(void *handle)
+{
+	struct nvkm_device *device = handle;
+
+	return nvkm_vgpu_mgr_is_enabled(device);
+}
+
+static void get_handle(void *handle,
+		       struct nvidia_vgpu_vfio_handle_data *data)
+{
+	struct nvkm_device *device = handle;
+	struct nvkm_vgpu_mgr *vgpu_mgr = &device->vgpu_mgr;
+
+	if (vgpu_mgr->vfio_handle_data.priv)
+		memcpy(data, &vgpu_mgr->vfio_handle_data, sizeof(*data));
+}
+
+static void detach_handle(void *handle)
+{
+	struct nvkm_device *device = handle;
+	struct nvkm_vgpu_mgr *vgpu_mgr = &device->vgpu_mgr;
+
+	vgpu_mgr->vfio_handle_data.priv = NULL;
+}
+
+static int attach_handle(void *handle,
+			 struct nvidia_vgpu_vfio_handle_data *data)
+{
+	struct nvkm_device *device = handle;
+	struct nvkm_vgpu_mgr *vgpu_mgr = &device->vgpu_mgr;
+
+	if (vgpu_mgr->vfio_handle_data.priv)
+		return -EEXIST;
+
+	memcpy(&vgpu_mgr->vfio_handle_data, data, sizeof(*data));
+	return 0;
+}
+
+struct nvkm_vgpu_mgr_vfio_ops nvkm_vgpu_mgr_vfio_ops = {
+	.vgpu_mgr_is_enabled = vgpu_mgr_is_enabled,
+	.get_handle = get_handle,
+	.attach_handle = attach_handle,
+	.detach_handle = detach_handle,
+};
+
+/**
+ * nvkm_vgpu_mgr_init_vfio_ops - init the callbacks for VFIO
+ * @vgpu_mgr: the nvkm vGPU manager
+ */
+void nvkm_vgpu_mgr_init_vfio_ops(struct nvkm_vgpu_mgr *vgpu_mgr)
+{
+	vgpu_mgr->vfio_ops = &nvkm_vgpu_mgr_vfio_ops;
+}
+
+struct nvkm_vgpu_mgr_vfio_ops *nvkm_vgpu_mgr_get_vfio_ops(void *handle)
+{
+	struct nvkm_device *device = handle;
+	struct nvkm_vgpu_mgr *vgpu_mgr = &device->vgpu_mgr;
+
+	return vgpu_mgr->vfio_ops;
+}
+EXPORT_SYMBOL(nvkm_vgpu_mgr_get_vfio_ops);
diff --git a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
index d2ea5a07cbfc..caeb805cf1c3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
@@ -1,4 +1,7 @@
 /* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 NVIDIA Corporation
+ */
 #include <core/device.h>
 #include <core/driver.h>
 #include <nvif/driverif.h>
@@ -132,6 +135,8 @@ int nvkm_vgpu_mgr_init(struct nvkm_device *device)
 	if (ret)
 		goto err_get_vmmu_seg_size;
 
+	nvkm_vgpu_mgr_init_vfio_ops(vgpu_mgr);
+
 	vgpu_mgr->enabled = true;
 	pci_info(nvkm_to_pdev(device),
 		 "NVIDIA vGPU mananger support is enabled.\n");
diff --git a/include/drm/nvkm_vgpu_mgr_vfio.h b/include/drm/nvkm_vgpu_mgr_vfio.h
new file mode 100644
index 000000000000..09ecc3dc454f
--- /dev/null
+++ b/include/drm/nvkm_vgpu_mgr_vfio.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 NVIDIA Corporation
+ */
+
+#ifndef __NVKM_VGPU_MGR_VFIO_H__
+#define __NVKM_VGPU_MGR_VFIO_H__
+
+struct nvidia_vgpu_vfio_handle_data {
+	void *priv;
+};
+
+struct nvkm_vgpu_mgr_vfio_ops {
+	bool (*vgpu_mgr_is_enabled)(void *handle);
+	void (*get_handle)(void *handle,
+		           struct nvidia_vgpu_vfio_handle_data *data);
+	int (*attach_handle)(void *handle,
+		             struct nvidia_vgpu_vfio_handle_data *data);
+	void (*detach_handle)(void *handle);
+};
+
+struct nvkm_vgpu_mgr_vfio_ops *nvkm_vgpu_mgr_get_vfio_ops(void *handle);
+
+#endif
-- 
2.34.1