[Intel-gfx] [RFC 12/29] drm/i915: gvt: vGPU life cycle management
Zhi Wang
zhi.a.wang at intel.com
Thu Jan 28 02:21:34 PST 2016
This patch introduces vGPU life cycle management framework. vGPU instance
is a collection of virtual GEN hardware status, like virtual CFG/MMIO registers,
how much GGTT memory space/Fence this vGPU owns, etc.
A vGPU instance consists following virtualized/limited resources:
- Configuration space(virtualized)
- MMIO registers(virtualized)
- GGTT memory space(limited)
- GGTT page table(shadowed)
- Fence(limited)
The framework is responsible for createing/destroying a vGPU instance,
allocating/freeing the per-vGPU resource from GVT-g resource allocator,
presenting them in the virtual PVINFO page located in virtual MMIO bar, which
provides the basic foundation blocks to GVT-g CFG/MMIO emulation framework.
A big picture here looks like:
+-----------------------------------------------------------------------+
| XEN/KVM Hypervisor |
+---------------------------+--------^----------------------------------+
CFG/MMIO emulate request | | Emulation is done
from hypervisor | | Return the result to hypervisor
| |
+---------------------------v--------+-----------------------------------+
| GVT-g CFG/MMIO emulation framework |
+-----+-----^----------------+-----^---------------------+-------^-------+
| | | | | |
| | *vGPU instance | | *vGPU 2 | |
+-----v-----+----------+-----v-----+-------------+ +-----v-------+-------+
| vConfiguration Space |vGTT/MMIO/Fence registers| | ... |
+----------------------+-------------------------+ +---------------------+
vGPU life cycle management
Signed-off-by: Zhi Wang <zhi.a.wang at intel.com>
---
drivers/gpu/drm/i915/gvt/Makefile | 2 +-
drivers/gpu/drm/i915/gvt/gvt.h | 66 ++++++++++
drivers/gpu/drm/i915/gvt/instance.c | 235 ++++++++++++++++++++++++++++++++++++
drivers/gpu/drm/i915/gvt/mmio.c | 32 +++++
drivers/gpu/drm/i915/gvt/params.c | 1 +
drivers/gpu/drm/i915/gvt/params.h | 1 +
drivers/gpu/drm/i915/gvt/reg.h | 29 ++++-
drivers/gpu/drm/i915/i915_vgpu.h | 5 +-
8 files changed, 365 insertions(+), 6 deletions(-)
create mode 100644 drivers/gpu/drm/i915/gvt/instance.c
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 5d28ed1..f4dcf9a 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -1,4 +1,4 @@
-GVT_SOURCE := gvt.o params.o aperture_gm.o mmio.o handlers.o fb_decoder.o
+GVT_SOURCE := gvt.o params.o aperture_gm.o mmio.o handlers.o instance.o fb_decoder.o
ccflags-y += -I$(src) -I$(src)/.. -Wall -Werror -Wno-unused-function
i915_gvt-y := $(GVT_SOURCE)
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 798e216..c58305f 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -75,6 +75,17 @@ struct gvt_gm_node {
struct drm_mm_node *high_gm_node;
};
+struct gvt_virtual_mmio_state {
+ void *vreg;
+ void *sreg;
+};
+
+struct gvt_virtual_cfg_state {
+ unsigned char space[GVT_CFG_SPACE_SZ];
+ bool bar_mapped[GVT_BAR_NUM];
+ u64 bar_size[GVT_BAR_NUM];
+};
+
struct gvt_virtual_gm_state {
u64 aperture_base;
void *aperture_base_va;
@@ -89,6 +100,8 @@ struct gvt_virtual_gm_state {
struct gvt_virtual_device_state {
struct gvt_virtual_gm_state gm;
+ struct gvt_virtual_mmio_state mmio;
+ struct gvt_virtual_cfg_state cfg;
};
struct vgt_device {
@@ -96,6 +109,7 @@ struct vgt_device {
int vm_id;
struct pgt_device *pdev;
bool warn_untrack;
+ atomic_t active;
struct gvt_virtual_device_state state;
};
@@ -359,4 +373,56 @@ extern bool gvt_setup_initial_mmio_state(struct pgt_device *pdev);
extern void gvt_clean_mmio_emulation_state(struct pgt_device *pdev);
extern bool gvt_setup_mmio_emulation_state(struct pgt_device *pdev);
+static inline void gvt_pci_bar_write_32(struct vgt_device *vgt, uint32_t bar_offset, uint32_t val)
+{
+ uint32_t* cfg_reg;
+
+ /* BAR offset should be 32 bits algiend */
+ cfg_reg = (u32 *)&vgt->state.cfg.space[bar_offset & ~3];
+
+ /* only write the bits 31-4, leave the 3-0 bits unchanged, as they are read-only */
+ *cfg_reg = (val & 0xFFFFFFF0) | (*cfg_reg & 0xF);
+}
+
+static inline int gvt_pci_mmio_is_enabled(struct vgt_device *vgt)
+{
+ return vgt->state.cfg.space[GVT_REG_CFG_COMMAND] &
+ _REGBIT_CFG_COMMAND_MEMORY;
+}
+
+#define __vreg(vgt, off) (*(u32*)(vgt->state.mmio.vreg + off))
+#define __vreg8(vgt, off) (*(u8*)(vgt->state.mmio.vreg + off))
+#define __vreg16(vgt, off) (*(u16*)(vgt->state.mmio.vreg + off))
+#define __vreg64(vgt, off) (*(u64*)(vgt->state.mmio.vreg + off))
+
+#define __sreg(vgt, off) (*(u32*)(vgt->state.mmio.sreg + off))
+#define __sreg8(vgt, off) (*(u8*)(vgt->state.mmio.sreg + off))
+#define __sreg16(vgt, off) (*(u16*)(vgt->state.mmio.sreg + off))
+#define __sreg64(vgt, off) (*(u64*)(vgt->state.mmio.sreg + off))
+
+static inline void gvt_set_instance_online(struct vgt_device *vgt)
+{
+ atomic_set(&vgt->active, 1);
+}
+
+static inline void gvt_set_instance_offline(struct vgt_device *vgt)
+{
+ atomic_set(&vgt->active, 0);
+}
+
+static inline bool gvt_instance_is_online(struct vgt_device *vgt)
+{
+ return atomic_read(&vgt->active);
+}
+
+#define for_each_online_instance(pdev, vgt, id) \
+ idr_for_each_entry(&pdev->instance_idr, vgt, id) \
+ if (gvt_instance_is_online(vgt))
+
+extern void gvt_init_shadow_mmio_register(struct vgt_device *pdev);
+extern void gvt_init_virtual_mmio_register(struct vgt_device *pdev);
+extern struct vgt_device *gvt_create_instance(struct pgt_device *pdev,
+ struct gvt_instance_info *info);
+extern void gvt_destroy_instance(struct vgt_device *vgt);
+
#endif
diff --git a/drivers/gpu/drm/i915/gvt/instance.c b/drivers/gpu/drm/i915/gvt/instance.c
new file mode 100644
index 0000000..07b797a
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/instance.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "gvt.h"
+#include "i915_vgpu.h"
+
+static void destroy_virtual_mmio_state(struct vgt_device *vgt)
+{
+ struct gvt_virtual_device_state *state = &vgt->state;
+
+ if (state->mmio.vreg) {
+ vfree(state->mmio.vreg);
+ state->mmio.vreg = NULL;
+ }
+ if (state->mmio.sreg) {
+ vfree(state->mmio.sreg);
+ state->mmio.sreg = NULL;
+ }
+}
+
+static bool create_virtual_mmio_state(struct vgt_device *vgt)
+{
+ struct pgt_device *pdev = vgt->pdev;
+ struct gvt_virtual_device_state *state = &vgt->state;
+
+ state->mmio.vreg = vzalloc(pdev->mmio_size);
+ state->mmio.sreg = vzalloc(pdev->mmio_size);
+
+ if (state->mmio.vreg == NULL || state->mmio.sreg == NULL ) {
+ gvt_err("fail to allocate memory for virtual states.");
+ goto err;
+ }
+
+ gvt_init_shadow_mmio_register(vgt);
+ gvt_init_virtual_mmio_register(vgt);
+
+ return true;
+err:
+ destroy_virtual_mmio_state(vgt);
+ return false;
+}
+
+static void init_virtual_cfg_space_state(struct vgt_device *vgt,
+ struct gvt_instance_info *info)
+{
+ struct pgt_device *pdev = vgt->pdev;
+ struct gvt_virtual_device_state *state = &vgt->state;
+ int i;
+
+ char *cfg_space;
+ u16 *gmch_ctl;
+
+ cfg_space = state->cfg.space;
+
+ memcpy(cfg_space, pdev->initial_cfg_space, GVT_CFG_SPACE_SZ);
+ cfg_space[GVT_REG_CFG_SPACE_MSAC] = state->cfg.bar_size[1];
+
+ if (info->primary == 0 || ((info->primary == -1) && !gvt.primary)) {
+ cfg_space[GVT_REG_CFG_CLASS_CODE] = GVT_PCI_CLASS_VGA;
+ cfg_space[GVT_REG_CFG_SUB_CLASS_CODE] = GVT_PCI_CLASS_VGA_OTHER;
+ cfg_space[GVT_REG_CFG_CLASS_PROG_IF] = GVT_PCI_CLASS_VGA_OTHER;
+ }
+
+ /* Show guest that there isn't any stolen memory.*/
+ gmch_ctl = (u16 *)(cfg_space + _REG_GMCH_CONTROL);
+ *gmch_ctl &= ~(_REGBIT_BDW_GMCH_GMS_MASK << _REGBIT_BDW_GMCH_GMS_SHIFT);
+
+ gvt_pci_bar_write_32(vgt, GVT_REG_CFG_SPACE_BAR1, phys_aperture_base(pdev));
+
+ cfg_space[GVT_REG_CFG_COMMAND] &= ~(_REGBIT_CFG_COMMAND_IO |
+ _REGBIT_CFG_COMMAND_MEMORY |
+ _REGBIT_CFG_COMMAND_MASTER);
+
+ /* Clear the bar upper 32bit and let hvmloader to assign the new value */
+ memset(&cfg_space[GVT_REG_CFG_SPACE_BAR0 + 4], 0, 4);
+ memset(&cfg_space[GVT_REG_CFG_SPACE_BAR1 + 4], 0, 4);
+
+ state->cfg.bar_size[0] = pdev->bar_size[0]; /* MMIOGTT */
+ state->cfg.bar_size[1] = pdev->bar_size[1];
+ state->cfg.bar_size[2] = pdev->bar_size[2]; /* PIO */
+ state->cfg.bar_size[3] = pdev->bar_size[3]; /* ROM */
+
+ for (i = 0; i < GVT_BAR_NUM; i++)
+ state->cfg.bar_mapped[i] = false;
+}
+
+static void destroy_virtual_gm_state(struct vgt_device *vgt)
+{
+ gvt_free_gm_and_fence_resource(vgt);
+}
+
+static void populate_pvinfo_page(struct vgt_device *vgt)
+{
+ /* setup the ballooning information */
+ __vreg64(vgt, _vgtif_reg(magic)) = VGT_MAGIC;
+ __vreg(vgt, _vgtif_reg(version_major)) = 1;
+ __vreg(vgt, _vgtif_reg(version_minor)) = 0;
+ __vreg(vgt, _vgtif_reg(display_ready)) = 0;
+ __vreg(vgt, _vgtif_reg(vgt_id)) = vgt->id;
+ __vreg(vgt, _vgtif_reg(avail_rs.mappable_gmadr.base)) = gvt_visible_gm_base(vgt);
+ __vreg(vgt, _vgtif_reg(avail_rs.mappable_gmadr.size)) = gvt_aperture_sz(vgt);
+ __vreg(vgt, _vgtif_reg(avail_rs.nonmappable_gmadr.base)) = gvt_hidden_gm_base(vgt);
+ __vreg(vgt, _vgtif_reg(avail_rs.nonmappable_gmadr.size)) = gvt_hidden_gm_sz(vgt);
+
+ __vreg(vgt, _vgtif_reg(avail_rs.fence_num)) = gvt_fence_sz(vgt);
+ gvt_info("filling VGT_PVINFO_PAGE for dom%d:"
+ " visable_gm_base=0x%llx, size=0x%llx"
+ " hidden_gm_base=0x%llx, size=0x%llx"
+ " fence_base=%d, num=%d",
+ vgt->id,
+ gvt_visible_gm_base(vgt), gvt_aperture_sz(vgt),
+ gvt_hidden_gm_base(vgt), gvt_hidden_gm_sz(vgt),
+ gvt_fence_base(vgt), gvt_fence_sz(vgt));
+
+ ASSERT(sizeof(struct vgt_if) == VGT_PVINFO_SIZE);
+}
+
+static bool create_virtual_gm_state(struct vgt_device *vgt,
+ struct gvt_instance_info *info)
+{
+ struct pgt_device *pdev = vgt->pdev;
+ struct gvt_virtual_device_state *state = &vgt->state;
+
+ if (gvt_alloc_gm_and_fence_resource(vgt, info) < 0) {
+ gvt_err("fail to allocate graphics memory and fence");
+ return false;
+ }
+
+ state->gm.aperture_offset = aperture_2_gm(pdev, state->gm.aperture_base);
+ state->gm.aperture_base_va = phys_aperture_vbase(pdev) + state->gm.aperture_offset;
+
+ populate_pvinfo_page(vgt);
+
+ return true;
+}
+
+static void destroy_virtual_device_state(struct vgt_device *vgt)
+{
+ destroy_virtual_mmio_state(vgt);
+ destroy_virtual_gm_state(vgt);
+}
+
+static bool create_virtual_device_state(struct vgt_device *vgt,
+ struct gvt_instance_info *info)
+{
+ if (!create_virtual_mmio_state(vgt))
+ return false;
+
+ if (!create_virtual_gm_state(vgt, info))
+ return false;
+
+ init_virtual_cfg_space_state(vgt, info);
+
+ return true;
+}
+
+void gvt_destroy_instance(struct vgt_device *vgt)
+{
+ struct pgt_device *pdev = vgt->pdev;
+
+ mutex_lock(&pdev->lock);
+ gvt_set_instance_offline(vgt);
+ if (vgt->id != -1)
+ idr_remove(&pdev->instance_idr, vgt->id);
+ mutex_unlock(&pdev->lock);
+
+ hypervisor_hvm_exit(vgt);
+ destroy_virtual_device_state(vgt);
+ vfree(vgt);
+}
+
+struct vgt_device *gvt_create_instance(struct pgt_device *pdev,
+ struct gvt_instance_info *info)
+{
+ struct vgt_device *vgt = NULL;
+ int id;
+
+ gvt_info("vm_id=%d, low_gm_sz=%dMB, high_gm_sz=%dMB, fence_sz=%d",
+ info->domid, info->low_gm_sz, info->high_gm_sz, info->fence_sz);
+
+ vgt = vzalloc(sizeof(*vgt));
+ if (vgt == NULL) {
+ gvt_err("fail to allocate memory for instance.");
+ return NULL;
+ }
+
+ mutex_lock(&pdev->lock);
+
+ gvt_set_instance_offline(vgt);
+ id = idr_alloc(&pdev->instance_idr, vgt, 1, GVT_MAX_VGPU - 1, GFP_KERNEL);
+ if (id < 0) {
+ gvt_err("fail to allocate id for vgt instance.");
+ goto err;
+ }
+
+ mutex_unlock(&pdev->lock);
+
+ vgt->vm_id = info->domid;
+ vgt->id = id;
+ vgt->pdev = pdev;
+
+ if (!create_virtual_device_state(vgt, info))
+ goto err;
+
+ if (hypervisor_hvm_init(vgt) < 0)
+ goto err;
+
+ gvt_set_instance_online(vgt);
+
+ return vgt;
+err:
+ mutex_unlock(&pdev->lock);
+ gvt_destroy_instance(vgt);
+ return NULL;
+}
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 0fbabd2..28e1393 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -288,3 +288,35 @@ err:
gvt_clean_mmio_emulation_state(pdev);
return false;
}
+
+void gvt_init_virtual_mmio_register(struct vgt_device *vgt)
+{
+ struct pgt_device *pdev = vgt->pdev;
+ int i;
+
+ for (i = 0; i < pdev->mmio_size; i += sizeof(u32)) {
+ /*
+ * skip the area of VGT PV INFO PAGE because we need keep
+ * its content across Dom0 S3.
+ */
+ if (i >= VGT_PVINFO_PAGE &&
+ i < VGT_PVINFO_PAGE + VGT_PVINFO_SIZE)
+ continue;
+
+ __vreg(vgt, i) = pdev->initial_mmio_state[REG_INDEX(i)];
+ }
+
+ /* set the bit 0:2 (Thread C-State) to C0
+ * TODO: consider other bit 3:31
+ */
+ __vreg(vgt, _GEN6_GT_THREAD_STATUS_REG) = 0;
+
+ /* set the bit 0:2(Core C-State ) to C0 */
+ __vreg(vgt, _GEN6_GT_CORE_STATUS) = 0;
+}
+
+void gvt_init_shadow_mmio_register(struct vgt_device *vgt)
+{
+ struct gvt_virtual_device_state *state = &vgt->state;
+ memcpy (state->mmio.sreg, vgt->pdev->initial_mmio_state, vgt->pdev->mmio_size);
+}
diff --git a/drivers/gpu/drm/i915/gvt/params.c b/drivers/gpu/drm/i915/gvt/params.c
index 6cd324c..fca49b0 100644
--- a/drivers/gpu/drm/i915/gvt/params.c
+++ b/drivers/gpu/drm/i915/gvt/params.c
@@ -25,6 +25,7 @@
struct gvt_kernel_params gvt = {
.enable = true,
+ .primary = true,
.debug = 0,
.dom0_low_gm_sz = 96,
.dom0_high_gm_sz = 384,
diff --git a/drivers/gpu/drm/i915/gvt/params.h b/drivers/gpu/drm/i915/gvt/params.h
index 0507870..80255c3 100644
--- a/drivers/gpu/drm/i915/gvt/params.h
+++ b/drivers/gpu/drm/i915/gvt/params.h
@@ -26,6 +26,7 @@
struct gvt_kernel_params {
bool enable;
+ bool primary;
int debug;
int dom0_low_gm_sz;
int dom0_high_gm_sz;
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
index 5682e1c..2edaf7c 100644
--- a/drivers/gpu/drm/i915/gvt/reg.h
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -27,12 +27,35 @@
#define GVT_CFG_SPACE_SZ 256
#define GVT_BAR_NUM 4
-#define GVT_REG_CFG_SPACE_BAR0 0x10
-#define GVT_REG_CFG_SPACE_BAR1 0x18
-#define GVT_REG_CFG_SPACE_BAR2 0x20
+#define GVT_PCI_CLASS_VGA 0x03
+#define GVT_PCI_CLASS_VGA_OTHER 0x80
+
+#define GVT_REG_CFG_VENDOR_ID 0x00
+#define GVT_REG_CFG_COMMAND 0x04
+#define _REGBIT_CFG_COMMAND_IO (1 << 0)
+#define _REGBIT_CFG_COMMAND_MEMORY (1 << 1)
+#define _REGBIT_CFG_COMMAND_MASTER (1 << 2)
+#define GVT_REG_CFG_CLASS_PROG_IF 0x09
+#define GVT_REG_CFG_SUB_CLASS_CODE 0x0A
+#define GVT_REG_CFG_CLASS_CODE 0x0B
+#define GVT_REG_CFG_SPACE_BAR0 0x10
+#define GVT_REG_CFG_SPACE_BAR1 0x18
+#define GVT_REG_CFG_SPACE_BAR2 0x20
+#define GVT_REG_CFG_SPACE_BAR_ROM 0x30
+#define GVT_REG_CFG_SPACE_MSAC 0x62
+#define GVT_REG_CFG_SWSCI_TRIGGER 0xE8
+#define _REGBIT_CFG_SWSCI_SCI_SELECT (1 << 15)
+#define _REGBIT_CFG_SWSCI_SCI_TRIGGER 1
+#define GVT_REG_CFG_OPREGION 0xFC
+
+#define _REG_GMCH_CONTROL 0x50
+#define _REGBIT_BDW_GMCH_GMS_SHIFT 8
+#define _REGBIT_BDW_GMCH_GMS_MASK 0xff
#define _PCH_GMBUS2 0xc5108
#define _GEN6_GDRST 0x941c
+#define _GEN6_GT_THREAD_STATUS_REG 0x13805c
+#define _GEN6_GT_CORE_STATUS 0x138060
#endif
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index 21c77a2..b6d9fc7 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -163,8 +163,9 @@ struct vgt_if {
uint32_t rsv6[0x200-25]; /* pad to one page */
} __packed;
-#define vgtif_reg(x) \
- _MMIO((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x))
+#define _vgtif_reg(x) \
+ ((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x))
+#define vgtif_reg(x) _MMIO(_vgtif_reg(x))
extern void i915_check_vgpu(struct drm_device *dev);
extern int intel_vgt_balloon(struct drm_device *dev);
--
1.9.1
More information about the Intel-gfx
mailing list