[PATCH 18/35] drm/amdkfd: validate vram svm range from TTM

Felix Kuehling Felix.Kuehling at amd.com
Thu Jan 7 03:01:10 UTC 2021


From: Philip Yang <Philip.Yang at amd.com>

If svm range perfetch location is not zero, use TTM to alloc
amdgpu_bo vram nodes to validate svm range, then map vram nodes to GPUs.

Use offset to sub allocate from the same amdgpu_bo to handle overlap
vram range while adding new range or unmapping range.

svm_bo has ref count to trace the shared ranges. If all ranges of shared
amdgpu_bo are migrated to ram, ref count becomes 0, then amdgpu_bo is
released, all ranges svm_bo is set to NULL.

To migrate range from ram back to vram, allocate the same amdgpu_bo
with previous offset if the range has svm_bo.

Signed-off-by: Philip Yang <Philip.Yang at amd.com>
Signed-off-by: Alex Sierra <alex.sierra at amd.com>
Signed-off-by: Felix Kuehling <Felix.Kuehling at amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 342 ++++++++++++++++++++++++---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h |  20 ++
 2 files changed, 335 insertions(+), 27 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index e3ba6e7262a7..7d91dc49a5a9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -35,7 +35,9 @@
  * svm_range_unlink - unlink svm_range from lists and interval tree
  * @prange: svm range structure to be removed
  *
- * Remove the svm range from svms interval tree and link list
+ * Remove the svm_range from the svms and svm_bo SRCU lists and the svms
+ * interval tree. After this call, synchronize_srcu is needed before the
+ * range can be freed safely.
  *
  * Context: The caller must hold svms_lock
  */
@@ -44,6 +46,12 @@ static void svm_range_unlink(struct svm_range *prange)
 	pr_debug("prange 0x%p [0x%lx 0x%lx]\n", prange, prange->it_node.start,
 		 prange->it_node.last);
 
+	if (prange->svm_bo) {
+		spin_lock(&prange->svm_bo->list_lock);
+		list_del(&prange->svm_bo_list);
+		spin_unlock(&prange->svm_bo->list_lock);
+	}
+
 	list_del_rcu(&prange->list);
 	interval_tree_remove(&prange->it_node, &prange->svms->objects);
 }
@@ -70,6 +78,12 @@ static void svm_range_remove(struct svm_range *prange)
 	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
 		 prange->it_node.start, prange->it_node.last);
 
+	if (prange->mm_nodes) {
+		pr_debug("vram prange svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
+			 prange->it_node.start, prange->it_node.last);
+		svm_range_vram_node_free(prange);
+	}
+
 	kvfree(prange->pages_addr);
 	kfree(prange);
 }
@@ -102,7 +116,9 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
 	INIT_LIST_HEAD(&prange->list);
 	INIT_LIST_HEAD(&prange->update_list);
 	INIT_LIST_HEAD(&prange->remove_list);
+	INIT_LIST_HEAD(&prange->svm_bo_list);
 	atomic_set(&prange->invalid, 0);
+	spin_lock_init(&prange->svm_bo_lock);
 	svm_range_set_default_attributes(&prange->preferred_loc,
 					 &prange->prefetch_loc,
 					 &prange->granularity, &prange->flags);
@@ -139,6 +155,16 @@ svm_get_supported_dev_by_id(struct kfd_process *p, uint32_t gpu_id,
 	return dev;
 }
 
+struct amdgpu_device *
+svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id)
+{
+	struct kfd_process *p =
+			container_of(prange->svms, struct kfd_process, svms);
+	struct kfd_dev *dev = svm_get_supported_dev_by_id(p, gpu_id, NULL);
+
+	return dev ? (struct amdgpu_device *)dev->kgd : NULL;
+}
+
 /**
  * svm_range_validate_ram - get system memory pages of svm range
  *
@@ -186,14 +212,226 @@ svm_range_validate_ram(struct mm_struct *mm, struct svm_range *prange)
 	return 0;
 }
 
+static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)
+{
+	if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref))
+		return false;
+
+	return true;
+}
+
+static struct svm_range_bo *svm_range_bo_ref(struct svm_range_bo *svm_bo)
+{
+	if (svm_bo)
+		kref_get(&svm_bo->kref);
+
+	return svm_bo;
+}
+
+static void svm_range_bo_release(struct kref *kref)
+{
+	struct svm_range_bo *svm_bo;
+
+	svm_bo = container_of(kref, struct svm_range_bo, kref);
+	/* This cleanup loop does not need to be SRCU safe because there
+	 * should be no SRCU readers while the ref count is 0. Any SRCU
+	 * reader that has a chance of reducing the ref count must take
+	 * an extra reference before srcu_read_lock and release it after
+	 * srcu_read_unlock.
+	 */
+	spin_lock(&svm_bo->list_lock);
+	while (!list_empty(&svm_bo->range_list)) {
+		struct svm_range *prange =
+				list_first_entry(&svm_bo->range_list,
+						struct svm_range, svm_bo_list);
+		pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
+			 prange->it_node.start, prange->it_node.last);
+		spin_lock(&prange->svm_bo_lock);
+		prange->svm_bo = NULL;
+		spin_unlock(&prange->svm_bo_lock);
+
+		/* list_del_init tells a concurrent svm_range_vram_node_new when
+		 * it's safe to reuse the svm_bo pointer and svm_bo_list head.
+		 */
+		list_del_init(&prange->svm_bo_list);
+	}
+	spin_unlock(&svm_bo->list_lock);
+
+	amdgpu_bo_unref(&svm_bo->bo);
+	kfree(svm_bo);
+}
+
+static void svm_range_bo_unref(struct svm_range_bo *svm_bo)
+{
+	if (!svm_bo)
+		return;
+
+	kref_put(&svm_bo->kref, svm_range_bo_release);
+}
+
+static struct svm_range_bo *svm_range_bo_new(void)
+{
+	struct svm_range_bo *svm_bo;
+
+	svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL);
+	if (!svm_bo)
+		return NULL;
+
+	kref_init(&svm_bo->kref);
+	INIT_LIST_HEAD(&svm_bo->range_list);
+	spin_lock_init(&svm_bo->list_lock);
+
+	return svm_bo;
+}
+
+int
+svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange,
+			bool clear)
+{
+	struct amdkfd_process_info *process_info;
+	struct amdgpu_bo_param bp;
+	struct svm_range_bo *svm_bo;
+	struct amdgpu_bo *bo;
+	struct kfd_process *p;
+	int r;
+
+	pr_debug("[0x%lx 0x%lx]\n", prange->it_node.start,
+		 prange->it_node.last);
+	spin_lock(&prange->svm_bo_lock);
+	if (prange->svm_bo) {
+		if (prange->mm_nodes) {
+			/* We still have a reference, all is well */
+			spin_unlock(&prange->svm_bo_lock);
+			return 0;
+		}
+		if (svm_bo_ref_unless_zero(prange->svm_bo)) {
+			/* The BO was still around and we got
+			 * a new reference to it
+			 */
+			spin_unlock(&prange->svm_bo_lock);
+			pr_debug("reuse old bo [0x%lx 0x%lx]\n",
+				prange->it_node.start, prange->it_node.last);
+
+			prange->mm_nodes = prange->svm_bo->bo->tbo.mem.mm_node;
+			return 0;
+		}
+
+		spin_unlock(&prange->svm_bo_lock);
+
+		/* We need a new svm_bo. Spin-loop to wait for concurrent
+		 * svm_range_bo_release to finish removing this range from
+		 * its range list. After this, it is safe to reuse the
+		 * svm_bo pointer and svm_bo_list head.
+		 */
+		while (!list_empty_careful(&prange->svm_bo_list))
+			;
+
+	} else {
+		spin_unlock(&prange->svm_bo_lock);
+	}
+
+	svm_bo = svm_range_bo_new();
+	if (!svm_bo) {
+		pr_debug("failed to alloc svm bo\n");
+		return -ENOMEM;
+	}
+
+	memset(&bp, 0, sizeof(bp));
+	bp.size = prange->npages * PAGE_SIZE;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+	bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+	bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
+	bp.type = ttm_bo_type_device;
+	bp.resv = NULL;
+
+	r = amdgpu_bo_create(adev, &bp, &bo);
+	if (r) {
+		pr_debug("failed %d to create bo\n", r);
+		kfree(svm_bo);
+		return r;
+	}
+
+	p = container_of(prange->svms, struct kfd_process, svms);
+	r = amdgpu_bo_reserve(bo, true);
+	if (r) {
+		pr_debug("failed %d to reserve bo\n", r);
+		goto reserve_bo_failed;
+	}
+
+	r = dma_resv_reserve_shared(bo->tbo.base.resv, 1);
+	if (r) {
+		pr_debug("failed %d to reserve bo\n", r);
+		amdgpu_bo_unreserve(bo);
+		goto reserve_bo_failed;
+	}
+	process_info = p->kgd_process_info;
+	amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
+
+	amdgpu_bo_unreserve(bo);
+
+	svm_bo->bo = bo;
+	prange->svm_bo = svm_bo;
+	prange->mm_nodes = bo->tbo.mem.mm_node;
+	prange->offset = 0;
+
+	spin_lock(&svm_bo->list_lock);
+	list_add(&prange->svm_bo_list, &svm_bo->range_list);
+	spin_unlock(&svm_bo->list_lock);
+
+	return 0;
+
+reserve_bo_failed:
+	kfree(svm_bo);
+	amdgpu_bo_unref(&bo);
+	prange->mm_nodes = NULL;
+
+	return r;
+}
+
+void svm_range_vram_node_free(struct svm_range *prange)
+{
+	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
+		 prange->it_node.start, prange->it_node.last);
+
+	svm_range_bo_unref(prange->svm_bo);
+	prange->mm_nodes = NULL;
+}
+
+static int svm_range_validate_vram(struct svm_range *prange)
+{
+	struct amdgpu_device *adev;
+	int r;
+
+	pr_debug("svms 0x%p [0x%lx 0x%lx] actual_loc 0x%x\n", prange->svms,
+		 prange->it_node.start, prange->it_node.last,
+		 prange->actual_loc);
+
+	adev = svm_range_get_adev_by_id(prange, prange->actual_loc);
+	if (!adev) {
+		pr_debug("failed to get device by id 0x%x\n",
+			 prange->actual_loc);
+		return -EINVAL;
+	}
+
+	r = svm_range_vram_node_new(adev, prange, true);
+	if (r)
+		pr_debug("failed %d to alloc vram\n", r);
+
+	return r;
+}
+
 static int
 svm_range_validate(struct mm_struct *mm, struct svm_range *prange)
 {
-	int r = 0;
+	int r;
 
 	pr_debug("actual loc 0x%x\n", prange->actual_loc);
 
-	r = svm_range_validate_ram(mm, prange);
+	if (!prange->actual_loc)
+		r = svm_range_validate_ram(mm, prange);
+	else
+		r = svm_range_validate_vram(prange);
 
 	pr_debug("svms 0x%p [0x%lx 0x%lx] ret %d invalid %d\n", prange->svms,
 		 prange->it_node.start, prange->it_node.last,
@@ -349,6 +587,35 @@ svm_range_split_pages(struct svm_range *new, struct svm_range *old,
 	return 0;
 }
 
+static int
+svm_range_split_nodes(struct svm_range *new, struct svm_range *old,
+		      uint64_t start, uint64_t last)
+{
+	pr_debug("svms 0x%p new start 0x%lx start 0x%llx last 0x%llx\n",
+		 new->svms, new->it_node.start, start, last);
+
+	old->npages = last - start + 1;
+
+	if (new->it_node.start == old->it_node.start) {
+		new->offset = old->offset;
+		old->offset += new->npages;
+	} else {
+		new->offset = old->offset + old->npages;
+	}
+
+	old->it_node.start = start;
+	old->it_node.last = last;
+
+	new->svm_bo = svm_range_bo_ref(old->svm_bo);
+	new->mm_nodes = old->mm_nodes;
+
+	spin_lock(&new->svm_bo->list_lock);
+	list_add(&new->svm_bo_list, &new->svm_bo->range_list);
+	spin_unlock(&new->svm_bo->list_lock);
+
+	return 0;
+}
+
 /**
  * svm_range_split_adjust - split range and adjust
  *
@@ -382,6 +649,8 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old,
 
 	if (old->pages_addr)
 		r = svm_range_split_pages(new, old, start, last);
+	else if (old->actual_loc && old->mm_nodes)
+		r = svm_range_split_nodes(new, old, start, last);
 	else
 		WARN_ONCE(1, "split adjust invalid pages_addr and nodes\n");
 	if (r)
@@ -438,17 +707,14 @@ svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,
 		return -EINVAL;
 
 	svms = prange->svms;
-	if (old_start == start) {
+	if (old_start == start)
 		*new = svm_range_new(svms, last + 1, old_last);
-		if (!*new)
-			return -ENOMEM;
-		r = svm_range_split_adjust(*new, prange, start, last);
-	} else {
+	else
 		*new = svm_range_new(svms, old_start, start - 1);
-		if (!*new)
-			return -ENOMEM;
-		r = svm_range_split_adjust(*new, prange, start, last);
-	}
+	if (!*new)
+		return -ENOMEM;
+
+	r = svm_range_split_adjust(*new, prange, start, last);
 
 	return r;
 }
@@ -550,7 +816,8 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange)
 	uint64_t pte_flags;
 
 	pte_flags = AMDGPU_PTE_VALID;
-	pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED;
+	if (!prange->mm_nodes)
+		pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED;
 
 	mapping_flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;
 
@@ -570,7 +837,9 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange)
 	/* Apply ASIC specific mapping flags */
 	amdgpu_gmc_get_vm_pte(adev, &prange->mapping, &pte_flags);
 
-	pr_debug("PTE flags 0x%llx\n", pte_flags);
+	pr_debug("svms 0x%p [0x%lx 0x%lx] vram %d system %d PTE flags 0x%llx\n",
+		 prange->svms, prange->it_node.start, prange->it_node.last,
+		 prange->mm_nodes ? 1:0, prange->pages_addr ? 1:0, pte_flags);
 
 	return pte_flags;
 }
@@ -656,7 +925,9 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct svm_range *prange, bool reserve_vm,
 		     struct dma_fence **fence)
 {
-	struct amdgpu_bo *root;
+	struct ttm_validate_buffer tv[2];
+	struct ww_acquire_ctx ticket;
+	struct list_head list;
 	dma_addr_t *pages_addr;
 	uint64_t pte_flags;
 	int r = 0;
@@ -665,13 +936,25 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		 prange->it_node.start, prange->it_node.last);
 
 	if (reserve_vm) {
-		root = amdgpu_bo_ref(vm->root.base.bo);
-		r = amdgpu_bo_reserve(root, true);
+		INIT_LIST_HEAD(&list);
+
+		tv[0].bo = &vm->root.base.bo->tbo;
+		tv[0].num_shared = 4;
+		list_add(&tv[0].head, &list);
+		if (prange->svm_bo && prange->mm_nodes) {
+			tv[1].bo = &prange->svm_bo->bo->tbo;
+			tv[1].num_shared = 1;
+			list_add(&tv[1].head, &list);
+		}
+		r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
 		if (r) {
-			pr_debug("failed %d to reserve root bo\n", r);
-			amdgpu_bo_unref(&root);
+			pr_debug("failed %d to reserve bo\n", r);
 			goto out;
 		}
+		if (prange->svm_bo && prange->mm_nodes &&
+		    prange->svm_bo->bo->tbo.evicted)
+			goto unreserve_out;
+
 		r = amdgpu_vm_validate_pt_bos(adev, vm, svm_range_bo_validate,
 					      NULL);
 		if (r) {
@@ -682,7 +965,7 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	prange->mapping.start = prange->it_node.start;
 	prange->mapping.last = prange->it_node.last;
-	prange->mapping.offset = 0;
+	prange->mapping.offset = prange->offset;
 	pte_flags = svm_range_get_pte_flags(adev, prange);
 	prange->mapping.flags = pte_flags;
 	pages_addr = prange->pages_addr;
@@ -690,7 +973,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	r = amdgpu_vm_bo_update_mapping(adev, adev, vm, false, false, NULL,
 					prange->mapping.start,
 					prange->mapping.last, pte_flags,
-					prange->mapping.offset, NULL,
+					prange->mapping.offset,
+					prange->mm_nodes,
 					pages_addr, &vm->last_update);
 	if (r) {
 		pr_debug("failed %d to map to gpu 0x%lx\n", r,
@@ -710,11 +994,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		*fence = dma_fence_get(vm->last_update);
 
 unreserve_out:
-	if (reserve_vm) {
-		amdgpu_bo_unreserve(root);
-		amdgpu_bo_unref(&root);
-	}
-
+	if (reserve_vm)
+		ttm_eu_backoff_reservation(&ticket, &list);
 out:
 	return r;
 }
@@ -929,7 +1210,14 @@ struct svm_range *svm_range_clone(struct svm_range *old)
 		memcpy(new->pages_addr, old->pages_addr,
 		       old->npages * sizeof(*old->pages_addr));
 	}
-
+	if (old->svm_bo) {
+		new->mm_nodes = old->mm_nodes;
+		new->offset = old->offset;
+		new->svm_bo = svm_range_bo_ref(old->svm_bo);
+		spin_lock(&new->svm_bo->list_lock);
+		list_add(&new->svm_bo_list, &new->svm_bo->range_list);
+		spin_unlock(&new->svm_bo->list_lock);
+	}
 	new->flags = old->flags;
 	new->preferred_loc = old->preferred_loc;
 	new->prefetch_loc = old->prefetch_loc;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index 4c7daf8e0b6f..b1d2db02043b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -32,6 +32,12 @@
 #include "amdgpu.h"
 #include "kfd_priv.h"
 
+struct svm_range_bo {
+	struct amdgpu_bo	*bo;
+	struct kref		kref;
+	struct list_head	range_list; /* all svm ranges shared this bo */
+	spinlock_t		list_lock;
+};
 /**
  * struct svm_range - shared virtual memory range
  *
@@ -45,6 +51,10 @@
  * @mapping:    bo_va mapping structure to create and update GPU page table
  * @npages:     number of pages
  * @pages_addr: list of system memory physical page address
+ * @mm_nodes:   vram nodes allocated
+ * @offset:     range start offset within mm_nodes
+ * @svm_bo:     struct to manage splited amdgpu_bo
+ * @svm_bo_list:link list node, to scan all ranges which share same svm_bo
  * @flags:      flags defined as KFD_IOCTL_SVM_FLAG_*
  * @perferred_loc: perferred location, 0 for CPU, or GPU id
  * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id
@@ -68,6 +78,11 @@ struct svm_range {
 	struct amdgpu_bo_va_mapping	mapping;
 	uint64_t			npages;
 	dma_addr_t			*pages_addr;
+	struct drm_mm_node		*mm_nodes;
+	uint64_t			offset;
+	struct svm_range_bo		*svm_bo;
+	struct list_head		svm_bo_list;
+	spinlock_t                      svm_bo_lock;
 	uint32_t			flags;
 	uint32_t			preferred_loc;
 	uint32_t			prefetch_loc;
@@ -95,5 +110,10 @@ void svm_range_list_fini(struct kfd_process *p);
 int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
 	      uint64_t size, uint32_t nattrs,
 	      struct kfd_ioctl_svm_attribute *attrs);
+struct amdgpu_device *svm_range_get_adev_by_id(struct svm_range *prange,
+					       uint32_t id);
+int svm_range_vram_node_new(struct amdgpu_device *adev,
+			    struct svm_range *prange, bool clear);
+void svm_range_vram_node_free(struct svm_range *prange);
 
 #endif /* KFD_SVM_H_ */
-- 
2.29.2



More information about the amd-gfx mailing list