[PATCH 2/2] drm/amdgpu: keep per VM BOs together on the LRU

[Christian König]

When one VM BOs validated move all of them them to the end of the LRU to
keep them together.

Signed-off-by: Christian König <christian.koenig at amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 82 +++++++++++++++++++++++++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h |  3 +-
 3 files changed, 78 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index d7d7ce1507ec..f8f2471d8606 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -638,7 +638,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 				       struct amdgpu_bo_list_entry,
 				       tv.head);
 
-	r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
+	r = amdgpu_vm_validate_pt_bos(p->adev, p->filp, &fpriv->vm,
 				      amdgpu_cs_validate, p);
 	if (r) {
 		DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index a2f8b57838ff..f81d19dae56f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -180,21 +180,80 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 	list_add(&entry->tv.head, validated);
 }
 
+/**
+ * amdgpu_vm_move_bos - move per VM BOs to the end of the LRU
+ *
+ * @id: unused
+ * @ptr: ptr to the GEM object
+ * @data: ptr to the VM
+ *
+ * Make sure that all per VM BOs are grouped together on their LRU.
+ */
+static int amdgpu_vm_move_bos(int id, void *ptr, void *data)
+{
+	struct amdgpu_bo *bo = gem_to_amdgpu_bo(ptr);
+	struct amdgpu_vm *vm = data;
+
+	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
+		return 0;
+
+	ttm_bo_move_to_lru_tail(&bo->tbo);
+	if (bo->shadow)
+		ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+
+	return 0;
+}
+
+/**
+ * amdgpu_vm_move_levels - move VM PDs/PTs to the end of the LRU
+ *
+ * @adev: amdgpu device structure
+ * @parent: the parent level with page tables
+ * @level: the current level
+ *
+ * Make sure that all per VM PDs/PTs are grouped together on their LRU.
+ */
+static void amdgpu_vm_move_levels(struct amdgpu_device *adev,
+				  struct amdgpu_vm_pt *parent,
+				  unsigned level)
+{
+	struct amdgpu_bo *bo = parent->base.bo;
+	unsigned i, num_entries;
+
+	if (bo) {
+		if (bo->parent)
+			ttm_bo_move_to_lru_tail(&bo->tbo);
+		if (bo->shadow)
+			ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+	}
+
+	if (!parent->entries)
+		return;
+
+	num_entries = amdgpu_vm_num_entries(adev, level);
+	for (i = 0; i < num_entries; i++)
+		amdgpu_vm_move_levels(adev, &parent->entries[i],
+				      level + 1);
+}
+
 /**
  * amdgpu_vm_validate_pt_bos - validate the page table BOs
  *
  * @adev: amdgpu device pointer
+ * @filp: drm file pointer
  * @vm: vm providing the BOs
  * @validate: callback to do the validation
  * @param: parameter for the validation callback
  *
  * Validate the page table BOs on command submission if neccessary.
  */
-int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct drm_file *filp,
+			      struct amdgpu_vm *vm,
 			      int (*validate)(void *p, struct amdgpu_bo *bo),
 			      void *param)
 {
 	struct ttm_bo_global *glob = adev->mman.bdev.glob;
+	bool validated = false;
 	int r;
 
 	spin_lock(&vm->status_lock);
@@ -213,12 +272,6 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			r = validate(param, bo);
 			if (r)
 				return r;
-
-			spin_lock(&glob->lru_lock);
-			ttm_bo_move_to_lru_tail(&bo->tbo);
-			if (bo->shadow)
-				ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
-			spin_unlock(&glob->lru_lock);
 		}
 
 		if (bo->tbo.type == ttm_bo_type_kernel &&
@@ -228,6 +281,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 				return r;
 		}
 
+		validated = true;
 		spin_lock(&vm->status_lock);
 		if (bo->tbo.type != ttm_bo_type_kernel)
 			list_move(&bo_base->vm_status, &vm->moved);
@@ -236,6 +290,20 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	}
 	spin_unlock(&vm->status_lock);
 
+	if (!validated)
+		return 0;
+
+	spin_lock(&glob->lru_lock);
+
+	spin_lock(&filp->table_lock);
+	idr_for_each(&filp->object_idr, amdgpu_vm_move_bos, vm);
+	spin_unlock(&filp->table_lock);
+
+	amdgpu_vm_move_levels(adev, &vm->root,
+			      adev->vm_manager.root_level);
+
+	spin_unlock(&glob->lru_lock);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index fabf44b262be..a43a493f5c08 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -257,7 +257,8 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 			 struct list_head *validated,
 			 struct amdgpu_bo_list_entry *entry);
 bool amdgpu_vm_ready(struct amdgpu_vm *vm);
-int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct drm_file *filp,
+			      struct amdgpu_vm *vm,
 			      int (*callback)(void *p, struct amdgpu_bo *bo),
 			      void *param);
 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
-- 
2.14.1