[PATCH 1/4] drm/radeon: invalidate moved BOs in the VM

[Christian König]

From: Christian König <christian.koenig at amd.com>

Don't wait for the BO to be used again, just
update the PT on the next VM use.

Signed-off-by: Christian König <christian.koenig at amd.com>
---
 drivers/gpu/drm/radeon/radeon.h    |  7 +++++-
 drivers/gpu/drm/radeon/radeon_cs.c |  3 ++-
 drivers/gpu/drm/radeon/radeon_vm.c | 51 ++++++++++++++++++++++++++++++--------
 3 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 5c9364b..0ec7864 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -447,7 +447,7 @@ struct radeon_bo_va {
 	uint64_t			soffset;
 	uint64_t			eoffset;
 	uint32_t			flags;
-	bool				valid;
+	uint64_t			addr;
 	unsigned			ref_count;
 
 	/* protected by vm mutex */
@@ -871,6 +871,9 @@ struct radeon_vm {
 	struct list_head		va;
 	unsigned			id;
 
+	/* BOs moved, but not yet updated in the PT */
+	struct list_head		invalidated;
+
 	/* BOs freed, but not yet updated in the PT */
 	struct list_head		freed;
 
@@ -2875,6 +2878,8 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev,
 				    struct radeon_vm *vm);
 int radeon_vm_clear_freed(struct radeon_device *rdev,
 			  struct radeon_vm *vm);
+int radeon_vm_clear_invalids(struct radeon_device *rdev,
+			     struct radeon_vm *vm);
 int radeon_vm_bo_update(struct radeon_device *rdev,
 			struct radeon_bo_va *bo_va,
 			struct ttm_mem_reg *mem);
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 09fcf4d..e372c4142 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -500,7 +500,8 @@ static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p,
 		if (r)
 			return r;
 	}
-	return 0;
+
+	return radeon_vm_clear_invalids(rdev, vm);
 }
 
 static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c
index 2726b46..f1030fc 100644
--- a/drivers/gpu/drm/radeon/radeon_vm.c
+++ b/drivers/gpu/drm/radeon/radeon_vm.c
@@ -328,7 +328,7 @@ struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
 	bo_va->soffset = 0;
 	bo_va->eoffset = 0;
 	bo_va->flags = 0;
-	bo_va->valid = false;
+	bo_va->addr = 0;
 	bo_va->ref_count = 1;
 	INIT_LIST_HEAD(&bo_va->bo_list);
 	INIT_LIST_HEAD(&bo_va->vm_list);
@@ -481,7 +481,7 @@ int radeon_vm_bo_set_addr(struct radeon_device *rdev,
 	bo_va->soffset = soffset;
 	bo_va->eoffset = eoffset;
 	bo_va->flags = flags;
-	bo_va->valid = false;
+	bo_va->addr = 0;
 	list_move(&bo_va->vm_list, head);
 
 	soffset = (soffset / RADEON_GPU_PAGE_SIZE) >> radeon_vm_block_size;
@@ -842,15 +842,13 @@ int radeon_vm_bo_update(struct radeon_device *rdev,
 	uint64_t addr;
 	int r;
 
-
 	if (!bo_va->soffset) {
 		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
 			bo_va->bo, vm);
 		return -EINVAL;
 	}
 
-	if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
-		return 0;
+	list_del_init(&bo_va->vm_status);
 
 	bo_va->flags &= ~RADEON_VM_PAGE_VALID;
 	bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
@@ -858,7 +856,6 @@ int radeon_vm_bo_update(struct radeon_device *rdev,
 		addr = mem->start << PAGE_SHIFT;
 		if (mem->mem_type != TTM_PL_SYSTEM) {
 			bo_va->flags |= RADEON_VM_PAGE_VALID;
-			bo_va->valid = true;
 		}
 		if (mem->mem_type == TTM_PL_TT) {
 			bo_va->flags |= RADEON_VM_PAGE_SYSTEM;
@@ -867,9 +864,12 @@ int radeon_vm_bo_update(struct radeon_device *rdev,
 		}
 	} else {
 		addr = 0;
-		bo_va->valid = false;
 	}
 
+	if (addr == bo_va->addr)
+		return 0;
+	bo_va->addr = addr;
+
 	trace_radeon_vm_bo_update(bo_va);
 
 	nptes = (bo_va->eoffset - bo_va->soffset) / RADEON_GPU_PAGE_SIZE;
@@ -932,7 +932,6 @@ int radeon_vm_clear_freed(struct radeon_device *rdev,
 	int r;
 
 	list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) {
-		list_del(&bo_va->vm_status);
 		r = radeon_vm_bo_update(rdev, bo_va, NULL);
 		kfree(bo_va);
 		if (r)
@@ -943,6 +942,31 @@ int radeon_vm_clear_freed(struct radeon_device *rdev,
 }
 
 /**
+ * radeon_vm_clear_invalids - clear invalidated BOs in the PT
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ *
+ * Make sure all invalidated BOs are cleared in the PT.
+ * Returns 0 for success.
+ *
+ * PTs have to be reserved and mutex must be locked!
+ */
+int radeon_vm_clear_invalids(struct radeon_device *rdev,
+			     struct radeon_vm *vm)
+{
+	struct radeon_bo_va *bo_va, *tmp;
+	int r;
+
+	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, vm_status) {
+		r = radeon_vm_bo_update(rdev, bo_va, NULL);
+		if (r)
+			return r;
+	}
+	return 0;
+}
+
+/**
  * radeon_vm_bo_rmv - remove a bo to a specific vm
  *
  * @rdev: radeon_device pointer
@@ -961,8 +985,9 @@ void radeon_vm_bo_rmv(struct radeon_device *rdev,
 
 	mutex_lock(&vm->mutex);
 	list_del(&bo_va->vm_list);
+	list_del(&bo_va->vm_status);
 
-	if (bo_va->soffset) {
+	if (bo_va->addr) {
 		bo_va->bo = NULL;
 		list_add(&bo_va->vm_status, &vm->freed);
 	} else {
@@ -987,7 +1012,12 @@ void radeon_vm_bo_invalidate(struct radeon_device *rdev,
 	struct radeon_bo_va *bo_va;
 
 	list_for_each_entry(bo_va, &bo->va, bo_list) {
-		bo_va->valid = false;
+		if (bo_va->addr) {
+			mutex_lock(&bo_va->vm->mutex);
+			list_del(&bo_va->vm_status);
+			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
+			mutex_unlock(&bo_va->vm->mutex);
+		}
 	}
 }
 
@@ -1012,6 +1042,7 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
 	vm->last_id_use = NULL;
 	mutex_init(&vm->mutex);
 	INIT_LIST_HEAD(&vm->va);
+	INIT_LIST_HEAD(&vm->invalidated);;
 	INIT_LIST_HEAD(&vm->freed);
 
 	pd_size = radeon_vm_directory_size(rdev);
-- 
1.9.1