[Intel-gfx] [PATCH 2/4] drm/i915/gtt: Recursive scratch page

Chris Wilson chris at chris-wilson.co.uk
Tue Mar 5 13:54:28 UTC 2019


Fill the scratch page with pointers to itself so that we can reuse it at
any level of the page tables.

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld at intel.com>
Cc: Mika Kuoppala <mika.kuoppala at intel.com>
---
 drivers/gpu/drm/i915/i915_gem_gtt.c | 202 ++++++++++++----------------
 drivers/gpu/drm/i915/i915_gem_gtt.h |   3 -
 2 files changed, 85 insertions(+), 120 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 4a681b3332ad..7f2d8462f324 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -701,9 +701,7 @@ static void free_pd(struct i915_address_space *vm,
 static void gen8_initialize_pd(struct i915_address_space *vm,
 			       struct i915_page_directory *pd)
 {
-	fill_px(vm, pd,
-		gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
-	memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);
+	fill_px(vm, pd, vm->scratch_pte);
 }
 
 static int __pdp_init(struct i915_address_space *vm,
@@ -711,13 +709,11 @@ static int __pdp_init(struct i915_address_space *vm,
 {
 	const unsigned int pdpes = i915_pdpes_per_pdp(vm);
 
-	pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
-					    I915_GFP_ALLOW_FAIL);
+	pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
+				      I915_GFP_ALLOW_FAIL);
 	if (unlikely(!pdp->page_directory))
 		return -ENOMEM;
 
-	memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
-
 	return 0;
 }
 
@@ -777,19 +773,13 @@ static void free_pdp(struct i915_address_space *vm,
 static void gen8_initialize_pdp(struct i915_address_space *vm,
 				struct i915_page_directory_pointer *pdp)
 {
-	gen8_ppgtt_pdpe_t scratch_pdpe;
-
-	scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
-
-	fill_px(vm, pdp, scratch_pdpe);
+	fill_px(vm, pdp, vm->scratch_pte);
 }
 
 static void gen8_initialize_pml4(struct i915_address_space *vm,
 				 struct i915_pml4 *pml4)
 {
-	fill_px(vm, pml4,
-		gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
-	memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
+	fill_px(vm, pml4, vm->scratch_pte);
 }
 
 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
@@ -825,17 +815,16 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
 	return false;
 }
 
-static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
-			       struct i915_page_directory *pd,
-			       struct i915_page_table *pt,
-			       unsigned int pde)
+static void gen8_ppgtt_clear_pde(struct i915_address_space *vm,
+				 struct i915_page_directory *pd,
+				 unsigned int pde)
 {
 	gen8_pde_t *vaddr;
 
-	pd->page_table[pde] = pt;
+	pd->page_table[pde] = NULL;
 
 	vaddr = kmap_atomic_px(pd);
-	vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
+	vaddr[pde] = vm->scratch_pte;
 	kunmap_atomic(vaddr);
 }
 
@@ -847,37 +836,34 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 	u32 pde;
 
 	gen8_for_each_pde(pt, pd, start, length, pde) {
-		GEM_BUG_ON(pt == vm->scratch_pt);
-
 		if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
 			continue;
 
-		gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
+		gen8_ppgtt_clear_pde(vm, pd, pde);
 		GEM_BUG_ON(!pd->used_pdes);
 		pd->used_pdes--;
-
 		free_pt(vm, pt);
 	}
 
 	return !pd->used_pdes;
 }
 
-static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
-				struct i915_page_directory_pointer *pdp,
-				struct i915_page_directory *pd,
-				unsigned int pdpe)
+static void gen8_ppgtt_clear_pdpe(struct i915_address_space *vm,
+				  struct i915_page_directory_pointer *pdp,
+				  unsigned int pdpe)
 {
 	gen8_ppgtt_pdpe_t *vaddr;
 
-	pdp->page_directory[pdpe] = pd;
+	pdp->page_directory[pdpe] = NULL;
 	if (!use_4lvl(vm))
 		return;
 
 	vaddr = kmap_atomic_px(pdp);
-	vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
+	vaddr[pdpe] = vm->scratch_pte;
 	kunmap_atomic(vaddr);
 }
 
+
 /* Removes entries from a single page dir pointer, releasing it if it's empty.
  * Caller can use the return value to update higher-level entries
  */
@@ -889,15 +875,12 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 	unsigned int pdpe;
 
 	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
-		GEM_BUG_ON(pd == vm->scratch_pd);
-
 		if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
 			continue;
 
-		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+		gen8_ppgtt_clear_pdpe(vm, pdp, pdpe);
 		GEM_BUG_ON(!pdp->used_pdpes);
 		pdp->used_pdpes--;
-
 		free_pd(vm, pd);
 	}
 
@@ -910,16 +893,16 @@ static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
 	gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
 }
 
-static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
-				 struct i915_page_directory_pointer *pdp,
-				 unsigned int pml4e)
+static void gen8_ppgtt_clear_pm4le(struct i915_address_space *vm,
+				   struct i915_pml4 *pml4,
+				   unsigned int pml4e)
 {
 	gen8_ppgtt_pml4e_t *vaddr;
 
-	pml4->pdps[pml4e] = pdp;
+	pml4->pdps[pml4e] = NULL;
 
 	vaddr = kmap_atomic_px(pml4);
-	vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
+	vaddr[pml4e] = vm->scratch_pte;
 	kunmap_atomic(vaddr);
 }
 
@@ -938,13 +921,10 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
 	GEM_BUG_ON(!use_4lvl(vm));
 
 	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
-		GEM_BUG_ON(pdp == vm->scratch_pdp);
-
 		if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
 			continue;
 
-		gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
-
+		gen8_ppgtt_clear_pm4le(vm, pml4, pml4e);
 		free_pdp(vm, pdp);
 	}
 }
@@ -1196,8 +1176,10 @@ static void gen8_free_page_tables(struct i915_address_space *vm,
 	int i;
 
 	for (i = 0; i < I915_PDES; i++) {
-		if (pd->page_table[i] != vm->scratch_pt)
-			free_pt(vm, pd->page_table[i]);
+		if (!pd->page_table[i])
+			continue;
+
+		free_pt(vm, pd->page_table[i]);
 	}
 }
 
@@ -1219,9 +1201,6 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
 		vm->scratch_order = clone->scratch_order;
 		vm->scratch_pte = clone->scratch_pte;
-		vm->scratch_pt  = clone->scratch_pt;
-		vm->scratch_pd  = clone->scratch_pd;
-		vm->scratch_pdp = clone->scratch_pdp;
 		return 0;
 	}
 
@@ -1233,42 +1212,9 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 		gen8_pte_encode(vm->scratch_page.daddr,
 				I915_CACHE_LLC,
 				PTE_READ_ONLY);
-
-	vm->scratch_pt = alloc_pt(vm);
-	if (IS_ERR(vm->scratch_pt)) {
-		ret = PTR_ERR(vm->scratch_pt);
-		goto free_scratch_page;
-	}
-
-	vm->scratch_pd = alloc_pd(vm);
-	if (IS_ERR(vm->scratch_pd)) {
-		ret = PTR_ERR(vm->scratch_pd);
-		goto free_pt;
-	}
-
-	if (use_4lvl(vm)) {
-		vm->scratch_pdp = alloc_pdp(vm);
-		if (IS_ERR(vm->scratch_pdp)) {
-			ret = PTR_ERR(vm->scratch_pdp);
-			goto free_pd;
-		}
-	}
-
-	gen8_initialize_pt(vm, vm->scratch_pt);
-	gen8_initialize_pd(vm, vm->scratch_pd);
-	if (use_4lvl(vm))
-		gen8_initialize_pdp(vm, vm->scratch_pdp);
+	fill_page_dma((vm), &vm->scratch_page, vm->scratch_pte);
 
 	return 0;
-
-free_pd:
-	free_pd(vm, vm->scratch_pd);
-free_pt:
-	free_pt(vm, vm->scratch_pt);
-free_scratch_page:
-	cleanup_scratch_page(vm);
-
-	return ret;
 }
 
 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
@@ -1308,10 +1254,6 @@ static void gen8_free_scratch(struct i915_address_space *vm)
 	if (!vm->scratch_page.daddr)
 		return;
 
-	if (use_4lvl(vm))
-		free_pdp(vm, vm->scratch_pdp);
-	free_pd(vm, vm->scratch_pd);
-	free_pt(vm, vm->scratch_pt);
 	cleanup_scratch_page(vm);
 }
 
@@ -1322,7 +1264,7 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
 	int i;
 
 	for (i = 0; i < pdpes; i++) {
-		if (pdp->page_directory[i] == vm->scratch_pd)
+		if (!pdp->page_directory[i])
 			continue;
 
 		gen8_free_page_tables(vm, pdp->page_directory[i]);
@@ -1337,7 +1279,7 @@ static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
-		if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp)
+		if (!ppgtt->pml4.pdps[i])
 			continue;
 
 		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]);
@@ -1362,6 +1304,49 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 	gen8_free_scratch(vm);
 }
 
+static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
+			       struct i915_page_directory *pd,
+			       struct i915_page_table *pt,
+			       unsigned int pde)
+{
+	gen8_pde_t *vaddr;
+
+	pd->page_table[pde] = pt;
+
+	vaddr = kmap_atomic_px(pd);
+	vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
+	kunmap_atomic(vaddr);
+}
+
+static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
+				struct i915_page_directory_pointer *pdp,
+				struct i915_page_directory *pd,
+				unsigned int pdpe)
+{
+	gen8_ppgtt_pdpe_t *vaddr;
+
+	pdp->page_directory[pdpe] = pd;
+	if (!use_4lvl(vm))
+		return;
+
+	vaddr = kmap_atomic_px(pdp);
+	vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
+	kunmap_atomic(vaddr);
+}
+
+static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
+				 struct i915_page_directory_pointer *pdp,
+				 unsigned int pml4e)
+{
+	gen8_ppgtt_pml4e_t *vaddr;
+
+	pml4->pdps[pml4e] = pdp;
+
+	vaddr = kmap_atomic_px(pml4);
+	vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
+	kunmap_atomic(vaddr);
+}
+
 static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
 			       struct i915_page_directory *pd,
 			       u64 start, u64 length)
@@ -1373,7 +1358,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
 	gen8_for_each_pde(pt, pd, start, length, pde) {
 		int count = gen8_pte_count(start, length);
 
-		if (pt == vm->scratch_pt) {
+		if (!pt) {
 			pd->used_pdes++;
 
 			pt = alloc_pt(vm);
@@ -1408,7 +1393,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 	int ret;
 
 	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
-		if (pd == vm->scratch_pd) {
+		if (!pd) {
 			pdp->used_pdpes++;
 
 			pd = alloc_pd(vm);
@@ -1431,7 +1416,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 
 unwind_pd:
 	if (!pd->used_pdes) {
-		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+		gen8_ppgtt_clear_pdpe(vm, pdp, pdpe);
 		GEM_BUG_ON(!pdp->used_pdpes);
 		pdp->used_pdpes--;
 		free_pd(vm, pd);
@@ -1459,7 +1444,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
 	int ret;
 
 	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
-		if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+		if (!pdp) {
 			pdp = alloc_pdp(vm);
 			if (IS_ERR(pdp))
 				goto unwind;
@@ -1477,7 +1462,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
 
 unwind_pdp:
 	if (!pdp->used_pdpes) {
-		gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
+		gen8_ppgtt_clear_pm4le(vm, pml4, pml4e);
 		free_pdp(vm, pdp);
 	}
 unwind:
@@ -1510,7 +1495,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
 unwind:
 	start -= from;
 	gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
-		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+		gen8_ppgtt_clear_pdpe(vm, pdp, pdpe);
 		free_pd(vm, pd);
 	}
 	pdp->used_pdpes = 0;
@@ -1673,8 +1658,6 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 		const unsigned int count = min(num_entries, GEN6_PTES - pte);
 		gen6_pte_t *vaddr;
 
-		GEM_BUG_ON(pt == vm->scratch_pt);
-
 		num_entries -= count;
 
 		GEM_BUG_ON(count > pt->used_ptes);
@@ -1710,8 +1693,6 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 	struct sgt_dma iter = sgt_dma(vma);
 	gen6_pte_t *vaddr;
 
-	GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt);
-
 	vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
 	do {
 		vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
@@ -1749,7 +1730,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 	gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {
 		const unsigned int count = gen6_pte_count(start, length);
 
-		if (pt == vm->scratch_pt) {
+		if (!pt) {
 			pt = alloc_pt(vm);
 			if (IS_ERR(pt))
 				goto unwind_out;
@@ -1784,8 +1765,6 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
 {
 	struct i915_address_space * const vm = &ppgtt->base.vm;
-	struct i915_page_table *unused;
-	u32 pde;
 	int ret;
 
 	ret = setup_scratch_page(vm, __GFP_HIGHMEM);
@@ -1795,23 +1774,13 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
 	vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 					 I915_CACHE_NONE,
 					 PTE_READ_ONLY);
-
-	vm->scratch_pt = alloc_pt(vm);
-	if (IS_ERR(vm->scratch_pt)) {
-		cleanup_scratch_page(vm);
-		return PTR_ERR(vm->scratch_pt);
-	}
-
-	gen6_initialize_pt(vm, vm->scratch_pt);
-	gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
-		ppgtt->base.pd.page_table[pde] = vm->scratch_pt;
+	fill_page_dma_32(vm, &vm->scratch_page, vm->scratch_pte);
 
 	return 0;
 }
 
 static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
 {
-	free_pt(vm, vm->scratch_pt);
 	cleanup_scratch_page(vm);
 }
 
@@ -1821,7 +1790,7 @@ static void gen6_ppgtt_free_pd(struct gen6_hw_ppgtt *ppgtt)
 	u32 pde;
 
 	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)
-		if (pt != ppgtt->base.vm.scratch_pt)
+		if (pt)
 			free_pt(&ppgtt->base.vm, pt);
 }
 
@@ -1873,7 +1842,6 @@ static int pd_vma_bind(struct i915_vma *vma,
 static void pd_vma_unbind(struct i915_vma *vma)
 {
 	struct gen6_hw_ppgtt *ppgtt = vma->private;
-	struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;
 	struct i915_page_table *pt;
 	unsigned int pde;
 
@@ -1882,11 +1850,11 @@ static void pd_vma_unbind(struct i915_vma *vma)
 
 	/* Free all no longer used page tables */
 	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {
-		if (pt->used_ptes || pt == scratch_pt)
+		if (!pt || pt->used_ptes)
 			continue;
 
 		free_pt(&ppgtt->base.vm, pt);
-		ppgtt->base.pd.page_table[pde] = scratch_pt;
+		ppgtt->base.pd.page_table[pde] = NULL;
 	}
 
 	ppgtt->scan_for_unused_pt = false;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 86065d75b3ac..c30565f25b3e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -294,9 +294,6 @@ struct i915_address_space {
 	u64 scratch_pte;
 	int scratch_order;
 	struct i915_page_dma scratch_page;
-	struct i915_page_table *scratch_pt;
-	struct i915_page_directory *scratch_pd;
-	struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
 
 	/**
 	 * List of vma currently bound.
-- 
2.20.1



More information about the Intel-gfx mailing list