[PATCH] drm/i915/gtt: Recursive scratch page
Chris Wilson
chris at chris-wilson.co.uk
Wed Mar 6 08:03:58 UTC 2019
Fill the scratch page with pointers to itself so that we can reuse it at
any level of the page tables.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld at intel.com>
Cc: Mika Kuoppala <mika.kuoppala at intel.com>
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 226 +++++++++++-----------------
drivers/gpu/drm/i915/i915_gem_gtt.h | 3 -
2 files changed, 85 insertions(+), 144 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index dac08d9c3fab..85eea61e3650 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -687,7 +687,7 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
return ERR_PTR(-ENOMEM);
}
- pd->used_pdes = 0;
+ fill_px(vm, pd, vm->scratch_pte);
return pd;
}
@@ -698,26 +698,16 @@ static void free_pd(struct i915_address_space *vm,
kfree(pd);
}
-static void gen8_initialize_pd(struct i915_address_space *vm,
- struct i915_page_directory *pd)
-{
- fill_px(vm, pd,
- gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
- memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);
-}
-
static int __pdp_init(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp)
{
const unsigned int pdpes = i915_pdpes_per_pdp(vm);
- pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
- I915_GFP_ALLOW_FAIL);
+ pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
+ I915_GFP_ALLOW_FAIL);
if (unlikely(!pdp->page_directory))
return -ENOMEM;
- memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
-
return 0;
}
@@ -752,6 +742,7 @@ alloc_pdp(struct i915_address_space *vm)
if (ret)
goto fail_page_m;
+ fill_px(vm, pdp, vm->scratch_pte);
return pdp;
fail_page_m:
@@ -774,24 +765,6 @@ static void free_pdp(struct i915_address_space *vm,
kfree(pdp);
}
-static void gen8_initialize_pdp(struct i915_address_space *vm,
- struct i915_page_directory_pointer *pdp)
-{
- gen8_ppgtt_pdpe_t scratch_pdpe;
-
- scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
-
- fill_px(vm, pdp, scratch_pdpe);
-}
-
-static void gen8_initialize_pml4(struct i915_address_space *vm,
- struct i915_pml4 *pml4)
-{
- fill_px(vm, pml4,
- gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
- memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
-}
-
/*
* PDE TLBs are a pain to invalidate on GEN8+. When we modify
* the page table structures, we mark them dirty so that
@@ -826,17 +799,16 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
return false;
}
-static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- struct i915_page_table *pt,
- unsigned int pde)
+static void gen8_ppgtt_clear_pde(struct i915_address_space *vm,
+ struct i915_page_directory *pd,
+ unsigned int pde)
{
gen8_pde_t *vaddr;
- pd->page_table[pde] = pt;
+ pd->page_table[pde] = NULL;
vaddr = kmap_atomic_px(pd);
- vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
+ vaddr[pde] = vm->scratch_pte;
kunmap_atomic(vaddr);
}
@@ -848,34 +820,30 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
u32 pde;
gen8_for_each_pde(pt, pd, start, length, pde) {
- GEM_BUG_ON(pt == vm->scratch_pt);
-
if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
continue;
- gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
+ gen8_ppgtt_clear_pde(vm, pd, pde);
GEM_BUG_ON(!pd->used_pdes);
pd->used_pdes--;
-
free_pt(vm, pt);
}
return !pd->used_pdes;
}
-static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
- struct i915_page_directory_pointer *pdp,
- struct i915_page_directory *pd,
- unsigned int pdpe)
+static void gen8_ppgtt_clear_pdpe(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp,
+ unsigned int pdpe)
{
gen8_ppgtt_pdpe_t *vaddr;
- pdp->page_directory[pdpe] = pd;
+ pdp->page_directory[pdpe] = NULL;
if (!use_4lvl(vm))
return;
vaddr = kmap_atomic_px(pdp);
- vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
+ vaddr[pdpe] = vm->scratch_pte;
kunmap_atomic(vaddr);
}
@@ -890,15 +858,12 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
unsigned int pdpe;
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- GEM_BUG_ON(pd == vm->scratch_pd);
-
if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
continue;
- gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+ gen8_ppgtt_clear_pdpe(vm, pdp, pdpe);
GEM_BUG_ON(!pdp->used_pdpes);
pdp->used_pdpes--;
-
free_pd(vm, pd);
}
@@ -911,16 +876,16 @@ static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
}
-static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
- struct i915_page_directory_pointer *pdp,
- unsigned int pml4e)
+static void gen8_ppgtt_clear_pm4le(struct i915_address_space *vm,
+ struct i915_pml4 *pml4,
+ unsigned int pml4e)
{
gen8_ppgtt_pml4e_t *vaddr;
- pml4->pdps[pml4e] = pdp;
+ pml4->pdps[pml4e] = NULL;
vaddr = kmap_atomic_px(pml4);
- vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
+ vaddr[pml4e] = vm->scratch_pte;
kunmap_atomic(vaddr);
}
@@ -939,13 +904,10 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
GEM_BUG_ON(!use_4lvl(vm));
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- GEM_BUG_ON(pdp == vm->scratch_pdp);
-
if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
continue;
- gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
-
+ gen8_ppgtt_clear_pm4le(vm, pml4, pml4e);
free_pdp(vm, pdp);
}
}
@@ -1197,8 +1159,10 @@ static void gen8_free_page_tables(struct i915_address_space *vm,
int i;
for (i = 0; i < I915_PDES; i++) {
- if (pd->page_table[i] != vm->scratch_pt)
- free_pt(vm, pd->page_table[i]);
+ if (!pd->page_table[i])
+ continue;
+
+ free_pt(vm, pd->page_table[i]);
}
}
@@ -1220,9 +1184,6 @@ static int gen8_init_scratch(struct i915_address_space *vm)
vm->scratch_order = clone->scratch_order;
vm->scratch_pte = clone->scratch_pte;
- vm->scratch_pt = clone->scratch_pt;
- vm->scratch_pd = clone->scratch_pd;
- vm->scratch_pdp = clone->scratch_pdp;
return 0;
}
@@ -1230,46 +1191,10 @@ static int gen8_init_scratch(struct i915_address_space *vm)
if (ret)
return ret;
- vm->scratch_pte =
- gen8_pte_encode(vm->scratch_page.daddr,
- I915_CACHE_LLC,
- PTE_READ_ONLY);
-
- vm->scratch_pt = alloc_pt(vm);
- if (IS_ERR(vm->scratch_pt)) {
- ret = PTR_ERR(vm->scratch_pt);
- goto free_scratch_page;
- }
-
- vm->scratch_pd = alloc_pd(vm);
- if (IS_ERR(vm->scratch_pd)) {
- ret = PTR_ERR(vm->scratch_pd);
- goto free_pt;
- }
-
- if (use_4lvl(vm)) {
- vm->scratch_pdp = alloc_pdp(vm);
- if (IS_ERR(vm->scratch_pdp)) {
- ret = PTR_ERR(vm->scratch_pdp);
- goto free_pd;
- }
- }
-
- gen8_initialize_pt(vm, vm->scratch_pt);
- gen8_initialize_pd(vm, vm->scratch_pd);
- if (use_4lvl(vm))
- gen8_initialize_pdp(vm, vm->scratch_pdp);
+ vm->scratch_pte = vm->scratch_page.daddr | PPAT_UNCACHED;
+ fill_page_dma(vm, &vm->scratch_page, vm->scratch_pte);
return 0;
-
-free_pd:
- free_pd(vm, vm->scratch_pd);
-free_pt:
- free_pt(vm, vm->scratch_pt);
-free_scratch_page:
- cleanup_scratch_page(vm);
-
- return ret;
}
static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
@@ -1309,10 +1234,6 @@ static void gen8_free_scratch(struct i915_address_space *vm)
if (!vm->scratch_page.daddr)
return;
- if (use_4lvl(vm))
- free_pdp(vm, vm->scratch_pdp);
- free_pd(vm, vm->scratch_pd);
- free_pt(vm, vm->scratch_pt);
cleanup_scratch_page(vm);
}
@@ -1323,7 +1244,7 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
int i;
for (i = 0; i < pdpes; i++) {
- if (pdp->page_directory[i] == vm->scratch_pd)
+ if (!pdp->page_directory[i])
continue;
gen8_free_page_tables(vm, pdp->page_directory[i]);
@@ -1338,7 +1259,7 @@ static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
int i;
for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
- if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp)
+ if (!ppgtt->pml4.pdps[i])
continue;
gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]);
@@ -1363,6 +1284,49 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
gen8_free_scratch(vm);
}
+static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
+ struct i915_page_directory *pd,
+ struct i915_page_table *pt,
+ unsigned int pde)
+{
+ gen8_pde_t *vaddr;
+
+ pd->page_table[pde] = pt;
+
+ vaddr = kmap_atomic_px(pd);
+ vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
+ kunmap_atomic(vaddr);
+}
+
+static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
+ struct i915_page_directory_pointer *pdp,
+ struct i915_page_directory *pd,
+ unsigned int pdpe)
+{
+ gen8_ppgtt_pdpe_t *vaddr;
+
+ pdp->page_directory[pdpe] = pd;
+ if (!use_4lvl(vm))
+ return;
+
+ vaddr = kmap_atomic_px(pdp);
+ vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
+ kunmap_atomic(vaddr);
+}
+
+static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
+ struct i915_page_directory_pointer *pdp,
+ unsigned int pml4e)
+{
+ gen8_ppgtt_pml4e_t *vaddr;
+
+ pml4->pdps[pml4e] = pdp;
+
+ vaddr = kmap_atomic_px(pml4);
+ vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
+ kunmap_atomic(vaddr);
+}
+
static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
struct i915_page_directory *pd,
u64 start, u64 length)
@@ -1374,7 +1338,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
gen8_for_each_pde(pt, pd, start, length, pde) {
int count = gen8_pte_count(start, length);
- if (pt == vm->scratch_pt) {
+ if (!pt) {
pd->used_pdes++;
pt = alloc_pt(vm);
@@ -1409,7 +1373,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
int ret;
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- if (pd == vm->scratch_pd) {
+ if (!pd) {
pdp->used_pdpes++;
pd = alloc_pd(vm);
@@ -1418,7 +1382,6 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
goto unwind;
}
- gen8_initialize_pd(vm, pd);
gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
}
@@ -1432,7 +1395,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
unwind_pd:
if (!pd->used_pdes) {
- gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+ gen8_ppgtt_clear_pdpe(vm, pdp, pdpe);
GEM_BUG_ON(!pdp->used_pdpes);
pdp->used_pdpes--;
free_pd(vm, pd);
@@ -1460,12 +1423,11 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
int ret;
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+ if (!pdp) {
pdp = alloc_pdp(vm);
if (IS_ERR(pdp))
goto unwind;
- gen8_initialize_pdp(vm, pdp);
gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
}
@@ -1478,7 +1440,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
unwind_pdp:
if (!pdp->used_pdpes) {
- gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
+ gen8_ppgtt_clear_pm4le(vm, pml4, pml4e);
free_pdp(vm, pdp);
}
unwind:
@@ -1500,7 +1462,6 @@ static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
if (IS_ERR(pd))
goto unwind;
- gen8_initialize_pd(vm, pd);
gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
pdp->used_pdpes++;
}
@@ -1511,7 +1472,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
unwind:
start -= from;
gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
- gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+ gen8_ppgtt_clear_pdpe(vm, pdp, pdpe);
free_pd(vm, pd);
}
pdp->used_pdpes = 0;
@@ -1563,7 +1524,7 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
if (err)
goto err_scratch;
- gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4);
+ fill_px(&ppgtt->vm, &ppgtt->pml4, ppgtt->vm.scratch_pte);
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
@@ -1674,8 +1635,6 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
const unsigned int count = min(num_entries, GEN6_PTES - pte);
gen6_pte_t *vaddr;
- GEM_BUG_ON(pt == vm->scratch_pt);
-
num_entries -= count;
GEM_BUG_ON(count > pt->used_ptes);
@@ -1711,8 +1670,6 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
struct sgt_dma iter = sgt_dma(vma);
gen6_pte_t *vaddr;
- GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt);
-
vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
do {
vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
@@ -1750,7 +1707,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {
const unsigned int count = gen6_pte_count(start, length);
- if (pt == vm->scratch_pt) {
+ if (!pt) {
pt = alloc_pt(vm);
if (IS_ERR(pt))
goto unwind_out;
@@ -1785,8 +1742,6 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
{
struct i915_address_space * const vm = &ppgtt->base.vm;
- struct i915_page_table *unused;
- u32 pde;
int ret;
ret = setup_scratch_page(vm, __GFP_HIGHMEM);
@@ -1796,23 +1751,13 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
I915_CACHE_NONE,
PTE_READ_ONLY);
-
- vm->scratch_pt = alloc_pt(vm);
- if (IS_ERR(vm->scratch_pt)) {
- cleanup_scratch_page(vm);
- return PTR_ERR(vm->scratch_pt);
- }
-
- gen6_initialize_pt(vm, vm->scratch_pt);
- gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
- ppgtt->base.pd.page_table[pde] = vm->scratch_pt;
+ fill_page_dma_32(vm, &vm->scratch_page, vm->scratch_pte);
return 0;
}
static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
{
- free_pt(vm, vm->scratch_pt);
cleanup_scratch_page(vm);
}
@@ -1822,7 +1767,7 @@ static void gen6_ppgtt_free_pd(struct gen6_hw_ppgtt *ppgtt)
u32 pde;
gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)
- if (pt != ppgtt->base.vm.scratch_pt)
+ if (pt)
free_pt(&ppgtt->base.vm, pt);
}
@@ -1874,7 +1819,6 @@ static int pd_vma_bind(struct i915_vma *vma,
static void pd_vma_unbind(struct i915_vma *vma)
{
struct gen6_hw_ppgtt *ppgtt = vma->private;
- struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;
struct i915_page_table *pt;
unsigned int pde;
@@ -1883,11 +1827,11 @@ static void pd_vma_unbind(struct i915_vma *vma)
/* Free all no longer used page tables */
gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {
- if (pt->used_ptes || pt == scratch_pt)
+ if (!pt || pt->used_ptes)
continue;
free_pt(&ppgtt->base.vm, pt);
- ppgtt->base.pd.page_table[pde] = scratch_pt;
+ ppgtt->base.pd.page_table[pde] = NULL;
}
ppgtt->scan_for_unused_pt = false;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index a47e11e6fc1b..bbdad496375e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -294,9 +294,6 @@ struct i915_address_space {
u64 scratch_pte;
int scratch_order;
struct i915_page_dma scratch_page;
- struct i915_page_table *scratch_pt;
- struct i915_page_directory *scratch_pd;
- struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
/**
* List of vma currently bound.
--
2.20.1
More information about the Intel-gfx-trybot
mailing list