[PATCH 06/10] drm/i915/gtt: Convert vm->scratch into an array
Chris Wilson
chris at chris-wilson.co.uk
Sat Jul 6 16:12:39 UTC 2019
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 142 ++++++++++++----------------
drivers/gpu/drm/i915/i915_gem_gtt.h | 5 +-
2 files changed, 61 insertions(+), 86 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index b0d1a0127a9e..04afd1d15d09 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -634,7 +634,7 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL;
do {
- int order = get_order(size);
+ unsigned int order = get_order(size);
struct page *page;
dma_addr_t addr;
@@ -653,8 +653,8 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
if (unlikely(!IS_ALIGNED(addr, size)))
goto unmap_page;
- vm->scratch_page.page = page;
- vm->scratch_page.daddr = addr;
+ vm->scratch[0].page = page;
+ vm->scratch[0].daddr = addr;
vm->scratch_order = order;
return 0;
@@ -673,8 +673,8 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
static void cleanup_scratch_page(struct i915_address_space *vm)
{
- struct i915_page_dma *p = &vm->scratch_page;
- int order = vm->scratch_order;
+ struct i915_page_dma *p = &vm->scratch[0];
+ unsigned int order = vm->scratch_order;
dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT,
PCI_DMA_BIDIRECTIONAL);
@@ -884,7 +884,7 @@ static void gen8_free_page_tables(struct i915_address_space *vm,
int i;
for (i = 0; i < I915_PDES; i++) {
- if (pd->entry[i] != &vm->scratch_pt)
+ if (pd->entry[i] != &vm->scratch[1])
free_pd(vm, pd->entry[i]);
}
}
@@ -896,7 +896,7 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
int i;
for (i = 0; i < pdpes; i++) {
- if (pdp->entry[i] == &vm->scratch_pd)
+ if (pdp->entry[i] == &vm->scratch[2])
continue;
gen8_free_page_tables(vm, pdp->entry[i]);
@@ -914,7 +914,7 @@ static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
- if (px_base(pdp) == &ppgtt->vm.scratch_pdp)
+ if (px_base(pdp) == &ppgtt->vm.scratch[3])
continue;
gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
@@ -925,15 +925,15 @@ static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
static void gen8_free_scratch(struct i915_address_space *vm)
{
- if (!vm->scratch_page.daddr) /* set to 0 on clones */
+ int i;
+
+ if (!vm->scratch[0].daddr) /* set to 0 on clones */
return;
- if (vm->scratch_pdp.daddr)
- cleanup_page_dma(vm, &vm->scratch_pdp);
- if (vm->scratch_pd.daddr)
- cleanup_page_dma(vm, &vm->scratch_pd);
- if (vm->scratch_pt.daddr)
- cleanup_page_dma(vm, &vm->scratch_pt);
+ for (i = ARRAY_SIZE(vm->scratch); --i; ) {
+ if (vm->scratch[i].daddr)
+ cleanup_page_dma(vm, &vm->scratch[i]);
+ }
cleanup_scratch_page(vm);
}
@@ -981,11 +981,11 @@ static void gen8_ppgtt_clear_pd(struct i915_address_space *vm,
u32 pde;
gen8_for_each_pde(pt, pd, start, length, pde) {
- GEM_BUG_ON(px_base(pt) == &vm->scratch_pt);
+ GEM_BUG_ON(px_base(pt) == &vm->scratch[1]);
atomic_inc(&pt->used);
gen8_ppgtt_clear_pt(vm, pt, start, length);
- if (release_pd_entry(pd, pde, pt, &vm->scratch_pt))
+ if (release_pd_entry(pd, pde, pt, &vm->scratch[1]))
free_px(vm, pt);
}
}
@@ -1001,11 +1001,11 @@ static void gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
unsigned int pdpe;
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- GEM_BUG_ON(px_base(pd) == &vm->scratch_pd);
+ GEM_BUG_ON(px_base(pd) == &vm->scratch[2]);
atomic_inc(px_used(pd));
gen8_ppgtt_clear_pd(vm, pd, start, length);
- if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))
+ if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch[2]))
free_px(vm, pd);
}
}
@@ -1031,11 +1031,11 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
GEM_BUG_ON(!i915_vm_is_4lvl(vm));
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- GEM_BUG_ON(px_base(pdp) == &vm->scratch_pdp);
+ GEM_BUG_ON(px_base(pdp) == &vm->scratch[3]);
atomic_inc(px_used(pdp));
gen8_ppgtt_clear_pdp(vm, pdp, start, length);
- if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))
+ if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch[3]))
free_px(vm, pdp);
}
}
@@ -1054,7 +1054,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
gen8_for_each_pde(pt, pd, start, length, pde) {
const int count = gen8_pte_count(start, length);
- if (px_base(pt) == &vm->scratch_pt) {
+ if (px_base(pt) == &vm->scratch[1]) {
spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc);
@@ -1069,7 +1069,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
fill_px(pt, vm->scratch_pte);
spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch_pt) {
+ if (pd->entry[pde] == &vm->scratch[1]) {
set_pd_entry(pd, pde, pt);
} else {
alloc = pt;
@@ -1101,7 +1101,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
spin_lock(&pdp->lock);
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- if (px_base(pd) == &vm->scratch_pd) {
+ if (px_base(pd) == &vm->scratch[2]) {
spin_unlock(&pdp->lock);
pd = fetch_and_zero(&alloc);
@@ -1112,10 +1112,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
goto unwind;
}
- init_pd(pd, &vm->scratch_pt);
+ init_pd(pd, &vm->scratch[1]);
spin_lock(&pdp->lock);
- if (pdp->entry[pdpe] == &vm->scratch_pd) {
+ if (pdp->entry[pdpe] == &vm->scratch[2]) {
set_pd_entry(pdp, pdpe, pd);
} else {
alloc = pd;
@@ -1136,7 +1136,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
goto out;
unwind_pd:
- if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))
+ if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch[2]))
free_px(vm, pd);
unwind:
gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
@@ -1165,7 +1165,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
spin_lock(&pml4->lock);
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- if (px_base(pdp) == &vm->scratch_pdp) {
+ if (px_base(pdp) == &vm->scratch[3]) {
spin_unlock(&pml4->lock);
pdp = fetch_and_zero(&alloc);
@@ -1176,10 +1176,10 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
goto unwind;
}
- init_pd(pdp, &vm->scratch_pd);
+ init_pd(pdp, &vm->scratch[2]);
spin_lock(&pml4->lock);
- if (pml4->entry[pml4e] == &vm->scratch_pdp) {
+ if (pml4->entry[pml4e] == &vm->scratch[3]) {
set_pd_entry(pml4, pml4e, pdp);
} else {
alloc = pdp;
@@ -1200,7 +1200,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
goto out;
unwind_pdp:
- if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))
+ if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch[3]))
free_px(vm, pdp);
unwind:
gen8_ppgtt_clear_4lvl(vm, from, start - from);
@@ -1456,7 +1456,9 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
static int gen8_init_scratch(struct i915_address_space *vm)
{
+ u64 pte;
int ret;
+ int i;
/*
* If everybody agrees to not to write into the scratch page,
@@ -1470,10 +1472,8 @@ static int gen8_init_scratch(struct i915_address_space *vm)
GEM_BUG_ON(!clone->has_read_only);
vm->scratch_order = clone->scratch_order;
- vm->scratch_pte = clone->scratch_pte;
- vm->scratch_pt = clone->scratch_pt;
- vm->scratch_pd = clone->scratch_pd;
- vm->scratch_pdp = clone->scratch_pdp;
+ memcpy(vm->scratch, clone->scratch, sizeof(vm->scratch));
+ vm->scratch[0].daddr = 0; /* no xfer of ownership */
return 0;
}
@@ -1482,43 +1482,24 @@ static int gen8_init_scratch(struct i915_address_space *vm)
return ret;
vm->scratch_pte =
- gen8_pte_encode(vm->scratch_page.daddr,
+ gen8_pte_encode(vm->scratch[0].daddr,
I915_CACHE_LLC,
vm->has_read_only);
- if (unlikely(setup_page_dma(vm, &vm->scratch_pt))) {
- ret = -ENOMEM;
- goto free_scratch_page;
- }
- fill_page_dma(&vm->scratch_pt, vm->scratch_pte);
+ pte = vm->scratch_pte;
+ for (i = 1; i <= vm->top; i++) {
+ if (unlikely(setup_page_dma(vm, &vm->scratch[i])))
+ goto free_scratch;
- if (unlikely(setup_page_dma(vm, &vm->scratch_pd))) {
- ret = -ENOMEM;
- goto free_pt;
- }
- fill_page_dma(&vm->scratch_pd,
- gen8_pde_encode(vm->scratch_pd.daddr, I915_CACHE_LLC));
-
- if (i915_vm_is_4lvl(vm)) {
- if (unlikely(setup_page_dma(vm, &vm->scratch_pdp))) {
- ret = -ENOMEM;
- goto free_pd;
- }
- fill_page_dma(&vm->scratch_pdp,
- gen8_pde_encode(vm->scratch_pdp.daddr,
- I915_CACHE_LLC));
+ fill_page_dma(&vm->scratch[i], pte);
+ pte = gen8_pde_encode(vm->scratch[i].daddr, I915_CACHE_LLC);
}
return 0;
-free_pd:
- cleanup_page_dma(vm, &vm->scratch_pd);
-free_pt:
- cleanup_page_dma(vm, &vm->scratch_pt);
-free_scratch_page:
- cleanup_scratch_page(vm);
-
- return ret;
+free_scratch:
+ gen8_free_scratch(vm);
+ return -ENOMEM;
}
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
@@ -1535,7 +1516,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
if (IS_ERR(pd))
goto unwind;
- init_pd(pd, &vm->scratch_pt);
+ init_pd(pd, &vm->scratch[1]);
set_pd_entry(pdp, pdpe, pd);
}
@@ -1588,7 +1569,7 @@ gen8_alloc_top_pd(struct i915_address_space *vm)
if (i915_vm_is_4lvl(vm)) {
pd = alloc_pd(vm);
if (!IS_ERR(pd))
- init_pd(pd, &vm->scratch_pdp);
+ init_pd(pd, &vm->scratch[3]);
return pd;
}
@@ -1605,7 +1586,7 @@ gen8_alloc_top_pd(struct i915_address_space *vm)
return ERR_PTR(-ENOMEM);
}
- init_pd_n(vm, pd, &vm->scratch_pd, GEN8_3LVL_PDPES);
+ init_pd_n(vm, pd, &vm->scratch[2], GEN8_3LVL_PDPES);
return pd;
}
@@ -1763,7 +1744,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
const unsigned int count = min(num_entries, GEN6_PTES - pte);
gen6_pte_t *vaddr;
- GEM_BUG_ON(px_base(pt) == &vm->scratch_pt);
+ GEM_BUG_ON(px_base(pt) == &vm->scratch[1]);
num_entries -= count;
@@ -1800,7 +1781,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
struct sgt_dma iter = sgt_dma(vma);
gen6_pte_t *vaddr;
- GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch_pt);
+ GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch[1]);
vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
do {
@@ -1845,7 +1826,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
gen6_for_each_pde(pt, pd, start, length, pde) {
const unsigned int count = gen6_pte_count(start, length);
- if (px_base(pt) == &vm->scratch_pt) {
+ if (px_base(pt) == &vm->scratch[1]) {
spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc);
@@ -1859,7 +1840,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
fill32_px(pt, vm->scratch_pte);
spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch_pt) {
+ if (pd->entry[pde] == &vm->scratch[1]) {
pd->entry[pde] = pt;
if (i915_vma_is_bound(ppgtt->vma,
I915_VMA_GLOBAL_BIND)) {
@@ -1896,33 +1877,30 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
{
struct i915_address_space * const vm = &ppgtt->base.vm;
struct i915_page_directory * const pd = ppgtt->base.pd;
- struct i915_page_table *unused;
- u32 pde;
int ret;
ret = setup_scratch_page(vm, __GFP_HIGHMEM);
if (ret)
return ret;
- vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
+ vm->scratch_pte = vm->pte_encode(vm->scratch[0].daddr,
I915_CACHE_NONE,
PTE_READ_ONLY);
- if (unlikely(setup_page_dma(vm, &vm->scratch_pt))) {
+ if (unlikely(setup_page_dma(vm, &vm->scratch[1]))) {
cleanup_scratch_page(vm);
return -ENOMEM;
}
- fill_page_dma_32(&vm->scratch_pt, vm->scratch_pte);
- gen6_for_all_pdes(unused, pd, pde)
- pd->entry[pde] = &vm->scratch_pt;
+ fill_page_dma_32(&vm->scratch[1], vm->scratch_pte);
+ memset_p(pd->entry, &vm->scratch[1], I915_PDES);
return 0;
}
static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
{
- cleanup_page_dma(vm, &vm->scratch_pt);
+ cleanup_page_dma(vm, &vm->scratch[1]);
cleanup_scratch_page(vm);
}
@@ -1933,7 +1911,7 @@ static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
u32 pde;
gen6_for_all_pdes(pt, pd, pde)
- if (px_base(pt) != &ppgtt->base.vm.scratch_pt)
+ if (px_base(pt) != &ppgtt->base.vm.scratch[1])
free_px(&ppgtt->base.vm, pt);
}
@@ -1991,7 +1969,7 @@ static void pd_vma_unbind(struct i915_vma *vma)
{
struct gen6_ppgtt *ppgtt = vma->private;
struct i915_page_directory * const pd = ppgtt->base.pd;
- struct i915_page_dma * const scratch = &ppgtt->base.vm.scratch_pt;
+ struct i915_page_dma * const scratch = &ppgtt->base.vm.scratch[1];
struct i915_page_table *pt;
unsigned int pde;
@@ -2998,7 +2976,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
}
ggtt->vm.scratch_pte =
- ggtt->vm.pte_encode(ggtt->vm.scratch_page.daddr,
+ ggtt->vm.pte_encode(ggtt->vm.scratch[0].daddr,
I915_CACHE_NONE, 0);
return 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 2740cc34c120..76b959862476 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -317,12 +317,9 @@ struct i915_address_space {
#define VM_CLASS_GGTT 0
#define VM_CLASS_PPGTT 1
+ struct i915_page_dma scratch[4];
u64 scratch_pte;
int scratch_order;
- struct i915_page_dma scratch_page;
- struct i915_page_dma scratch_pt;
- struct i915_page_dma scratch_pd;
- struct i915_page_dma scratch_pdp; /* GEN8+ & 48b PPGTT */
int top;
/**
--
2.20.1
More information about the Intel-gfx-trybot
mailing list