[PATCH 09/10] drm/i915/gtt: Recursive ppgtt alloc for gen8
Chris Wilson
chris at chris-wilson.co.uk
Sat Jul 6 19:16:50 UTC 2019
Refactor the separate allocation routines into a single recursive
function.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 274 ++++++++--------------------
1 file changed, 81 insertions(+), 193 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index ee3fef237bdf..90ad1c4de25b 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -978,187 +978,93 @@ static void gen8_ppgtt_clear(struct i915_address_space *vm,
&start, start + length, vm->top);
}
-static void gen8_ppgtt_clear_pd(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- u64 start, u64 length)
-{
- __gen8_ppgtt_clear(vm, pd, &start, start + length, 1);
-}
-
-static void gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
- struct i915_page_directory * const pdp,
- u64 start, u64 length)
-{
- __gen8_ppgtt_clear(vm, pdp, &start, start + length, 2);
-}
-
-static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- u64 start, u64 length)
+static int __gen8_ppgtt_alloc(struct i915_address_space *vm,
+ struct i915_page_directory *pd,
+ u64 *start, u64 end, int lvl)
{
+ struct i915_page_scratch * const scratch = &vm->scratch[lvl];
struct i915_page_table *pt, *alloc = NULL;
- u64 from = start;
- unsigned int pde;
+ unsigned int idx, len, count;
int ret = 0;
+ len = gen8_pd_range(*start, end, lvl, &idx);
+ GEM_BUG_ON(!len || (idx + len - 1) >> pd_lvl(1));
+
spin_lock(&pd->lock);
- gen8_for_each_pde(pt, pd, start, length, pde) {
- const int count = gen8_pte_count(start, length);
+ do {
+ GEM_BUG_ON(!atomic_read(px_used(pd))); /* Must be pinned! */
+ pt = pd->entry[idx];
- if (px_base(pt) == px_base(&vm->scratch[1])) {
+ if (px_base(pt) == px_base(scratch)) {
spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc);
if (!pt)
- pt = alloc_pt(vm);
+ pt = (lvl > 1) ? &alloc_pd(vm)->pt : alloc_pt(vm);
if (IS_ERR(pt)) {
ret = PTR_ERR(pt);
- goto unwind;
+ break;
}
- if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
- fill_px(pt, vm->scratch[0].encode);
+ count = gen8_pd_count(*start, end, lvl - 1);
+ if (count < I915_PDES || intel_vgpu_active(vm->i915)) {
+ fill_px(pt, vm->scratch[lvl - 1].encode);
+ if (lvl > 1) {
+ memset_p(as_pd(pt)->entry,
+ &vm->scratch[lvl - 1],
+ I915_PDES);
+ }
+ }
spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch[1]) {
- set_pd_entry(pd, pde, pt);
- } else {
- alloc = pt;
- pt = pd->entry[pde];
- }
+ if (likely(pd->entry[idx] == scratch))
+ set_pd_entry(pd, idx, pt);
+ else
+ alloc = pt, pt = pd->entry[idx];
}
- atomic_add(count, &pt->used);
- }
- spin_unlock(&pd->lock);
- goto out;
-
-unwind:
- gen8_ppgtt_clear_pd(vm, pd, from, start - from);
-out:
- if (alloc)
- free_px(vm, alloc);
- return ret;
-}
-
-static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
- struct i915_page_directory *pdp,
- u64 start, u64 length)
-{
- struct i915_page_directory *pd, *alloc = NULL;
- u64 from = start;
- unsigned int pdpe;
- int ret = 0;
+ if (lvl > 1) {
+ atomic_inc(&pt->used);
+ spin_unlock(&pd->lock);
- spin_lock(&pdp->lock);
- gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- if (px_base(pd) == px_base(&vm->scratch[2])) {
- spin_unlock(&pdp->lock);
-
- pd = fetch_and_zero(&alloc);
- if (!pd)
- pd = alloc_pd(vm);
- if (IS_ERR(pd)) {
- ret = PTR_ERR(pd);
- goto unwind;
+ ret = __gen8_ppgtt_alloc(vm, as_pd(pt),
+ start, end, lvl - 1);
+ if (unlikely(ret)) {
+ if (release_pd_entry(pd, idx, pt, scratch))
+ free_px(vm, pt);
+ break;
}
- init_pd(pd, &vm->scratch[1]);
-
- spin_lock(&pdp->lock);
- if (pdp->entry[pdpe] == &vm->scratch[2]) {
- set_pd_entry(pdp, pdpe, pd);
- } else {
- alloc = pd;
- pd = pdp->entry[pdpe];
- }
+ spin_lock(&pd->lock);
+ atomic_dec(&pt->used);
+ GEM_BUG_ON(!atomic_read(&pt->used));
+ } else {
+ count = gen8_pt_count(*start, end);
+ atomic_add(count, &pt->used);
+ *start += count << gen8_pd_shift(0);
}
- atomic_inc(px_used(pd));
- spin_unlock(&pdp->lock);
-
- ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
- if (unlikely(ret))
- goto unwind_pd;
-
- spin_lock(&pdp->lock);
- atomic_dec(px_used(pd));
- }
- spin_unlock(&pdp->lock);
- goto out;
+ } while (idx++, --len);
+ spin_unlock(&pd->lock);
-unwind_pd:
- if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch[2]))
- free_px(vm, pd);
-unwind:
- gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
-out:
if (alloc)
free_px(vm, alloc);
- return ret;
-}
-static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
- u64 start, u64 length)
-{
- return gen8_ppgtt_alloc_pdp(vm,
- i915_vm_to_ppgtt(vm)->pd, start, length);
+ return ret;
}
-static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
- u64 start, u64 length)
+static int gen8_ppgtt_alloc(struct i915_address_space *vm,
+ u64 start, u64 length)
{
- struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
- struct i915_page_directory * const pml4 = ppgtt->pd;
- struct i915_page_directory *pdp, *alloc = NULL;
u64 from = start;
- int ret = 0;
- u32 pml4e;
-
- spin_lock(&pml4->lock);
- gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- if (px_base(pdp) == px_base(&vm->scratch[3])) {
- spin_unlock(&pml4->lock);
-
- pdp = fetch_and_zero(&alloc);
- if (!pdp)
- pdp = alloc_pd(vm);
- if (IS_ERR(pdp)) {
- ret = PTR_ERR(pdp);
- goto unwind;
- }
-
- init_pd(pdp, &vm->scratch[2]);
-
- spin_lock(&pml4->lock);
- if (pml4->entry[pml4e] == &vm->scratch[3]) {
- set_pd_entry(pml4, pml4e, pdp);
- } else {
- alloc = pdp;
- pdp = pml4->entry[pml4e];
- }
- }
- atomic_inc(px_used(pdp));
- spin_unlock(&pml4->lock);
-
- ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
- if (unlikely(ret))
- goto unwind_pdp;
+ int err;
- spin_lock(&pml4->lock);
- atomic_dec(px_used(pdp));
- }
- spin_unlock(&pml4->lock);
- goto out;
+ err = __gen8_ppgtt_alloc(vm, i915_vm_to_ppgtt(vm)->pd,
+ &start, start + length, vm->top);
+ if (unlikely(err))
+ __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,
+ &from, start, vm->top);
-unwind_pdp:
- if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch[3]))
- free_px(vm, pdp);
-unwind:
- gen8_ppgtt_clear(vm, from, start - from);
-out:
- if (alloc)
- free_px(vm, alloc);
- return ret;
+ return err;
}
static inline struct sgt_dma {
@@ -1457,26 +1363,28 @@ static int gen8_init_scratch(struct i915_address_space *vm)
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
{
struct i915_address_space *vm = &ppgtt->vm;
- struct i915_page_directory *pdp = ppgtt->pd;
- struct i915_page_directory *pd;
- u64 start = 0, length = ppgtt->vm.total;
- u64 from = start;
- unsigned int pdpe;
+ struct i915_page_directory *pd = ppgtt->pd;
+ unsigned int idx;
+
+ GEM_BUG_ON(vm->top != 2);
+ GEM_BUG_ON((ppgtt->vm.total >> gen8_pd_shift(2)) != GEN8_3LVL_PDPES);
+
+ for (idx = 0; idx < GEN8_3LVL_PDPES; idx++) {
+ struct i915_page_directory *pde;
- gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- pd = alloc_pd(vm);
- if (IS_ERR(pd))
+ pde = alloc_pd(vm);
+ if (IS_ERR(pde))
goto unwind;
- init_pd(pd, &vm->scratch[1]);
- set_pd_entry(pdp, pdpe, pd);
+ init_pd(pde, &vm->scratch[1]);
+ set_pd_entry(pd, idx, &pde->pt);
}
return 0;
unwind:
- gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
- atomic_set(px_used(pdp), 0);
+ gen8_ppgtt_clear(vm, 0, idx << gen8_pd_shift(2));
+ atomic_set(px_used(pd), 0);
return -ENOMEM;
}
@@ -1499,46 +1407,27 @@ static void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt)
ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;
}
-static void init_pd_n(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- struct i915_page_scratch *scratch,
- const unsigned int entries)
-{
- u64 * const vaddr = kmap_atomic_px(pd);
-
- memset64(vaddr, scratch->encode, entries);
- kunmap_atomic(vaddr);
-
- memset_p(pd->entry, scratch, entries);
-}
-
static struct i915_page_directory *
gen8_alloc_top_pd(struct i915_address_space *vm)
{
+ const int top = vm->top;
+ const int count = vm->total >> gen8_pd_shift(top);
struct i915_page_directory *pd;
+ void *vaddr;
- if (i915_vm_is_4lvl(vm)) {
- pd = alloc_pd(vm);
- if (!IS_ERR(pd))
- init_pd(pd, &vm->scratch[3]);
+ GEM_BUG_ON((vm->total - 1) >> gen8_pd_shift(top + 1));
+ pd = alloc_pd(vm);
+ if (IS_ERR(pd))
return pd;
- }
-
- /* 3lvl */
- pd = __alloc_pd();
- if (!pd)
- return ERR_PTR(-ENOMEM);
- pd->entry[GEN8_3LVL_PDPES] = NULL;
-
- if (unlikely(setup_page_dma(vm, px_base(pd)))) {
- kfree(pd);
- return ERR_PTR(-ENOMEM);
- }
+ vaddr = kmap_atomic_px(pd);
+ memset64(vaddr, vm->scratch[top].encode, count);
+ kunmap_atomic(vaddr);
- init_pd_n(vm, pd, &vm->scratch[2], GEN8_3LVL_PDPES);
+ memset_p(pd->entry, &vm->scratch[top], count);
+ atomic_inc(px_used(pd)); /* mark as pinned */
return pd;
}
@@ -1585,7 +1474,6 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
}
if (i915_vm_is_4lvl(&ppgtt->vm)) {
- ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
} else {
if (intel_vgpu_active(i915)) {
@@ -1594,10 +1482,10 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
goto err_free_pd;
}
- ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;
}
+ ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
ppgtt->vm.clear_range = gen8_ppgtt_clear;
if (intel_vgpu_active(i915))
--
2.20.1
More information about the Intel-gfx-trybot
mailing list