[PATCH 7/9] drm/i915/gtt: Use NULL to encode scratch shadow entries
Chris Wilson
chris at chris-wilson.co.uk
Sat Jul 6 23:43:45 UTC 2019
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 125 +++++++++-------------------
1 file changed, 38 insertions(+), 87 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 375083485592..4744565aed6d 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -596,18 +596,17 @@ static void cleanup_page_dma(struct i915_address_space *vm,
#define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
-#define fill_px(px, v) fill_page_dma(px_base(px), (v))
-#define fill32_px(px, v) fill_page_dma_32(px_base(px), (v))
-
-static void fill_page_dma(struct i915_page_dma *p, const u64 val)
+static void
+fill_page_dma(struct i915_page_dma *p, const u64 val, unsigned int count)
{
- kunmap_atomic(memset64(kmap_atomic(p->page), val, I915_PDES));
+ kunmap_atomic(memset64(kmap_atomic(p->page), val, count));
}
-static void fill_page_dma_32(struct i915_page_dma *p, const u32 v)
-{
- fill_page_dma(p, (u64)v << 32 | v);
-}
+#define fill_px(px, v) fill_page_dma(px_base(px), (v), I915_PDES)
+#define fill32_px(px, v) do { \
+ u64 vv = lower_32_bits(v); \
+ fill_px(px, vv << 32 | vv); \
+} while (0)
static int
setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
@@ -711,7 +710,6 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
}
atomic_set(&pt->used, 0);
-
return pt;
}
@@ -719,13 +717,11 @@ static struct i915_page_directory *__alloc_pd(void)
{
struct i915_page_directory *pd;
- pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
+ pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
if (unlikely(!pd))
return NULL;
- atomic_set(px_used(pd), 0);
spin_lock_init(&pd->lock);
-
return pd;
}
@@ -753,13 +749,6 @@ static void free_pd(struct i915_address_space *vm, struct i915_page_dma *pd)
#define free_px(vm, px) free_pd(vm, px_base(px))
-static void init_pd(struct i915_page_directory *pd,
- struct i915_page_scratch *scratch)
-{
- fill_px(pd, scratch->encode);
- memset_p(pd->entry, scratch, 512);
-}
-
static inline void
write_dma_entry(struct i915_page_dma * const pdma,
const unsigned short pde,
@@ -795,7 +784,7 @@ clear_pd_entry(struct i915_page_directory * const pd,
GEM_BUG_ON(atomic_read(px_used(pd)) == 0);
write_dma_entry(px_base(pd), pde, scratch->encode);
- pd->entry[pde] = scratch;
+ pd->entry[pde] = NULL;
atomic_dec(px_used(pd));
}
@@ -903,7 +892,7 @@ static void gen8_free_page_tables(struct i915_address_space *vm,
int i;
for (i = 0; i < I915_PDES; i++) {
- if (pd->entry[i] != &vm->scratch[1])
+ if (pd->entry[i])
free_pd(vm, pd->entry[i]);
}
}
@@ -915,7 +904,7 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
int i;
for (i = 0; i < pdpes; i++) {
- if (pdp->entry[i] == &vm->scratch[2])
+ if (!pdp->entry[i])
continue;
gen8_free_page_tables(vm, pdp->entry[i]);
@@ -933,7 +922,7 @@ static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
- if (px_base(pdp) == px_base(&ppgtt->vm.scratch[3]))
+ if (!pdp)
continue;
gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
@@ -987,8 +976,6 @@ static void gen8_ppgtt_clear_pd(struct i915_address_space *vm,
u32 pde;
gen8_for_each_pde(pt, pd, start, length, pde) {
- GEM_BUG_ON(px_base(pt) == px_base(&vm->scratch[1]));
-
atomic_inc(&pt->used);
gen8_ppgtt_clear_pt(vm, pt, start, length);
if (release_pd_entry(pd, pde, pt, &vm->scratch[1]))
@@ -1007,8 +994,6 @@ static void gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
unsigned int pdpe;
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- GEM_BUG_ON(px_base(pd) == px_base(&vm->scratch[2]));
-
atomic_inc(px_used(pd));
gen8_ppgtt_clear_pd(vm, pd, start, length);
if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch[2]))
@@ -1037,8 +1022,6 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
GEM_BUG_ON(!i915_vm_is_4lvl(vm));
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- GEM_BUG_ON(px_base(pdp) == px_base(&vm->scratch[3]));
-
atomic_inc(px_used(pdp));
gen8_ppgtt_clear_pdp(vm, pdp, start, length);
if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch[3]))
@@ -1059,7 +1042,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
gen8_for_each_pde(pt, pd, start, length, pde) {
const int count = gen8_pte_count(start, length);
- if (px_base(pt) == px_base(&vm->scratch[1])) {
+ if (!pt) {
spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc);
@@ -1074,7 +1057,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
fill_px(pt, vm->scratch[0].encode);
spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch[1]) {
+ if (!pd->entry[pde]) {
set_pd_entry(pd, pde, pt);
} else {
alloc = pt;
@@ -1106,7 +1089,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
spin_lock(&pdp->lock);
gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- if (px_base(pd) == px_base(&vm->scratch[2])) {
+ if (!pd) {
spin_unlock(&pdp->lock);
pd = fetch_and_zero(&alloc);
@@ -1117,10 +1100,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
goto unwind;
}
- init_pd(pd, &vm->scratch[1]);
+ fill_px(pd, vm->scratch[1].encode);
spin_lock(&pdp->lock);
- if (pdp->entry[pdpe] == &vm->scratch[2]) {
+ if (!pdp->entry[pdpe]) {
set_pd_entry(pdp, pdpe, pd);
} else {
alloc = pd;
@@ -1170,7 +1153,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
spin_lock(&pml4->lock);
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- if (px_base(pdp) == px_base(&vm->scratch[3])) {
+ if (!pdp) {
spin_unlock(&pml4->lock);
pdp = fetch_and_zero(&alloc);
@@ -1181,10 +1164,10 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
goto unwind;
}
- init_pd(pdp, &vm->scratch[2]);
+ fill_px(pdp, vm->scratch[2].encode);
spin_lock(&pml4->lock);
- if (pml4->entry[pml4e] == &vm->scratch[3]) {
+ if (!pml4->entry[pml4e]) {
set_pd_entry(pml4, pml4e, pdp);
} else {
alloc = pdp;
@@ -1520,7 +1503,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
if (IS_ERR(pd))
goto unwind;
- init_pd(pd, &vm->scratch[1]);
+ fill_px(pd, vm->scratch[1].encode);
set_pd_entry(pdp, pdpe, pd);
}
@@ -1551,46 +1534,20 @@ static void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt)
ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;
}
-static void init_pd_n(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- struct i915_page_scratch *scratch,
- const unsigned int entries)
-{
- u64 * const vaddr = kmap_atomic_px(pd);
-
- memset64(vaddr, scratch->encode, entries);
- kunmap_atomic(vaddr);
-
- memset_p(pd->entry, scratch, entries);
-}
-
static struct i915_page_directory *
gen8_alloc_top_pd(struct i915_address_space *vm)
{
struct i915_page_directory *pd;
- if (i915_vm_is_4lvl(vm)) {
- pd = alloc_pd(vm);
- if (!IS_ERR(pd))
- init_pd(pd, &vm->scratch[3]);
+ GEM_BUG_ON((vm->total - 1) >> gen8_pd_shift(vm->top + 1));
+ pd = alloc_pd(vm);
+ if (IS_ERR(pd))
return pd;
- }
-
- /* 3lvl */
- pd = __alloc_pd();
- if (!pd)
- return ERR_PTR(-ENOMEM);
-
- pd->entry[GEN8_3LVL_PDPES] = NULL;
-
- if (unlikely(setup_page_dma(vm, px_base(pd)))) {
- kfree(pd);
- return ERR_PTR(-ENOMEM);
- }
-
- init_pd_n(vm, pd, &vm->scratch[2], GEN8_3LVL_PDPES);
+ fill_page_dma(px_base(pd),
+ vm->scratch[vm->top].encode,
+ vm->total >> gen8_pd_shift(vm->top));
return pd;
}
@@ -1747,8 +1704,6 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
const unsigned int count = min(num_entries, GEN6_PTES - pte);
gen6_pte_t *vaddr;
- GEM_BUG_ON(px_base(pt) == px_base(&vm->scratch[1]));
-
num_entries -= count;
GEM_BUG_ON(count > atomic_read(&pt->used));
@@ -1784,8 +1739,6 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
struct sgt_dma iter = sgt_dma(vma);
gen6_pte_t *vaddr;
- GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch[1]);
-
vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
do {
vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
@@ -1829,7 +1782,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
gen6_for_each_pde(pt, pd, start, length, pde) {
const unsigned int count = gen6_pte_count(start, length);
- if (px_base(pt) == px_base(&vm->scratch[1])) {
+ if (!pt) {
spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc);
@@ -1843,7 +1796,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
fill32_px(pt, vm->scratch[0].encode);
spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch[1]) {
+ if (!pd->entry[pde]) {
pd->entry[pde] = pt;
if (i915_vma_is_bound(ppgtt->vma,
I915_VMA_GLOBAL_BIND)) {
@@ -1879,7 +1832,6 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
{
struct i915_address_space * const vm = &ppgtt->base.vm;
- struct i915_page_directory * const pd = ppgtt->base.pd;
int ret;
ret = setup_scratch_page(vm, __GFP_HIGHMEM);
@@ -1896,8 +1848,6 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
}
fill32_px(&vm->scratch[1], vm->scratch[0].encode);
- memset_p(pd->entry, &vm->scratch[1], I915_PDES);
-
return 0;
}
@@ -1907,9 +1857,12 @@ static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
struct i915_page_table *pt;
u32 pde;
- gen6_for_all_pdes(pt, pd, pde)
- if (px_base(pt) != px_base(&ppgtt->base.vm.scratch[1]))
- free_px(&ppgtt->base.vm, pt);
+ gen6_for_all_pdes(pt, pd, pde) {
+ if (!pt)
+ continue;
+
+ free_px(&ppgtt->base.vm, pt);
+ }
}
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
@@ -1966,8 +1919,6 @@ static void pd_vma_unbind(struct i915_vma *vma)
{
struct gen6_ppgtt *ppgtt = vma->private;
struct i915_page_directory * const pd = ppgtt->base.pd;
- struct i915_page_dma * const scratch =
- px_base(&ppgtt->base.vm.scratch[1]);
struct i915_page_table *pt;
unsigned int pde;
@@ -1976,11 +1927,11 @@ static void pd_vma_unbind(struct i915_vma *vma)
/* Free all no longer used page tables */
gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
- if (px_base(pt) == scratch || atomic_read(&pt->used))
+ if (!pt || atomic_read(&pt->used))
continue;
free_px(&ppgtt->base.vm, pt);
- pd->entry[pde] = scratch;
+ pd->entry[pde] = NULL;
}
ppgtt->scan_for_unused_pt = false;
--
2.20.1
More information about the Intel-gfx-trybot
mailing list