[PATCH 3/3] drm/i915/gtt: Fully initialise 32b page directory
Chris Wilson
chris at chris-wilson.co.uk
Sat Jul 6 12:37:22 UTC 2019
Fill the unused portion of the 32b page directories with scratch so that
we can use the same code throughout.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem_gtt.c | 862 +++++++++++-----------------
drivers/gpu/drm/i915/i915_gem_gtt.h | 100 +---
2 files changed, 347 insertions(+), 615 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 8c9bb4a9235f..011753b781e1 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -652,8 +652,8 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
if (unlikely(!IS_ALIGNED(addr, size)))
goto unmap_page;
- vm->scratch_page.page = page;
- vm->scratch_page.daddr = addr;
+ vm->scratch[0].page = page;
+ vm->scratch[0].daddr = addr;
vm->scratch_order = order;
return 0;
@@ -672,8 +672,8 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
static void cleanup_scratch_page(struct i915_address_space *vm)
{
- struct i915_page_dma *p = &vm->scratch_page;
- int order = vm->scratch_order;
+ struct i915_page_dma *p = &vm->scratch[0];
+ unsigned int order = vm->scratch_order;
dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT,
PCI_DMA_BIDIRECTIONAL);
@@ -698,12 +698,6 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
return pt;
}
-static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
-{
- cleanup_page_dma(vm, &pt->base);
- kfree(pt);
-}
-
static struct i915_page_directory *__alloc_pd(void)
{
struct i915_page_directory *pd;
@@ -734,18 +728,17 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
return pd;
}
-static void free_pd(struct i915_address_space *vm,
- struct i915_page_directory *pd)
+static void free_pd(struct i915_address_space *vm, struct i915_page_dma *pd)
{
- cleanup_page_dma(vm, px_base(pd));
+ cleanup_page_dma(vm, pd);
kfree(pd);
}
static void init_pd(struct i915_page_directory *pd,
struct i915_page_dma *scratch)
{
- fill_px(pd, gen8_pde_encode(scratch->daddr, I915_CACHE_LLC));
- memset_p(pd->entry, scratch, 512);
+ fill_px(&pd->pt, gen8_pde_encode(scratch->daddr, I915_CACHE_LLC));
+ memset_p(pd->entry, scratch, I915_PDES);
}
static inline void
@@ -765,7 +758,7 @@ __set_pd_entry(struct i915_page_directory * const pd,
struct i915_page_dma * const to,
u64 (*encode)(const dma_addr_t, const enum i915_cache_level))
{
- GEM_BUG_ON(atomic_read(px_used(pd)) > 512);
+ GEM_BUG_ON(atomic_read(px_used(pd)) > I915_PDES);
atomic_inc(px_used(pd));
pd->entry[pde] = to;
@@ -778,7 +771,7 @@ __clear_pd_entry(struct i915_page_directory * const pd,
struct i915_page_dma * const to,
u64 (*encode)(const dma_addr_t, const enum i915_cache_level))
{
- GEM_BUG_ON(atomic_read(px_used(pd)) == 0);
+ GEM_BUG_ON(!atomic_read(px_used(pd)));
write_dma_entry(px_base(pd), pde, encode(to->daddr, I915_CACHE_LLC));
pd->entry[pde] = to;
@@ -786,7 +779,7 @@ __clear_pd_entry(struct i915_page_directory * const pd,
}
#define set_pd_entry(pd, pde, to) \
- __set_pd_entry((pd), (pde), px_base(to), gen8_pde_encode)
+ __set_pd_entry((pd), (pde), &(to)->base, gen8_pde_encode)
#define clear_pd_entry(pd, pde, to) \
__clear_pd_entry((pd), (pde), (to), gen8_pde_encode)
@@ -820,90 +813,254 @@ static void mark_tlbs_dirty(struct i915_ppgtt *ppgtt)
ppgtt->pd_dirty_engines = ALL_ENGINES;
}
-/* Removes entries from a single page table, releasing it if it's empty.
- * Caller can use the return value to update higher-level entries.
- */
-static void gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
- struct i915_page_table *pt,
- u64 start, u64 length)
+static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
{
- const unsigned int num_entries = gen8_pte_count(start, length);
- gen8_pte_t *vaddr;
+ struct i915_address_space *vm = &ppgtt->vm;
+ struct drm_i915_private *dev_priv = vm->i915;
+ enum vgt_g2v_type msg;
+ int i;
- vaddr = kmap_atomic_px(pt);
- memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
- kunmap_atomic(vaddr);
+ if (create)
+ atomic_inc(px_used(ppgtt->pd)); /* never remove */
+ else
+ atomic_dec(px_used(ppgtt->pd));
+
+ if (i915_vm_is_4lvl(vm)) {
+ const u64 daddr = px_dma(ppgtt->pd);
+
+ I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
+ I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
+
+ msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
+ VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
+ } else {
+ for (i = 0; i < GEN8_3LVL_PDPES; i++) {
+ const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
+
+ I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
+ I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
+ }
+
+ msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
+ VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
+ }
- GEM_BUG_ON(num_entries > atomic_read(&pt->used));
+ I915_WRITE(vgtif_reg(g2v_notify), msg);
- atomic_sub(num_entries, &pt->used);
+ return 0;
}
-static void gen8_ppgtt_clear_pd(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- u64 start, u64 length)
+#define pd_lvl(lvl) ((lvl) * ilog2(I915_PDES))
+#define gen8_pd_shift(lvl) (GEN8_PTE_SHIFT + pd_lvl(lvl))
+#define gen8_pd_size(lvl) BIT_ULL(gen8_pd_shift(lvl))
+#define gen8_pte_count(s, l) i915_pte_count((s), (l), gen8_pd_shift(1))
+#define gen8_pde_index(a, l) i915_pde_index((a), gen8_pd_shift(l))
+
+#define as_pd(x) container_of((x), typeof(struct i915_page_directory), pt)
+
+static inline unsigned int
+gen8_pde_range(u64 addr, u64 length, int lvl, unsigned int *idx)
{
- struct i915_page_table *pt;
- u32 pde;
+ const int shift = gen8_pd_shift(lvl);
+ const u64 mask = ~0ull << gen8_pd_shift(lvl + 1);
+ const u64 end = addr + length;
- gen8_for_each_pde(pt, pd, start, length, pde) {
- GEM_BUG_ON(px_base(pt) == &vm->scratch_pt);
+ *idx = i915_pde_index(addr, shift);
+ if ((addr & mask) != (end & mask))
+ return I915_PDES - *idx;
- atomic_inc(&pt->used);
- gen8_ppgtt_clear_pt(vm, pt, start, length);
- if (release_pd_entry(pd, pde, pt, &vm->scratch_pt))
- free_pt(vm, pt);
+ return i915_pde_index(end + (~mask >> pd_lvl(1)), shift) - *idx;
+}
+
+static void __gen8_ppgtt_cleanup(struct i915_address_space *vm,
+ struct i915_page_directory *pd,
+ int count, int lvl)
+{
+ void **pde = pd->entry;
+
+ do {
+ if (*pde == &vm->scratch[lvl])
+ continue;
+
+ if (lvl == 1)
+ free_pd(vm, *pde);
+ else
+ __gen8_ppgtt_cleanup(vm, *pde, I915_PDES, lvl - 1);
+ } while (pde++, --count);
+
+ free_pd(vm, px_base(pd));
+}
+
+static void gen8_free_scratch(struct i915_address_space *vm)
+{
+ int i;
+
+ if (!vm->scratch[0].daddr) /* set to 0 on clones */
+ return;
+
+ for (i = ARRAY_SIZE(vm->scratch); --i; ) {
+ if (vm->scratch[i].daddr)
+ cleanup_page_dma(vm, &vm->scratch[i]);
}
+
+ cleanup_scratch_page(vm);
}
-/* Removes entries from a single page dir pointer, releasing it if it's empty.
- * Caller can use the return value to update higher-level entries
- */
-static void gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
- struct i915_page_directory * const pdp,
- u64 start, u64 length)
+static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
{
- struct i915_page_directory *pd;
- unsigned int pdpe;
+ struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
- gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- GEM_BUG_ON(px_base(pd) == &vm->scratch_pd);
+ if (intel_vgpu_active(vm->i915))
+ gen8_ppgtt_notify_vgt(ppgtt, false);
- atomic_inc(px_used(pd));
- gen8_ppgtt_clear_pd(vm, pd, start, length);
- if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))
- free_pd(vm, pd);
+ __gen8_ppgtt_cleanup(vm, ppgtt->pd,
+ vm->total >> gen8_pd_shift(ppgtt->top),
+ ppgtt->top);
+ gen8_free_scratch(vm);
+}
+
+static void __gen8_ppgtt_clear(struct i915_address_space *vm,
+ struct i915_page_directory * const pd,
+ u64 *start, u64 *length, int lvl)
+{
+ struct i915_page_dma * const scratch = &vm->scratch[lvl];
+ unsigned int idx, len;
+
+ len = gen8_pde_range(*start, *length, lvl, &idx);
+ GEM_BUG_ON(len > atomic_read(px_used(pd)));
+
+ if (!lvl) {
+ u64 *vaddr;
+
+ vaddr = kmap_atomic_px(pd);
+ memset64(vaddr + idx, vm->scratch_pte, len);
+ kunmap_atomic(vaddr);
+
+ atomic_sub(len, px_used(pd));
+ *start += len << gen8_pd_shift(lvl);
+ *length -= len << gen8_pd_shift(lvl);
+ return;
}
+
+ do {
+ struct i915_page_table *pt = pd->entry[idx];
+
+ GEM_BUG_ON(pd->entry[idx] == scratch);
+
+ if (*length >= gen8_pd_size(lvl) &&
+ atomic_read(&pt->used) == I915_PDES) {
+ clear_pd_entry(pd, idx, scratch);
+ __gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl - 1);
+ *start += gen8_pd_size(lvl);
+ *length -= gen8_pd_size(lvl);
+ continue;
+ }
+
+ atomic_inc(&pt->used);
+ __gen8_ppgtt_clear(vm, as_pd(pt), start, length, lvl - 1);
+ if (release_pd_entry(pd, idx, pt, scratch))
+ free_pd(vm, px_base(pt));
+ } while (idx++, --len);
}
-static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
- u64 start, u64 length)
+static void gen8_ppgtt_clear(struct i915_address_space *vm,
+ u64 start, u64 length)
{
- gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length);
+ __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,
+ &start, &length, i915_vm_to_ppgtt(vm)->top);
+ GEM_BUG_ON(length);
}
-/* Removes entries from a single pml4.
- * This is the top-level structure in 4-level page tables used on gen8+.
- * Empty entries are always scratch pml4e.
- */
-static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
- u64 start, u64 length)
+static int __gen8_ppgtt_alloc(struct i915_address_space *vm,
+ struct i915_page_directory *pd,
+ u64 *start, u64 *length, int lvl)
{
- struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
- struct i915_page_directory * const pml4 = ppgtt->pd;
- struct i915_page_directory *pdp;
- unsigned int pml4e;
+ struct i915_page_table *pt, *alloc = NULL;
+ unsigned int idx, len;
+ int ret = 0;
+
+ GEM_BUG_ON(!px_used(pd)); /* Our parents must be pinned! */
+
+ len = gen8_pde_range(*start, *length, lvl, &idx);
+ GEM_BUG_ON(!len);
+
+ spin_lock(&pd->lock);
+ do {
+ GEM_BUG_ON(!atomic_read(px_used(pd)));
+ pt = pd->entry[idx];
+
+ if (&pt->base == &vm->scratch[lvl]) {
+ spin_unlock(&pd->lock);
+
+ pt = fetch_and_zero(&alloc);
+ if (!pt)
+ pt = (lvl > 1) ? &alloc_pd(vm)->pt : alloc_pt(vm);
+ if (IS_ERR(pt)) {
+ ret = PTR_ERR(pt);
+ goto out;
+ }
- GEM_BUG_ON(!i915_vm_is_4lvl(vm));
+ if (lvl > 1)
+ init_pd(as_pd(pt), &vm->scratch[lvl - 1]);
+ else if (len < I915_PDES || intel_vgpu_active(vm->i915))
+ fill_px(pt, vm->scratch_pte);
- gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- GEM_BUG_ON(px_base(pdp) == &vm->scratch_pdp);
+ spin_lock(&pd->lock);
+ if (pd->entry[idx] == &vm->scratch[lvl]) {
+ set_pd_entry(pd, idx, pt);
+ } else {
+ alloc = pt;
+ pt = pd->entry[idx];
+ }
+ }
- atomic_inc(px_used(pdp));
- gen8_ppgtt_clear_pdp(vm, pdp, start, length);
- if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))
- free_pd(vm, pdp);
+ if (lvl > 1) {
+ atomic_inc(&pt->used);
+ spin_unlock(&pd->lock);
+
+ ret = __gen8_ppgtt_alloc(vm, as_pd(pt),
+ start, length, lvl - 1);
+ if (unlikely(ret))
+ goto release;
+
+ spin_lock(&pd->lock);
+ atomic_dec(&pt->used);
+ GEM_BUG_ON(!atomic_read(&pt->used));
+ } else {
+ unsigned int count = gen8_pte_count(*start, *length);
+
+ atomic_add(count, &pt->used);
+ *start += count << GEN8_PTE_SHIFT;
+ *length -= count << GEN8_PTE_SHIFT;
+ }
+ } while (idx++, --len);
+ spin_unlock(&pd->lock);
+ goto out;
+
+release:
+ if (release_pd_entry(pd, idx, pt, &vm->scratch[lvl]))
+ free_pd(vm, px_base(pt));
+out:
+ if (alloc)
+ free_pd(vm, &alloc->base);
+ return ret;
+}
+
+static int gen8_ppgtt_alloc(struct i915_address_space *vm,
+ u64 start, u64 length)
+{
+ u64 from = start;
+ int err;
+
+ err = __gen8_ppgtt_alloc(vm, i915_vm_to_ppgtt(vm)->pd,
+ &start, &length, i915_vm_to_ppgtt(vm)->top);
+ if (!err) {
+ GEM_BUG_ON(length);
+ return 0;
}
+
+ gen8_ppgtt_clear(vm, from, start - from);
+ return err;
}
static inline struct sgt_dma {
@@ -915,47 +1072,33 @@ static inline struct sgt_dma {
return (struct sgt_dma) { sg, addr, addr + sg->length };
}
-struct gen8_insert_pte {
- u16 pml4e;
- u16 pdpe;
- u16 pde;
- u16 pte;
-};
-
-static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
-{
- return (struct gen8_insert_pte) {
- gen8_pml4e_index(start),
- gen8_pdpe_index(start),
- gen8_pde_index(start),
- gen8_pte_index(start),
- };
-}
+#define pde_idx(x, lvl) (((x) >> pd_lvl(lvl)) & I915_PDE_MASK)
-static __always_inline bool
+static __always_inline u64
gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
struct i915_page_directory *pdp,
struct sgt_dma *iter,
- struct gen8_insert_pte *idx,
+ u64 idx,
enum i915_cache_level cache_level,
u32 flags)
{
struct i915_page_directory *pd;
const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
gen8_pte_t *vaddr;
- bool ret;
- GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
- pd = i915_pd_entry(pdp, idx->pdpe);
- vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
+ pd = i915_pd_entry(pdp, pde_idx(idx, 2));
+ GEM_BUG_ON(px_base(pd) == &ppgtt->vm.scratch[2]);
+ GEM_BUG_ON(pd->entry[pde_idx(idx, 1)] == &ppgtt->vm.scratch[1]);
+
+ vaddr = kmap_atomic_px(i915_pt_entry(pd, pde_idx(idx, 1)));
do {
- vaddr[idx->pte] = pte_encode | iter->dma;
+ vaddr[pde_idx(idx, 0)] = pte_encode | iter->dma;
iter->dma += I915_GTT_PAGE_SIZE;
if (iter->dma >= iter->max) {
iter->sg = __sg_next(iter->sg);
if (!iter->sg) {
- ret = false;
+ idx = 0;
break;
}
@@ -963,30 +1106,24 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
iter->max = iter->dma + iter->sg->length;
}
- if (++idx->pte == GEN8_PTES) {
- idx->pte = 0;
-
- if (++idx->pde == I915_PDES) {
- idx->pde = 0;
-
+ if (pde_idx(++idx, 0) == 0) {
+ if (pde_idx(idx, 1) == 0) {
/* Limited by sg length for 3lvl */
- if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
- idx->pdpe = 0;
- ret = true;
+ if (pde_idx(idx, 2) == 0)
break;
- }
- GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
- pd = pdp->entry[idx->pdpe];
+ pd = pdp->entry[pde_idx(idx, 2)];
+ GEM_BUG_ON(px_base(pd) == &ppgtt->vm.scratch[2]);
}
kunmap_atomic(vaddr);
- vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
+ GEM_BUG_ON(pd->entry[pde_idx(idx, 1)] == &ppgtt->vm.scratch[1]);
+ vaddr = kmap_atomic_px(i915_pt_entry(pd, pde_idx(idx, 1)));
}
} while (1);
kunmap_atomic(vaddr);
- return ret;
+ return idx;
}
static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
@@ -996,9 +1133,9 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
{
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
struct sgt_dma iter = sgt_dma(vma);
- struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
- gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,
+ gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter,
+ vma->node.start >> GEN8_PTE_SHIFT,
cache_level, flags);
vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
@@ -1015,38 +1152,36 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
dma_addr_t rem = iter->sg->length;
do {
- struct gen8_insert_pte idx = gen8_insert_pte(start);
struct i915_page_directory *pdp =
- i915_pdp_entry(pml4, idx.pml4e);
- struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);
+ i915_pd_entry(pml4, gen8_pde_index(start, 3));
+ struct i915_page_directory *pd =
+ i915_pd_entry(pdp, gen8_pde_index(start, 2));
+ gen8_pte_t encode = pte_encode;
unsigned int page_size;
bool maybe_64K = false;
- gen8_pte_t encode = pte_encode;
gen8_pte_t *vaddr;
- u16 index, max;
+ u16 index;
if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&
IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
- rem >= I915_GTT_PAGE_SIZE_2M && !idx.pte) {
- index = idx.pde;
- max = I915_PDES;
- page_size = I915_GTT_PAGE_SIZE_2M;
-
+ rem >= I915_GTT_PAGE_SIZE_2M && !gen8_pde_index(start, 0)) {
+ index = gen8_pde_index(start, 1);
encode |= GEN8_PDE_PS_2M;
+ page_size = SZ_2M;
vaddr = kmap_atomic_px(pd);
} else {
- struct i915_page_table *pt = i915_pt_entry(pd, idx.pde);
+ struct i915_page_table *pt =
+ i915_pt_entry(pd, gen8_pde_index(start, 1));
- index = idx.pte;
- max = GEN8_PTES;
- page_size = I915_GTT_PAGE_SIZE;
+ index = gen8_pde_index(start, 0);
+ page_size = SZ_4K;
if (!index &&
vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
(IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
- rem >= (max - index) * I915_GTT_PAGE_SIZE))
+ rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE))
maybe_64K = true;
vaddr = kmap_atomic_px(pt);
@@ -1068,16 +1203,16 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
iter->dma = sg_dma_address(iter->sg);
iter->max = iter->dma + rem;
- if (maybe_64K && index < max &&
+ if (maybe_64K && index < I915_PDES &&
!(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
(IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
- rem >= (max - index) * I915_GTT_PAGE_SIZE)))
+ rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)))
maybe_64K = false;
if (unlikely(!IS_ALIGNED(iter->dma, page_size)))
break;
}
- } while (rem >= page_size && index < max);
+ } while (rem >= page_size && index < I915_PDES);
kunmap_atomic(vaddr);
@@ -1088,13 +1223,13 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
* enough padding.
*/
if (maybe_64K &&
- (index == max ||
+ (index == I915_PDES ||
(i915_vm_has_scratch_64K(vma->vm) &&
!iter->sg && IS_ALIGNED(vma->node.start +
vma->node.size,
I915_GTT_PAGE_SIZE_2M)))) {
vaddr = kmap_atomic_px(pd);
- vaddr[idx.pde] |= GEN8_PDE_IPS_64K;
+ vaddr[gen8_pde_index(start - SZ_2M, 1)] |= GEN8_PDE_IPS_64K;
kunmap_atomic(vaddr);
page_size = I915_GTT_PAGE_SIZE_64K;
@@ -1111,8 +1246,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
u16 i;
encode = vma->vm->scratch_pte;
- vaddr = kmap_atomic_px(i915_pt_entry(pd,
- idx.pde));
+ vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pde_index(start - SZ_2M, 1)));
for (i = 1; i < index; i += 16)
memset64(vaddr + i, encode, 15);
@@ -1138,32 +1272,23 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
gen8_ppgtt_insert_huge_entries(vma, pml4, &iter, cache_level,
flags);
} else {
- struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
+ u64 idx = vma->node.start >> GEN8_PTE_SHIFT;
- while (gen8_ppgtt_insert_pte_entries(ppgtt,
- i915_pdp_entry(pml4, idx.pml4e++),
- &iter, &idx, cache_level,
- flags))
- GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
+ while ((idx = gen8_ppgtt_insert_pte_entries(ppgtt,
+ i915_pd_entry(pml4, pde_idx(idx, 3)),
+ &iter, idx, cache_level,
+ flags)))
+ ;
vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
}
}
-static void gen8_free_page_tables(struct i915_address_space *vm,
- struct i915_page_directory *pd)
-{
- int i;
-
- for (i = 0; i < I915_PDES; i++) {
- if (pd->entry[i] != &vm->scratch_pt)
- free_pt(vm, pd->entry[i]);
- }
-}
-
static int gen8_init_scratch(struct i915_address_space *vm)
{
+ u64 pte;
int ret;
+ int i;
/*
* If everybody agrees to not to write into the scratch page,
@@ -1177,10 +1302,8 @@ static int gen8_init_scratch(struct i915_address_space *vm)
GEM_BUG_ON(!clone->has_read_only);
vm->scratch_order = clone->scratch_order;
- vm->scratch_pte = clone->scratch_pte;
- vm->scratch_pt = clone->scratch_pt;
- vm->scratch_pd = clone->scratch_pd;
- vm->scratch_pdp = clone->scratch_pdp;
+ memcpy(vm->scratch, clone->scratch, sizeof(vm->scratch));
+ vm->scratch[0].daddr = 0;
return 0;
}
@@ -1189,339 +1312,51 @@ static int gen8_init_scratch(struct i915_address_space *vm)
return ret;
vm->scratch_pte =
- gen8_pte_encode(vm->scratch_page.daddr,
+ gen8_pte_encode(vm->scratch[0].daddr,
I915_CACHE_LLC,
vm->has_read_only);
- if (unlikely(setup_page_dma(vm, &vm->scratch_pt))) {
- ret = -ENOMEM;
- goto free_scratch_page;
- }
- fill_page_dma(&vm->scratch_pt, vm->scratch_pte);
+ pte = vm->scratch_pte;
+ for (i = 1; i <= i915_vm_to_ppgtt(vm)->top; i++) {
+ if (unlikely(setup_page_dma(vm, &vm->scratch[i])))
+ goto free_scratch;
- if (unlikely(setup_page_dma(vm, &vm->scratch_pd))) {
- ret = -ENOMEM;
- goto free_pt;
- }
- fill_page_dma(&vm->scratch_pd,
- gen8_pde_encode(vm->scratch_pd.daddr, I915_CACHE_LLC));
-
- if (i915_vm_is_4lvl(vm)) {
- if (unlikely(setup_page_dma(vm, &vm->scratch_pdp))) {
- ret = -ENOMEM;
- goto free_pd;
- }
- fill_page_dma(&vm->scratch_pdp,
- gen8_pde_encode(vm->scratch_pdp.daddr,
- I915_CACHE_LLC));
+ fill_page_dma(&vm->scratch[i], pte);
+ pte = gen8_pde_encode(vm->scratch[i].daddr, I915_CACHE_LLC);
}
return 0;
-free_pd:
- cleanup_page_dma(vm, &vm->scratch_pd);
-free_pt:
- cleanup_page_dma(vm, &vm->scratch_pt);
-free_scratch_page:
- cleanup_scratch_page(vm);
-
- return ret;
-}
-
-static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
-{
- struct i915_address_space *vm = &ppgtt->vm;
- struct drm_i915_private *dev_priv = vm->i915;
- enum vgt_g2v_type msg;
- int i;
-
- if (create)
- atomic_inc(px_used(ppgtt->pd)); /* never remove */
- else
- atomic_dec(px_used(ppgtt->pd));
-
- if (i915_vm_is_4lvl(vm)) {
- const u64 daddr = px_dma(ppgtt->pd);
-
- I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
- I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
-
- msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
- VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
- } else {
- for (i = 0; i < GEN8_3LVL_PDPES; i++) {
- const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
-
- I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
- I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
- }
-
- msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
- VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
- }
-
- I915_WRITE(vgtif_reg(g2v_notify), msg);
-
- return 0;
-}
-
-static void gen8_free_scratch(struct i915_address_space *vm)
-{
- if (!vm->scratch_page.daddr) /* set to 0 on clones */
- return;
-
- if (vm->scratch_pdp.daddr)
- cleanup_page_dma(vm, &vm->scratch_pdp);
- if (vm->scratch_pd.daddr)
- cleanup_page_dma(vm, &vm->scratch_pd);
- if (vm->scratch_pt.daddr)
- cleanup_page_dma(vm, &vm->scratch_pt);
-
- cleanup_scratch_page(vm);
-}
-
-static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
- struct i915_page_directory *pdp)
-{
- const unsigned int pdpes = i915_pdpes_per_pdp(vm);
- int i;
-
- for (i = 0; i < pdpes; i++) {
- if (pdp->entry[i] == &vm->scratch_pd)
- continue;
-
- gen8_free_page_tables(vm, pdp->entry[i]);
- free_pd(vm, pdp->entry[i]);
- }
-
- free_pd(vm, pdp);
-}
-
-static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
-{
- struct i915_page_directory * const pml4 = ppgtt->pd;
- int i;
-
- for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
- struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
-
- if (px_base(pdp) == &ppgtt->vm.scratch_pdp)
- continue;
-
- gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
- }
-
- free_pd(&ppgtt->vm, pml4);
-}
-
-static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
-{
- struct drm_i915_private *i915 = vm->i915;
- struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-
- if (intel_vgpu_active(i915))
- gen8_ppgtt_notify_vgt(ppgtt, false);
-
- if (i915_vm_is_4lvl(vm))
- gen8_ppgtt_cleanup_4lvl(ppgtt);
- else
- gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);
-
+free_scratch:
gen8_free_scratch(vm);
-}
-
-static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- u64 start, u64 length)
-{
- struct i915_page_table *pt, *alloc = NULL;
- u64 from = start;
- unsigned int pde;
- int ret = 0;
-
- spin_lock(&pd->lock);
- gen8_for_each_pde(pt, pd, start, length, pde) {
- const int count = gen8_pte_count(start, length);
-
- if (px_base(pt) == &vm->scratch_pt) {
- spin_unlock(&pd->lock);
-
- pt = fetch_and_zero(&alloc);
- if (!pt)
- pt = alloc_pt(vm);
- if (IS_ERR(pt)) {
- ret = PTR_ERR(pt);
- goto unwind;
- }
-
- if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
- fill_px(pt, vm->scratch_pte);
-
- spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch_pt) {
- set_pd_entry(pd, pde, pt);
- } else {
- alloc = pt;
- pt = pd->entry[pde];
- }
- }
-
- atomic_add(count, &pt->used);
- }
- spin_unlock(&pd->lock);
- goto out;
-
-unwind:
- gen8_ppgtt_clear_pd(vm, pd, from, start - from);
-out:
- if (alloc)
- free_pt(vm, alloc);
- return ret;
-}
-
-static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
- struct i915_page_directory *pdp,
- u64 start, u64 length)
-{
- struct i915_page_directory *pd, *alloc = NULL;
- u64 from = start;
- unsigned int pdpe;
- int ret = 0;
-
- spin_lock(&pdp->lock);
- gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- if (px_base(pd) == &vm->scratch_pd) {
- spin_unlock(&pdp->lock);
-
- pd = fetch_and_zero(&alloc);
- if (!pd)
- pd = alloc_pd(vm);
- if (IS_ERR(pd)) {
- ret = PTR_ERR(pd);
- goto unwind;
- }
-
- init_pd(pd, &vm->scratch_pt);
-
- spin_lock(&pdp->lock);
- if (pdp->entry[pdpe] == &vm->scratch_pd) {
- set_pd_entry(pdp, pdpe, pd);
- } else {
- alloc = pd;
- pd = pdp->entry[pdpe];
- }
- }
- atomic_inc(px_used(pd));
- spin_unlock(&pdp->lock);
-
- ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
- if (unlikely(ret))
- goto unwind_pd;
-
- spin_lock(&pdp->lock);
- atomic_dec(px_used(pd));
- }
- spin_unlock(&pdp->lock);
- goto out;
-
-unwind_pd:
- if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))
- free_pd(vm, pd);
-unwind:
- gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
-out:
- if (alloc)
- free_pd(vm, alloc);
- return ret;
-}
-
-static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
- u64 start, u64 length)
-{
- return gen8_ppgtt_alloc_pdp(vm,
- i915_vm_to_ppgtt(vm)->pd, start, length);
-}
-
-static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
- u64 start, u64 length)
-{
- struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
- struct i915_page_directory * const pml4 = ppgtt->pd;
- struct i915_page_directory *pdp, *alloc = NULL;
- u64 from = start;
- int ret = 0;
- u32 pml4e;
-
- spin_lock(&pml4->lock);
- gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
- if (px_base(pdp) == &vm->scratch_pdp) {
- spin_unlock(&pml4->lock);
-
- pdp = fetch_and_zero(&alloc);
- if (!pdp)
- pdp = alloc_pd(vm);
- if (IS_ERR(pdp)) {
- ret = PTR_ERR(pdp);
- goto unwind;
- }
-
- init_pd(pdp, &vm->scratch_pd);
-
- spin_lock(&pml4->lock);
- if (pml4->entry[pml4e] == &vm->scratch_pdp) {
- set_pd_entry(pml4, pml4e, pdp);
- } else {
- alloc = pdp;
- pdp = pml4->entry[pml4e];
- }
- }
- atomic_inc(px_used(pdp));
- spin_unlock(&pml4->lock);
-
- ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
- if (unlikely(ret))
- goto unwind_pdp;
-
- spin_lock(&pml4->lock);
- atomic_dec(px_used(pdp));
- }
- spin_unlock(&pml4->lock);
- goto out;
-
-unwind_pdp:
- if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))
- free_pd(vm, pdp);
-unwind:
- gen8_ppgtt_clear_4lvl(vm, from, start - from);
-out:
- if (alloc)
- free_pd(vm, alloc);
- return ret;
+ return -ENOMEM;
}
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
{
struct i915_address_space *vm = &ppgtt->vm;
- struct i915_page_directory *pdp = ppgtt->pd;
- struct i915_page_directory *pd;
- u64 start = 0, length = ppgtt->vm.total;
- u64 from = start;
- unsigned int pdpe;
+ struct i915_page_directory *pd = ppgtt->pd;
+ unsigned int idx;
+
+ GEM_BUG_ON(ppgtt->top != 2);
+ GEM_BUG_ON(GEN8_3LVL_PDPES != ppgtt->vm.total >> gen8_pd_shift(2));
- gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
- pd = alloc_pd(vm);
- if (IS_ERR(pd))
+ for (idx = 0; idx < GEN8_3LVL_PDPES; idx++) {
+ struct i915_page_directory *pde;
+
+ pde = alloc_pd(vm);
+ if (IS_ERR(pde))
goto unwind;
- init_pd(pd, &vm->scratch_pt);
- set_pd_entry(pdp, pdpe, pd);
+ init_pd(pde, &vm->scratch[1]);
+ set_pd_entry(pd, idx, &pde->pt);
}
return 0;
unwind:
- gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
- atomic_set(px_used(pdp), 0);
+ gen8_ppgtt_clear(vm, 0, idx << gen8_pd_shift(2));
+ atomic_set(px_used(pd), 0);
return -ENOMEM;
}
@@ -1540,49 +1375,33 @@ static void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt)
ppgtt->vm.vma_ops.unbind_vma = ppgtt_unbind_vma;
ppgtt->vm.vma_ops.set_pages = ppgtt_set_pages;
ppgtt->vm.vma_ops.clear_pages = clear_pages;
-}
-static void init_pd_n(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- struct i915_page_dma *to,
- const unsigned int entries)
-{
- const u64 daddr = gen8_pde_encode(to->daddr, I915_CACHE_LLC);
- u64 * const vaddr = kmap_atomic_px(pd);
-
- memset64(vaddr, daddr, entries);
- kunmap_atomic(vaddr);
-
- memset_p(pd->entry, to, entries);
+ ppgtt->top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;
}
static struct i915_page_directory *
gen8_alloc_top_pd(struct i915_address_space *vm)
{
+ const int top = i915_vm_to_ppgtt(vm)->top;
+ const int count = vm->total >> gen8_pd_shift(top);
struct i915_page_directory *pd;
+ void *vaddr;
- if (i915_vm_is_4lvl(vm)) {
- pd = alloc_pd(vm);
- if (!IS_ERR(pd))
- init_pd(pd, &vm->scratch_pdp);
+ GEM_BUG_ON((vm->total - 1) >> gen8_pd_shift(top + 1));
+ pd = alloc_pd(vm);
+ if (IS_ERR(pd))
return pd;
- }
- /* 3lvl */
- pd = __alloc_pd();
- if (!pd)
- return ERR_PTR(-ENOMEM);
-
- pd->entry[GEN8_3LVL_PDPES] = NULL;
-
- if (unlikely(setup_page_dma(vm, px_base(pd)))) {
- kfree(pd);
- return ERR_PTR(-ENOMEM);
- }
+ vaddr = kmap_atomic_px(pd);
+ memset64(vaddr,
+ gen8_pde_encode(px_dma(&vm->scratch[top]), I915_CACHE_LLC),
+ count);
+ kunmap_atomic(vaddr);
- init_pd_n(vm, pd, &vm->scratch_pd, GEN8_3LVL_PDPES);
+ memset_p(pd->entry, &vm->scratch[top], count);
+ atomic_inc(px_used(pd)); /* mark as pinned */
return pd;
}
@@ -1629,9 +1448,7 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
}
if (i915_vm_is_4lvl(&ppgtt->vm)) {
- ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
- ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;
} else {
if (intel_vgpu_active(i915)) {
err = gen8_preallocate_top_level_pdp(ppgtt);
@@ -1639,11 +1456,12 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
goto err_free_pd;
}
- ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;
- ppgtt->vm.clear_range = gen8_ppgtt_clear_3lvl;
}
+ ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
+ ppgtt->vm.clear_range = gen8_ppgtt_clear;
+
if (intel_vgpu_active(i915))
gen8_ppgtt_notify_vgt(ppgtt, true);
@@ -1652,7 +1470,7 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
return ppgtt;
err_free_pd:
- free_pd(&ppgtt->vm, ppgtt->pd);
+ free_pd(&ppgtt->vm, px_base(ppgtt->pd));
err_free_scratch:
gen8_free_scratch(&ppgtt->vm);
err_free:
@@ -1739,7 +1557,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
const unsigned int count = min(num_entries, GEN6_PTES - pte);
gen6_pte_t *vaddr;
- GEM_BUG_ON(px_base(pt) == &vm->scratch_pt);
+ GEM_BUG_ON(&pt->base == &vm->scratch[1]);
num_entries -= count;
@@ -1776,7 +1594,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
struct sgt_dma iter = sgt_dma(vma);
gen6_pte_t *vaddr;
- GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch_pt);
+ GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch[1]);
vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
do {
@@ -1821,7 +1639,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
gen6_for_each_pde(pt, pd, start, length, pde) {
const unsigned int count = gen6_pte_count(start, length);
- if (px_base(pt) == &vm->scratch_pt) {
+ if (&pt->base == &vm->scratch[1]) {
spin_unlock(&pd->lock);
pt = fetch_and_zero(&alloc);
@@ -1835,7 +1653,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
fill32_px(pt, vm->scratch_pte);
spin_lock(&pd->lock);
- if (pd->entry[pde] == &vm->scratch_pt) {
+ if (pd->entry[pde] == &vm->scratch[1]) {
pd->entry[pde] = pt;
if (i915_vma_is_bound(ppgtt->vma,
I915_VMA_GLOBAL_BIND)) {
@@ -1863,7 +1681,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
gen6_ppgtt_clear_range(vm, from, start - from);
out:
if (alloc)
- free_pt(vm, alloc);
+ free_pd(vm, &alloc->base);
intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
return ret;
}
@@ -1872,33 +1690,29 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
{
struct i915_address_space * const vm = &ppgtt->base.vm;
struct i915_page_directory * const pd = ppgtt->base.pd;
- struct i915_page_table *unused;
- u32 pde;
int ret;
ret = setup_scratch_page(vm, __GFP_HIGHMEM);
if (ret)
return ret;
- vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
+ vm->scratch_pte = vm->pte_encode(vm->scratch[0].daddr,
I915_CACHE_NONE,
PTE_READ_ONLY);
- if (unlikely(setup_page_dma(vm, &vm->scratch_pt))) {
+ if (unlikely(setup_page_dma(vm, &vm->scratch[1]))) {
cleanup_scratch_page(vm);
return -ENOMEM;
}
- fill_page_dma_32(&vm->scratch_pt, vm->scratch_pte);
-
- gen6_for_all_pdes(unused, pd, pde)
- pd->entry[pde] = &vm->scratch_pt;
+ fill_page_dma_32(&vm->scratch[1], vm->scratch_pte);
+ memset_p(pd->entry, &vm->scratch[1], I915_PDES);
return 0;
}
static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
{
- cleanup_page_dma(vm, &vm->scratch_pt);
+ cleanup_page_dma(vm, &vm->scratch[1]);
cleanup_scratch_page(vm);
}
@@ -1909,8 +1723,8 @@ static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
u32 pde;
gen6_for_all_pdes(pt, pd, pde)
- if (px_base(pt) != &ppgtt->base.vm.scratch_pt)
- free_pt(&ppgtt->base.vm, pt);
+ if (&pt->base != &ppgtt->base.vm.scratch[1])
+ free_pd(&ppgtt->base.vm, &pt->base);
}
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
@@ -1967,7 +1781,7 @@ static void pd_vma_unbind(struct i915_vma *vma)
{
struct gen6_ppgtt *ppgtt = vma->private;
struct i915_page_directory * const pd = ppgtt->base.pd;
- struct i915_page_dma * const scratch = &ppgtt->base.vm.scratch_pt;
+ struct i915_page_dma * const scratch = &ppgtt->base.vm.scratch[1];
struct i915_page_table *pt;
unsigned int pde;
@@ -1979,7 +1793,7 @@ static void pd_vma_unbind(struct i915_vma *vma)
if (atomic_read(&pt->used) || px_base(pt) == scratch)
continue;
- free_pt(&ppgtt->base.vm, pt);
+ free_pd(&ppgtt->base.vm, &pt->base);
pd->entry[pde] = scratch;
}
@@ -2973,7 +2787,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
}
ggtt->vm.scratch_pte =
- ggtt->vm.pte_encode(ggtt->vm.scratch_page.daddr,
+ ggtt->vm.pte_encode(ggtt->vm.scratch[0].daddr,
I915_CACHE_NONE, 0);
return 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 7019bb2b1941..917a8778736d 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -115,30 +115,19 @@ typedef u64 gen8_pte_t;
#define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0))
#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr)
-/* GEN8 32b style address is defined as a 3 level page table:
+/*
+ * GEN8 32b style address is defined as a 3 level page table:
* 31:30 | 29:21 | 20:12 | 11:0
* PDPE | PDE | PTE | offset
* The difference as compared to normal x86 3 level page table is the PDPEs are
* programmed via register.
- */
-#define GEN8_3LVL_PDPES 4
-#define GEN8_PDE_SHIFT 21
-#define GEN8_PDE_MASK 0x1ff
-#define GEN8_PTE_SHIFT 12
-#define GEN8_PTE_MASK 0x1ff
-#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t))
-
-/* GEN8 48b style address is defined as a 4 level page table:
+ *
+ * GEN8 48b style address is defined as a 4 level page table:
* 47:39 | 38:30 | 29:21 | 20:12 | 11:0
* PML4E | PDPE | PDE | PTE | offset
*/
-#define GEN8_PML4ES_PER_PML4 512
-#define GEN8_PML4E_SHIFT 39
-#define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1)
-#define GEN8_PDPE_SHIFT 30
-/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
- * tables */
-#define GEN8_PDPE_MASK 0x1ff
+#define GEN8_3LVL_PDPES 4
+#define GEN8_PTE_SHIFT 12
#define PPAT_UNCACHED (_PAGE_PWT | _PAGE_PCD)
#define PPAT_CACHED_PDE 0 /* WB LLC */
@@ -317,12 +306,9 @@ struct i915_address_space {
#define VM_CLASS_GGTT 0
#define VM_CLASS_PPGTT 1
+ struct i915_page_dma scratch[4];
u64 scratch_pte;
int scratch_order;
- struct i915_page_dma scratch_page;
- struct i915_page_dma scratch_pt;
- struct i915_page_dma scratch_pd;
- struct i915_page_dma scratch_pdp; /* GEN8+ & 48b PPGTT */
/**
* List of vma currently bound.
@@ -430,6 +416,7 @@ struct i915_ppgtt {
intel_engine_mask_t pd_dirty_engines;
struct i915_page_directory *pd;
+ int top;
};
struct gen6_ppgtt {
@@ -519,15 +506,6 @@ static inline u32 gen6_pde_index(u32 addr)
return i915_pde_index(addr, GEN6_PDE_SHIFT);
}
-static inline unsigned int
-i915_pdpes_per_pdp(const struct i915_address_space *vm)
-{
- if (i915_vm_is_4lvl(vm))
- return GEN8_PML4ES_PER_PML4;
-
- return GEN8_3LVL_PDPES;
-}
-
static inline struct i915_page_table *
i915_pt_entry(const struct i915_page_directory * const pd,
const unsigned short n)
@@ -542,70 +520,10 @@ i915_pd_entry(const struct i915_page_directory * const pdp,
return pdp->entry[n];
}
-static inline struct i915_page_directory *
-i915_pdp_entry(const struct i915_page_directory * const pml4,
- const unsigned short n)
-{
- return pml4->entry[n];
-}
-
-/* Equivalent to the gen6 version, For each pde iterates over every pde
- * between from start until start + length. On gen8+ it simply iterates
- * over every page directory entry in a page directory.
- */
-#define gen8_for_each_pde(pt, pd, start, length, iter) \
- for (iter = gen8_pde_index(start); \
- length > 0 && iter < I915_PDES && \
- (pt = i915_pt_entry(pd, iter), true); \
- ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \
- temp = min(temp - start, length); \
- start += temp, length -= temp; }), ++iter)
-
-#define gen8_for_each_pdpe(pd, pdp, start, length, iter) \
- for (iter = gen8_pdpe_index(start); \
- length > 0 && iter < i915_pdpes_per_pdp(vm) && \
- (pd = i915_pd_entry(pdp, iter), true); \
- ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \
- temp = min(temp - start, length); \
- start += temp, length -= temp; }), ++iter)
-
-#define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \
- for (iter = gen8_pml4e_index(start); \
- length > 0 && iter < GEN8_PML4ES_PER_PML4 && \
- (pdp = i915_pdp_entry(pml4, iter), true); \
- ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \
- temp = min(temp - start, length); \
- start += temp, length -= temp; }), ++iter)
-
-static inline u32 gen8_pte_index(u64 address)
-{
- return i915_pte_index(address, GEN8_PDE_SHIFT);
-}
-
-static inline u32 gen8_pde_index(u64 address)
-{
- return i915_pde_index(address, GEN8_PDE_SHIFT);
-}
-
-static inline u32 gen8_pdpe_index(u64 address)
-{
- return (address >> GEN8_PDPE_SHIFT) & GEN8_PDPE_MASK;
-}
-
-static inline u32 gen8_pml4e_index(u64 address)
-{
- return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK;
-}
-
-static inline u64 gen8_pte_count(u64 address, u64 length)
-{
- return i915_pte_count(address, length, GEN8_PDE_SHIFT);
-}
-
static inline dma_addr_t
i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n)
{
- return px_dma(i915_pdp_entry(ppgtt->pd, n));
+ return px_dma(i915_pt_entry(ppgtt->pd, n));
}
static inline struct i915_ggtt *
--
2.20.1
More information about the Intel-gfx-trybot
mailing list