[Intel-gfx] [PATCH 03/11] drm/i915/gtt: Reorder gen8 ppgtt free/clear/alloc
Mika Kuoppala
mika.kuoppala at linux.intel.com
Tue Jul 9 14:59:38 UTC 2019
Chris Wilson <chris at chris-wilson.co.uk> writes:
> In preparation for refactoring the free/clear/alloc, first move the code
> around so that we can avoid forward declarations in the next set of
> patches.
>
> Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Diff got funky at some point but after applying
end result looked good.
Reviewed-by: Mika Kuoppala <mika.kuoppala at linux.intel.com>
> ---
> drivers/gpu/drm/i915/i915_gem_gtt.c | 673 ++++++++++++++--------------
> 1 file changed, 337 insertions(+), 336 deletions(-)
>
> diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
> index 1fa93f56792e..da4db76ce054 100644
> --- a/drivers/gpu/drm/i915/i915_gem_gtt.c
> +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
> @@ -831,6 +831,104 @@ static void mark_tlbs_dirty(struct i915_ppgtt *ppgtt)
> ppgtt->pd_dirty_engines = ALL_ENGINES;
> }
>
> +static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
> +{
> + struct i915_address_space *vm = &ppgtt->vm;
> + struct drm_i915_private *dev_priv = vm->i915;
> + enum vgt_g2v_type msg;
> + int i;
> +
> + if (create)
> + atomic_inc(px_used(ppgtt->pd)); /* never remove */
> + else
> + atomic_dec(px_used(ppgtt->pd));
> +
> + if (i915_vm_is_4lvl(vm)) {
> + const u64 daddr = px_dma(ppgtt->pd);
> +
> + I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
> + I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
> +
> + msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
> + VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
> + } else {
> + for (i = 0; i < GEN8_3LVL_PDPES; i++) {
> + const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
> +
> + I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
> + I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
> + }
> +
> + msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
> + VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
> + }
> +
> + I915_WRITE(vgtif_reg(g2v_notify), msg);
> +
> + return 0;
> +}
> +
> +static void gen8_free_page_tables(struct i915_address_space *vm,
> + struct i915_page_directory *pd)
> +{
> + int i;
> +
> + for (i = 0; i < I915_PDES; i++) {
> + if (pd->entry[i] != &vm->scratch_pt)
> + free_pd(vm, pd->entry[i]);
> + }
> +}
> +
> +static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
> + struct i915_page_directory *pdp)
> +{
> + const unsigned int pdpes = i915_pdpes_per_pdp(vm);
> + int i;
> +
> + for (i = 0; i < pdpes; i++) {
> + if (pdp->entry[i] == &vm->scratch_pd)
> + continue;
> +
> + gen8_free_page_tables(vm, pdp->entry[i]);
> + free_pd(vm, pdp->entry[i]);
> + }
> +
> + free_px(vm, pdp);
> +}
> +
> +static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
> +{
> + struct i915_page_directory * const pml4 = ppgtt->pd;
> + int i;
> +
> + for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
> + struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
> +
> + if (px_base(pdp) == &ppgtt->vm.scratch_pdp)
> + continue;
> +
> + gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
> + }
> +
> + free_px(&ppgtt->vm, pml4);
> +}
> +
> +static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
> +{
> + struct drm_i915_private *i915 = vm->i915;
> + struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
> +
> + if (intel_vgpu_active(i915))
> + gen8_ppgtt_notify_vgt(ppgtt, false);
> +
> + if (i915_vm_is_4lvl(vm))
> + gen8_ppgtt_cleanup_4lvl(ppgtt);
> + else
> + gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);
> +
> + free_scratch(vm);
> +}
> +
> /* Removes entries from a single page table, releasing it if it's empty.
> * Caller can use the return value to update higher-level entries.
> */
> @@ -917,95 +1015,265 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
> }
> }
>
> -static inline struct sgt_dma {
> - struct scatterlist *sg;
> - dma_addr_t dma, max;
> -} sgt_dma(struct i915_vma *vma) {
> - struct scatterlist *sg = vma->pages->sgl;
> - dma_addr_t addr = sg_dma_address(sg);
> - return (struct sgt_dma) { sg, addr, addr + sg->length };
> -}
> -
> -struct gen8_insert_pte {
> - u16 pml4e;
> - u16 pdpe;
> - u16 pde;
> - u16 pte;
> -};
>
> -static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
> +static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
> + struct i915_page_directory *pd,
> + u64 start, u64 length)
> {
> - return (struct gen8_insert_pte) {
> - gen8_pml4e_index(start),
> - gen8_pdpe_index(start),
> - gen8_pde_index(start),
> - gen8_pte_index(start),
> - };
> -}
> + struct i915_page_table *pt, *alloc = NULL;
> + u64 from = start;
> + unsigned int pde;
> + int ret = 0;
>
> -static __always_inline bool
> -gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
> - struct i915_page_directory *pdp,
> - struct sgt_dma *iter,
> - struct gen8_insert_pte *idx,
> - enum i915_cache_level cache_level,
> - u32 flags)
> -{
> - struct i915_page_directory *pd;
> - const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
> - gen8_pte_t *vaddr;
> - bool ret;
> + spin_lock(&pd->lock);
> + gen8_for_each_pde(pt, pd, start, length, pde) {
> + const int count = gen8_pte_count(start, length);
>
> - GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
> - pd = i915_pd_entry(pdp, idx->pdpe);
> - vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
> - do {
> - vaddr[idx->pte] = pte_encode | iter->dma;
> + if (px_base(pt) == &vm->scratch_pt) {
> + spin_unlock(&pd->lock);
>
> - iter->dma += I915_GTT_PAGE_SIZE;
> - if (iter->dma >= iter->max) {
> - iter->sg = __sg_next(iter->sg);
> - if (!iter->sg) {
> - ret = false;
> - break;
> + pt = fetch_and_zero(&alloc);
> + if (!pt)
> + pt = alloc_pt(vm);
> + if (IS_ERR(pt)) {
> + ret = PTR_ERR(pt);
> + goto unwind;
> }
>
> - iter->dma = sg_dma_address(iter->sg);
> - iter->max = iter->dma + iter->sg->length;
> + if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
> + fill_px(pt, vm->scratch_pte);
> +
> + spin_lock(&pd->lock);
> + if (pd->entry[pde] == &vm->scratch_pt) {
> + set_pd_entry(pd, pde, pt);
> + } else {
> + alloc = pt;
> + pt = pd->entry[pde];
> + }
> }
>
> - if (++idx->pte == GEN8_PTES) {
> - idx->pte = 0;
> + atomic_add(count, &pt->used);
> + }
> + spin_unlock(&pd->lock);
> + goto out;
>
> - if (++idx->pde == I915_PDES) {
> - idx->pde = 0;
> +unwind:
> + gen8_ppgtt_clear_pd(vm, pd, from, start - from);
> +out:
> + if (alloc)
> + free_px(vm, alloc);
> + return ret;
> +}
>
> - /* Limited by sg length for 3lvl */
> - if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
> - idx->pdpe = 0;
> - ret = true;
> - break;
> - }
> +static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
> + struct i915_page_directory *pdp,
> + u64 start, u64 length)
> +{
> + struct i915_page_directory *pd, *alloc = NULL;
> + u64 from = start;
> + unsigned int pdpe;
> + int ret = 0;
>
> - GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
> - pd = pdp->entry[idx->pdpe];
> + spin_lock(&pdp->lock);
> + gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
> + if (px_base(pd) == &vm->scratch_pd) {
> + spin_unlock(&pdp->lock);
> +
> + pd = fetch_and_zero(&alloc);
> + if (!pd)
> + pd = alloc_pd(vm);
> + if (IS_ERR(pd)) {
> + ret = PTR_ERR(pd);
> + goto unwind;
> }
>
> - kunmap_atomic(vaddr);
> - vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
> + init_pd(pd, &vm->scratch_pt);
> +
> + spin_lock(&pdp->lock);
> + if (pdp->entry[pdpe] == &vm->scratch_pd) {
> + set_pd_entry(pdp, pdpe, pd);
> + } else {
> + alloc = pd;
> + pd = pdp->entry[pdpe];
> + }
> }
> - } while (1);
> - kunmap_atomic(vaddr);
> + atomic_inc(px_used(pd));
> + spin_unlock(&pdp->lock);
> +
> + ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
> + if (unlikely(ret))
> + goto unwind_pd;
> +
> + spin_lock(&pdp->lock);
> + atomic_dec(px_used(pd));
> + }
> + spin_unlock(&pdp->lock);
> + goto out;
>
> +unwind_pd:
> + if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))
> + free_px(vm, pd);
> +unwind:
> + gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
> +out:
> + if (alloc)
> + free_px(vm, alloc);
> return ret;
> }
>
> -static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
> - struct i915_vma *vma,
> - enum i915_cache_level cache_level,
> - u32 flags)
> +static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
> + u64 start, u64 length)
> {
> - struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
> + return gen8_ppgtt_alloc_pdp(vm,
> + i915_vm_to_ppgtt(vm)->pd, start, length);
> +}
> +
> +static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
> + u64 start, u64 length)
> +{
> + struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
> + struct i915_page_directory * const pml4 = ppgtt->pd;
> + struct i915_page_directory *pdp, *alloc = NULL;
> + u64 from = start;
> + int ret = 0;
> + u32 pml4e;
> +
> + spin_lock(&pml4->lock);
> + gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
> + if (px_base(pdp) == &vm->scratch_pdp) {
> + spin_unlock(&pml4->lock);
> +
> + pdp = fetch_and_zero(&alloc);
> + if (!pdp)
> + pdp = alloc_pd(vm);
> + if (IS_ERR(pdp)) {
> + ret = PTR_ERR(pdp);
> + goto unwind;
> + }
> +
> + init_pd(pdp, &vm->scratch_pd);
> +
> + spin_lock(&pml4->lock);
> + if (pml4->entry[pml4e] == &vm->scratch_pdp) {
> + set_pd_entry(pml4, pml4e, pdp);
> + } else {
> + alloc = pdp;
> + pdp = pml4->entry[pml4e];
> + }
> + }
> + atomic_inc(px_used(pdp));
> + spin_unlock(&pml4->lock);
> +
> + ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
> + if (unlikely(ret))
> + goto unwind_pdp;
> +
> + spin_lock(&pml4->lock);
> + atomic_dec(px_used(pdp));
> + }
> + spin_unlock(&pml4->lock);
> + goto out;
> +
> +unwind_pdp:
> + if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))
> + free_px(vm, pdp);
> +unwind:
> + gen8_ppgtt_clear_4lvl(vm, from, start - from);
> +out:
> + if (alloc)
> + free_px(vm, alloc);
> + return ret;
> +}
> +
> +static inline struct sgt_dma {
> + struct scatterlist *sg;
> + dma_addr_t dma, max;
> +} sgt_dma(struct i915_vma *vma) {
> + struct scatterlist *sg = vma->pages->sgl;
> + dma_addr_t addr = sg_dma_address(sg);
> + return (struct sgt_dma) { sg, addr, addr + sg->length };
> +}
> +
> +struct gen8_insert_pte {
> + u16 pml4e;
> + u16 pdpe;
> + u16 pde;
> + u16 pte;
> +};
> +
> +static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
> +{
> + return (struct gen8_insert_pte) {
> + gen8_pml4e_index(start),
> + gen8_pdpe_index(start),
> + gen8_pde_index(start),
> + gen8_pte_index(start),
> + };
> +}
> +
> +static __always_inline bool
> +gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
> + struct i915_page_directory *pdp,
> + struct sgt_dma *iter,
> + struct gen8_insert_pte *idx,
> + enum i915_cache_level cache_level,
> + u32 flags)
> +{
> + struct i915_page_directory *pd;
> + const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
> + gen8_pte_t *vaddr;
> + bool ret;
> +
> + GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
> + pd = i915_pd_entry(pdp, idx->pdpe);
> + vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
> + do {
> + vaddr[idx->pte] = pte_encode | iter->dma;
> +
> + iter->dma += I915_GTT_PAGE_SIZE;
> + if (iter->dma >= iter->max) {
> + iter->sg = __sg_next(iter->sg);
> + if (!iter->sg) {
> + ret = false;
> + break;
> + }
> +
> + iter->dma = sg_dma_address(iter->sg);
> + iter->max = iter->dma + iter->sg->length;
> + }
> +
> + if (++idx->pte == GEN8_PTES) {
> + idx->pte = 0;
> +
> + if (++idx->pde == I915_PDES) {
> + idx->pde = 0;
> +
> + /* Limited by sg length for 3lvl */
> + if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
> + idx->pdpe = 0;
> + ret = true;
> + break;
> + }
> +
> + GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
> + pd = pdp->entry[idx->pdpe];
> + }
> +
> + kunmap_atomic(vaddr);
> + vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
> + }
> + } while (1);
> + kunmap_atomic(vaddr);
> +
> + return ret;
> +}
> +
> +static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
> + struct i915_vma *vma,
> + enum i915_cache_level cache_level,
> + u32 flags)
> +{
> + struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
> struct sgt_dma iter = sgt_dma(vma);
> struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
>
> @@ -1161,17 +1429,6 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
> }
> }
>
> -static void gen8_free_page_tables(struct i915_address_space *vm,
> - struct i915_page_directory *pd)
> -{
> - int i;
> -
> - for (i = 0; i < I915_PDES; i++) {
> - if (pd->entry[i] != &vm->scratch_pt)
> - free_pd(vm, pd->entry[i]);
> - }
> -}
> -
> static int gen8_init_scratch(struct i915_address_space *vm)
> {
> int ret;
> @@ -1239,262 +1496,6 @@ static int gen8_init_scratch(struct i915_address_space *vm)
> return ret;
> }
>
> -static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
> -{
> - struct i915_address_space *vm = &ppgtt->vm;
> - struct drm_i915_private *dev_priv = vm->i915;
> - enum vgt_g2v_type msg;
> - int i;
> -
> - if (create)
> - atomic_inc(px_used(ppgtt->pd)); /* never remove */
> - else
> - atomic_dec(px_used(ppgtt->pd));
> -
> - if (i915_vm_is_4lvl(vm)) {
> - const u64 daddr = px_dma(ppgtt->pd);
> -
> - I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
> - I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
> -
> - msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
> - VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
> - } else {
> - for (i = 0; i < GEN8_3LVL_PDPES; i++) {
> - const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
> -
> - I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
> - I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
> - }
> -
> - msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
> - VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
> - }
> -
> - I915_WRITE(vgtif_reg(g2v_notify), msg);
> -
> - return 0;
> -}
> -
> -static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
> - struct i915_page_directory *pdp)
> -{
> - const unsigned int pdpes = i915_pdpes_per_pdp(vm);
> - int i;
> -
> - for (i = 0; i < pdpes; i++) {
> - if (pdp->entry[i] == &vm->scratch_pd)
> - continue;
> -
> - gen8_free_page_tables(vm, pdp->entry[i]);
> - free_pd(vm, pdp->entry[i]);
> - }
> -
> - free_px(vm, pdp);
> -}
> -
> -static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
> -{
> - struct i915_page_directory * const pml4 = ppgtt->pd;
> - int i;
> -
> - for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
> - struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
> -
> - if (px_base(pdp) == &ppgtt->vm.scratch_pdp)
> - continue;
> -
> - gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
> - }
> -
> - free_px(&ppgtt->vm, pml4);
> -}
> -
> -static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
> -{
> - struct drm_i915_private *i915 = vm->i915;
> - struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
> -
> - if (intel_vgpu_active(i915))
> - gen8_ppgtt_notify_vgt(ppgtt, false);
> -
> - if (i915_vm_is_4lvl(vm))
> - gen8_ppgtt_cleanup_4lvl(ppgtt);
> - else
> - gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);
> -
> - free_scratch(vm);
> -}
> -
> -static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
> - struct i915_page_directory *pd,
> - u64 start, u64 length)
> -{
> - struct i915_page_table *pt, *alloc = NULL;
> - u64 from = start;
> - unsigned int pde;
> - int ret = 0;
> -
> - spin_lock(&pd->lock);
> - gen8_for_each_pde(pt, pd, start, length, pde) {
> - const int count = gen8_pte_count(start, length);
> -
> - if (px_base(pt) == &vm->scratch_pt) {
> - spin_unlock(&pd->lock);
> -
> - pt = fetch_and_zero(&alloc);
> - if (!pt)
> - pt = alloc_pt(vm);
> - if (IS_ERR(pt)) {
> - ret = PTR_ERR(pt);
> - goto unwind;
> - }
> -
> - if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
> - fill_px(pt, vm->scratch_pte);
> -
> - spin_lock(&pd->lock);
> - if (pd->entry[pde] == &vm->scratch_pt) {
> - set_pd_entry(pd, pde, pt);
> - } else {
> - alloc = pt;
> - pt = pd->entry[pde];
> - }
> - }
> -
> - atomic_add(count, &pt->used);
> - }
> - spin_unlock(&pd->lock);
> - goto out;
> -
> -unwind:
> - gen8_ppgtt_clear_pd(vm, pd, from, start - from);
> -out:
> - if (alloc)
> - free_px(vm, alloc);
> - return ret;
> -}
> -
> -static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
> - struct i915_page_directory *pdp,
> - u64 start, u64 length)
> -{
> - struct i915_page_directory *pd, *alloc = NULL;
> - u64 from = start;
> - unsigned int pdpe;
> - int ret = 0;
> -
> - spin_lock(&pdp->lock);
> - gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
> - if (px_base(pd) == &vm->scratch_pd) {
> - spin_unlock(&pdp->lock);
> -
> - pd = fetch_and_zero(&alloc);
> - if (!pd)
> - pd = alloc_pd(vm);
> - if (IS_ERR(pd)) {
> - ret = PTR_ERR(pd);
> - goto unwind;
> - }
> -
> - init_pd(pd, &vm->scratch_pt);
> -
> - spin_lock(&pdp->lock);
> - if (pdp->entry[pdpe] == &vm->scratch_pd) {
> - set_pd_entry(pdp, pdpe, pd);
> - } else {
> - alloc = pd;
> - pd = pdp->entry[pdpe];
> - }
> - }
> - atomic_inc(px_used(pd));
> - spin_unlock(&pdp->lock);
> -
> - ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
> - if (unlikely(ret))
> - goto unwind_pd;
> -
> - spin_lock(&pdp->lock);
> - atomic_dec(px_used(pd));
> - }
> - spin_unlock(&pdp->lock);
> - goto out;
> -
> -unwind_pd:
> - if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))
> - free_px(vm, pd);
> -unwind:
> - gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
> -out:
> - if (alloc)
> - free_px(vm, alloc);
> - return ret;
> -}
> -
> -static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
> - u64 start, u64 length)
> -{
> - return gen8_ppgtt_alloc_pdp(vm,
> - i915_vm_to_ppgtt(vm)->pd, start, length);
> -}
> -
> -static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
> - u64 start, u64 length)
> -{
> - struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
> - struct i915_page_directory * const pml4 = ppgtt->pd;
> - struct i915_page_directory *pdp, *alloc = NULL;
> - u64 from = start;
> - int ret = 0;
> - u32 pml4e;
> -
> - spin_lock(&pml4->lock);
> - gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
> - if (px_base(pdp) == &vm->scratch_pdp) {
> - spin_unlock(&pml4->lock);
> -
> - pdp = fetch_and_zero(&alloc);
> - if (!pdp)
> - pdp = alloc_pd(vm);
> - if (IS_ERR(pdp)) {
> - ret = PTR_ERR(pdp);
> - goto unwind;
> - }
> -
> - init_pd(pdp, &vm->scratch_pd);
> -
> - spin_lock(&pml4->lock);
> - if (pml4->entry[pml4e] == &vm->scratch_pdp) {
> - set_pd_entry(pml4, pml4e, pdp);
> - } else {
> - alloc = pdp;
> - pdp = pml4->entry[pml4e];
> - }
> - }
> - atomic_inc(px_used(pdp));
> - spin_unlock(&pml4->lock);
> -
> - ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
> - if (unlikely(ret))
> - goto unwind_pdp;
> -
> - spin_lock(&pml4->lock);
> - atomic_dec(px_used(pdp));
> - }
> - spin_unlock(&pml4->lock);
> - goto out;
> -
> -unwind_pdp:
> - if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))
> - free_px(vm, pdp);
> -unwind:
> - gen8_ppgtt_clear_4lvl(vm, from, start - from);
> -out:
> - if (alloc)
> - free_px(vm, alloc);
> - return ret;
> -}
> -
> static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
> {
> struct i915_address_space *vm = &ppgtt->vm;
> --
> 2.20.1
More information about the Intel-gfx
mailing list