[PATCH] Add 2-level GPUVM pagetables support to radeon driver.
Jerome Glisse
j.glisse at gmail.com
Thu Sep 13 11:17:45 PDT 2012
On Thu, Sep 13, 2012 at 10:13 AM, Dmitry Cherkasov
<dcherkassov at gmail.com> wrote:
> PDE/PTE update code uses CP ring for memory writes.
> All page table entries are preallocated for now in alloc_pt().
>
> It is made as whole because it's hard to divide it to several patches
> that compile and doesn't break anything being applied separately.
>
> Tested on cayman card.
>
> Signed-off-by: Dmitry Cherkasov <Dmitrii.Cherkasov at amd.com>
> ---
> I couldn't test in on SI card, so would be happy if someone could check it there.
I wonder how this could have work as you don't set
PAGE_TABLE_BLOCK_SIZE field so each page directory entry cover only 1
page.
Cheers,
Jerome
>
> drivers/gpu/drm/radeon/ni.c | 64 ++++++++++++++++++-------
> drivers/gpu/drm/radeon/radeon.h | 79 +++++++++++++++++++++++++++---
> drivers/gpu/drm/radeon/radeon_asic.c | 1 +
> drivers/gpu/drm/radeon/radeon_asic.h | 13 +++--
> drivers/gpu/drm/radeon/radeon_gart.c | 88 ++++++++++++++++++++++++++++++----
> drivers/gpu/drm/radeon/si.c | 4 +-
> 6 files changed, 210 insertions(+), 39 deletions(-)
>
> diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
> index b238216..31d9027 100644
> --- a/drivers/gpu/drm/radeon/ni.c
> +++ b/drivers/gpu/drm/radeon/ni.c
> @@ -782,7 +782,7 @@ static int cayman_pcie_gart_enable(struct radeon_device *rdev)
> (u32)(rdev->dummy_page.addr >> 12));
> WREG32(VM_CONTEXT1_CNTL2, 0);
> WREG32(VM_CONTEXT1_CNTL, 0);
> - WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
> + WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
> RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
>
> cayman_pcie_gart_tlb_flush(rdev);
> @@ -1517,37 +1517,65 @@ uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
> return r600_flags;
> }
>
> -/**
> - * cayman_vm_set_page - update the page tables using the CP
> - *
> - * @rdev: radeon_device pointer
> - *
> - * Update the page tables using the CP (cayman-si).
> - */
> -void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
> - unsigned pfn, struct ttm_mem_reg *mem,
> - unsigned npages, uint32_t flags)
> +void cayman_vm_set_page(struct radeon_device *rdev,
> + struct ttm_mem_reg *mem,
> + uint64_t pte_gpu_addr,
> + uint64_t mem_pfn_offset,
> + unsigned count, uint32_t flags)
> {
> struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];
> - uint64_t addr, pt = vm->pt_gpu_addr + pfn * 8;
> + uint64_t addr;
> int i;
>
> addr = flags = cayman_vm_page_flags(rdev, flags);
>
> - radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, 1 + npages * 2));
> - radeon_ring_write(ring, pt & 0xffffffff);
> - radeon_ring_write(ring, (pt >> 32) & 0xff);
> - for (i = 0; i < npages; ++i) {
> + radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, 1 + count * 2));
> + radeon_ring_write(ring, pte_gpu_addr & 0xffffffff);
> + radeon_ring_write(ring, (pte_gpu_addr >> 32) & 0xff);
> + for (i = 0; i < count; ++i) {
> if (mem) {
> - addr = radeon_vm_get_addr(rdev, mem, i);
> + addr = radeon_vm_get_addr(rdev, mem,
> + i + mem_pfn_offset);
> addr = addr & 0xFFFFFFFFFFFFF000ULL;
> addr |= flags;
> }
> +
> + radeon_ring_write(ring, addr & 0xffffffff);
> + radeon_ring_write(ring, (addr >> 32) & 0xffffffff);
> + }
> +}
> +
> +void cayman_vm_set_pdes(struct radeon_device *rdev,
> + struct ttm_mem_reg *mem,
> + struct radeon_vm *vm,
> + uint64_t pde_index,
> + unsigned pde_count, uint32_t flags)
> +{
> + struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];
> + uint64_t addr;
> + int i;
> + uint64_t pde_gpu_addr = RADEON_BASE_GPU_ADDR(vm) +
> + PDE_OFFSET(rdev, pde_index);
> +
> + addr = 0;
> +
> + radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, 1 + pde_count * 2));
> + radeon_ring_write(ring, pde_gpu_addr & 0xffffffff);
> + radeon_ring_write(ring, (pde_gpu_addr >> 32) & 0xff);
> +
> + for (i = 0; i < pde_count; ++i) {
> + addr = RADEON_BASE_GPU_ADDR(vm) +
> + PTE_OFFSET(rdev, i + pde_index, 0);
> + addr = addr & 0xFFFFFFFFFFFFF000ULL;
> + addr |= flags;
> +
> radeon_ring_write(ring, addr & 0xffffffff);
> radeon_ring_write(ring, (addr >> 32) & 0xffffffff);
> }
> }
>
> +
> +
> /**
> * cayman_vm_flush - vm flush using the CP
> *
> @@ -1571,7 +1599,7 @@ void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ib *ib)
> radeon_ring_write(ring, vm->last_pfn);
>
> radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
> - radeon_ring_write(ring, vm->pt_gpu_addr >> 12);
> + radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
>
> /* flush hdp cache */
> radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
> diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
> index 4d67f0f..062896f 100644
> --- a/drivers/gpu/drm/radeon/radeon.h
> +++ b/drivers/gpu/drm/radeon/radeon.h
> @@ -655,8 +655,8 @@ struct radeon_vm {
> struct list_head va;
> unsigned id;
> unsigned last_pfn;
> - u64 pt_gpu_addr;
> - u64 *pt;
> + u64 pd_gpu_addr;
> + u64 __iomem *pd_addr;
> struct radeon_sa_bo *sa_bo;
> struct mutex mutex;
> /* last fence for cs using this vm */
> @@ -679,6 +679,57 @@ struct radeon_vm_manager {
> bool enabled;
> };
>
> +
> +
> +/* We consider the case where BLOCK_SIZE is 0 */
> +/* So PDE is 19 bits long, PTE is 9 and OFFSET is 12 */
> +#define RADEON_BLOCK_SIZE 0
> +
> +/* By default there are 512 entries in Page Table */
> +#define RADEON_DEFAULT_PTE_COUNT (1 << 9)
> +
> +/* number of PTEs in Page Table */
> +#define RADEON_PTE_COUNT (RADEON_DEFAULT_PTE_COUNT << RADEON_BLOCK_SIZE)
> +
> +/* Get last PDE number containing nth PTE */
> +#define RADEON_GET_LAST_PDE_FOR_PFN(_n) ((_n) / RADEON_PTE_COUNT)
> +
> +/* Get PTE number to containing nth pfn */
> +#define RADEON_GET_PTE_FOR_PFN(_n) ((_n) % RADEON_PTE_COUNT)
> +
> +/* Number of PDE tables to cover n PTEs */
> +#define RADEON_PDE_COUNT_FOR_N_PAGES(_n) \
> + (((_n) + RADEON_PTE_COUNT - 1) / RADEON_PTE_COUNT)
> +
> +/* Number of PDE tables to cover max_pfn (maximum number of PTEs) */
> +#define RADEON_TOTAL_PDE_COUNT(rdev) \
> + RADEON_PDE_COUNT_FOR_N_PAGES(rdev->vm_manager.max_pfn)
> +
> +#define PTE_SIZE 8
> +#define PDE_SIZE 8
> +
> +/* offset for nth PDE starting from beginning of PDE table */
> +#define PDE_OFFSET(_rdev, _npde) ((_npde) * PDE_SIZE)
> +
> +#define PT_OFFSET(_rdev) \
> + (RADEON_GPU_PAGE_ALIGN(RADEON_TOTAL_PDE_COUNT(rdev) * PDE_SIZE))
> +
> +/* offset for nth PTE of mth PDE starting from beginning of PDE table */
> +#define PTE_OFFSET(_rdev, _npde, _npte) \
> + (PT_OFFSET(_rdev) + \
> + (_npde) * RADEON_PTE_COUNT * PTE_SIZE + \
> + (_npte) * PTE_SIZE)
> +
> +/* cpu address of gpuvm page table */
> +#define RADEON_BASE_CPU_ADDR(_vm) \
> + radeon_sa_bo_cpu_addr(vm->sa_bo)
> +
> +/* gpu address of gpuvm page table */
> +#define RADEON_BASE_GPU_ADDR(_vm) \
> + radeon_sa_bo_gpu_addr(vm->sa_bo)
> +
> +#define RADEON_PDE_FLAG_VALID 1
> +
> /*
> * file private structure
> */
> @@ -1141,9 +1192,20 @@ struct radeon_asic {
> void (*fini)(struct radeon_device *rdev);
>
> u32 pt_ring_index;
> - void (*set_page)(struct radeon_device *rdev, struct radeon_vm *vm,
> - unsigned pfn, struct ttm_mem_reg *mem,
> - unsigned npages, uint32_t flags);
> + void (*set_page)(
> + struct radeon_device *rdev,
> + struct ttm_mem_reg *mem,
> + uint64_t pte_gpu_addr, uint64_t mem_pfn_offset,
> + unsigned count, uint32_t flags
> +);
> +
> + void (*set_pdes)(
> + struct radeon_device *rdev,
> + struct ttm_mem_reg *mem,
> + struct radeon_vm *vm,
> + uint64_t pde_index,
> + unsigned pde_count, uint32_t flags);
> +
> } vm;
> /* ring specific callbacks */
> struct {
> @@ -1755,7 +1817,12 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
> #define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart.set_page((rdev), (i), (p))
> #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev))
> #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev))
> -#define radeon_asic_vm_set_page(rdev, v, pfn, mem, npages, flags) (rdev)->asic->vm.set_page((rdev), (v), (pfn), (mem), (npages), (flags))
> +#define radeon_asic_vm_set_page(rdev, mem, start_pte, start_pfn, count, flags) \
> +((rdev)->asic->vm.set_page((rdev), (mem), (start_pte), (start_pfn), \
> + (count), (flags)))
> +#define radeon_asic_vm_set_pdes(rdev, mem, vm, pde_index, pde_count, flags) \
> +((rdev)->asic->vm.set_pdes((rdev), (mem), (vm), (pde_index), \
> + (pde_count), (flags)))
> #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)].ring_start((rdev), (cp))
> #define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)].ring_test((rdev), (cp))
> #define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)].ib_test((rdev), (cp))
> diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
> index 2f7adea..0df6a55 100644
> --- a/drivers/gpu/drm/radeon/radeon_asic.c
> +++ b/drivers/gpu/drm/radeon/radeon_asic.c
> @@ -1377,6 +1377,7 @@ static struct radeon_asic cayman_asic = {
> .fini = &cayman_vm_fini,
> .pt_ring_index = RADEON_RING_TYPE_GFX_INDEX,
> .set_page = &cayman_vm_set_page,
> + .set_pdes = &cayman_vm_set_pdes,
> },
> .ring = {
> [RADEON_RING_TYPE_GFX_INDEX] = {
> diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
> index 29b3d05..7a694c0 100644
> --- a/drivers/gpu/drm/radeon/radeon_asic.h
> +++ b/drivers/gpu/drm/radeon/radeon_asic.h
> @@ -442,9 +442,16 @@ int cayman_vm_init(struct radeon_device *rdev);
> void cayman_vm_fini(struct radeon_device *rdev);
> void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ib *ib);
> uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags);
> -void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
> - unsigned pfn, struct ttm_mem_reg *mem,
> - unsigned npages, uint32_t flags);
> +void cayman_vm_set_page(struct radeon_device *rdev,
> + struct ttm_mem_reg *mem,
> + uint64_t pte_gpu_addr, uint64_t mem_pfn_offset,
> + unsigned count, uint32_t flags);
> +void cayman_vm_set_pdes(struct radeon_device *rdev,
> + struct ttm_mem_reg *mem,
> + struct radeon_vm *vm,
> + uint64_t pde_index,
> + unsigned pde_count, uint32_t flags);
> +
> int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
>
> /* DCE6 - SI */
> diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
> index 2f28ff3..f9bda6e 100644
> --- a/drivers/gpu/drm/radeon/radeon_gart.c
> +++ b/drivers/gpu/drm/radeon/radeon_gart.c
> @@ -490,7 +490,6 @@ static void radeon_vm_free_pt(struct radeon_device *rdev,
>
> list_del_init(&vm->list);
> radeon_sa_bo_free(rdev, &vm->sa_bo, vm->fence);
> - vm->pt = NULL;
>
> list_for_each_entry(bo_va, &vm->va, vm_list) {
> bo_va->valid = false;
> @@ -547,6 +546,10 @@ int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
> struct radeon_vm *vm_evict;
> int r;
>
> + int gpuvm_tables_sz =
> + RADEON_GPU_PAGE_ALIGN(RADEON_TOTAL_PDE_COUNT(rdev) * PDE_SIZE) +
> + vm->last_pfn * PTE_SIZE;
> +
> if (vm == NULL) {
> return -EINVAL;
> }
> @@ -560,7 +563,7 @@ int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
>
> retry:
> r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo,
> - RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8),
> + gpuvm_tables_sz,
> RADEON_GPU_PAGE_SIZE, false);
> if (r == -ENOMEM) {
> if (list_empty(&rdev->vm_manager.lru_vm)) {
> @@ -576,9 +579,13 @@ retry:
> return r;
> }
>
> - vm->pt = radeon_sa_bo_cpu_addr(vm->sa_bo);
> - vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo);
> - memset(vm->pt, 0, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8));
> + DRM_INFO("radeon: reserved %d kb pd & pt tables\n",
> + gpuvm_tables_sz/1024);
> +
> + vm->pd_addr = radeon_sa_bo_cpu_addr(vm->sa_bo);
> + vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo);
> + memset(vm->pd_addr, 0, gpuvm_tables_sz);
> +
>
> list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
> return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo,
> @@ -861,6 +868,63 @@ u64 radeon_vm_get_addr(struct radeon_device *rdev,
> }
>
> /**
> + * radeon_vm_bo_set_pages - update PDE and PTE for pfn
> + *
> + * @rdev: radeon_device pointer
> + * @first_pfn: first pfn in bo address space to start mapping from
> + * @mem: ttm mem
> + * @vm: requested vm
> + * @bo: radeon buffer object
> + *
> + * Update page directory and table entries for pfn (cayman+).
> + */
> +
> +int radeon_vm_bo_set_pages(struct radeon_device *rdev, struct radeon_vm *vm,
> + unsigned first_pfn, struct ttm_mem_reg *mem,
> + unsigned npages, uint32_t flags)
> +{
> + u64 pde_num, pte_num, start_pde, pde_count = 0;
> +
> + unsigned pt_total, count, pfn_idx;
> + unsigned last_pfn = first_pfn + npages, pfns_to_pt_edge, pfns_to_end;
> + unsigned mem_pfn_offset;
> +
> + pt_total = RADEON_PDE_COUNT_FOR_N_PAGES(npages);
> + count = RADEON_PTE_COUNT;
> +
> + pfn_idx = first_pfn;
> +
> + for (mem_pfn_offset = 0; mem_pfn_offset < npages;) {
> +
> + pfns_to_end = last_pfn - pfn_idx;
> + pfns_to_pt_edge = RADEON_PTE_COUNT -
> + (pfn_idx % RADEON_PTE_COUNT);
> +
> + count = pfns_to_pt_edge < pfns_to_end ?
> + pfns_to_pt_edge : pfns_to_end;
> +
> + pde_num = RADEON_GET_LAST_PDE_FOR_PFN(pfn_idx);
> + pte_num = RADEON_GET_PTE_FOR_PFN(pfn_idx);
> +
> + radeon_asic_vm_set_page(rdev, mem,
> + vm->pd_gpu_addr +
> + PTE_OFFSET(rdev, pde_num, pte_num),
> + mem_pfn_offset, count, flags);
> + pfn_idx += count;
> + mem_pfn_offset += count;
> +
> + pde_count++;
> + }
> +
> + start_pde = RADEON_GET_LAST_PDE_FOR_PFN(first_pfn);
> +
> + radeon_asic_vm_set_pdes(rdev, mem, vm,
> + start_pde, pde_count, RADEON_PDE_FLAG_VALID);
> +
> + return 0;
> +}
> +
> +/**
> * radeon_vm_bo_update_pte - map a bo into the vm page table
> *
> * @rdev: radeon_device pointer
> @@ -882,7 +946,7 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
> struct radeon_ring *ring = &rdev->ring[ridx];
> struct radeon_semaphore *sem = NULL;
> struct radeon_bo_va *bo_va;
> - unsigned ngpu_pages, ndw;
> + unsigned ngpu_pages, ndw, npdes;
> uint64_t pfn;
> int r;
>
> @@ -935,10 +999,12 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
> }
> }
>
> - /* estimate number of dw needed */
> + npdes = (RADEON_PDE_COUNT_FOR_N_PAGES(pfn + ngpu_pages) -
> + RADEON_GET_LAST_PDE_FOR_PFN(pfn));
> +
> ndw = 32;
> - ndw += (ngpu_pages >> 12) * 3;
> - ndw += ngpu_pages * 2;
> + ndw += ngpu_pages * 2 + 3 * npdes;
> + ndw += npdes * 2 + 3;
>
> r = radeon_ring_lock(rdev, ring, ndw);
> if (r) {
> @@ -950,7 +1016,7 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
> radeon_fence_note_sync(vm->fence, ridx);
> }
>
> - radeon_asic_vm_set_page(rdev, vm, pfn, mem, ngpu_pages, bo_va->flags);
> + radeon_vm_bo_set_pages(rdev, vm, pfn, mem, ngpu_pages, bo_va->flags);
>
> radeon_fence_unref(&vm->fence);
> r = radeon_fence_emit(rdev, &vm->fence, ridx);
> @@ -958,9 +1024,11 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
> radeon_ring_unlock_undo(rdev, ring);
> return r;
> }
> +
> radeon_ring_unlock_commit(rdev, ring);
> radeon_semaphore_free(rdev, &sem, vm->fence);
> radeon_fence_unref(&vm->last_flush);
> +
> return 0;
> }
>
> diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
> index 2a5c337..156c994 100644
> --- a/drivers/gpu/drm/radeon/si.c
> +++ b/drivers/gpu/drm/radeon/si.c
> @@ -2426,7 +2426,7 @@ static int si_pcie_gart_enable(struct radeon_device *rdev)
> WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
> (u32)(rdev->dummy_page.addr >> 12));
> WREG32(VM_CONTEXT1_CNTL2, 0);
> - WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
> + WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
> RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
>
> si_pcie_gart_tlb_flush(rdev);
> @@ -2804,7 +2804,7 @@ void si_vm_flush(struct radeon_device *rdev, struct radeon_ib *ib)
> radeon_ring_write(ring, PACKET0(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
> + ((vm->id - 8) << 2), 0));
> }
> - radeon_ring_write(ring, vm->pt_gpu_addr >> 12);
> + radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
>
> /* flush hdp cache */
> radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
> --
> 1.7.10.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
More information about the dri-devel
mailing list