[PATCH 3/4] drm/ttm: rework BO delayed delete.
Felix Kuehling
felix.kuehling at amd.com
Fri Nov 15 22:47:01 UTC 2019
On 2019-11-11 9:58 a.m., Christian König wrote:
> This patch reworks the whole delayed deletion of BOs which aren't idle.
>
> Instead of having two counters for the BO structure we resurrect the BO
> when we find that a deleted BO is not idle yet.
>
> This has many advantages, especially that we don't need to
> increment/decrement the BOs reference counter any more when it
> moves on the LRUs.
>
> Signed-off-by: Christian König <christian.koenig at amd.com>
I reviewed it fairly thoroughly and couldn't find anything wrong. But I
don't know this code well enough, including all the implications of all
the calls being moved around, to give a confident R-b.
Acked-by: Felix Kuehling <Felix.Kuehling>
> ---
> drivers/gpu/drm/ttm/ttm_bo.c | 215 +++++++++++++-----------------
> drivers/gpu/drm/ttm/ttm_bo_util.c | 1 -
> include/drm/ttm/ttm_bo_api.h | 11 +-
> 3 files changed, 97 insertions(+), 130 deletions(-)
>
> diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
> index 1178980f4147..570b0e1089b7 100644
> --- a/drivers/gpu/drm/ttm/ttm_bo.c
> +++ b/drivers/gpu/drm/ttm/ttm_bo.c
> @@ -145,26 +145,6 @@ static inline uint32_t ttm_bo_type_flags(unsigned type)
> return 1 << (type);
> }
>
> -static void ttm_bo_release_list(struct kref *list_kref)
> -{
> - struct ttm_buffer_object *bo =
> - container_of(list_kref, struct ttm_buffer_object, list_kref);
> - size_t acc_size = bo->acc_size;
> -
> - BUG_ON(kref_read(&bo->list_kref));
> - BUG_ON(kref_read(&bo->kref));
> - BUG_ON(bo->mem.mm_node != NULL);
> - BUG_ON(!list_empty(&bo->lru));
> - BUG_ON(!list_empty(&bo->ddestroy));
> - ttm_tt_destroy(bo->ttm);
> - atomic_dec(&ttm_bo_glob.bo_count);
> - dma_fence_put(bo->moving);
> - if (!ttm_bo_uses_embedded_gem_object(bo))
> - dma_resv_fini(&bo->base._resv);
> - bo->destroy(bo);
> - ttm_mem_global_free(&ttm_mem_glob, acc_size);
> -}
> -
> static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
> struct ttm_mem_reg *mem)
> {
> @@ -181,21 +161,14 @@ static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
>
> man = &bdev->man[mem->mem_type];
> list_add_tail(&bo->lru, &man->lru[bo->priority]);
> - kref_get(&bo->list_kref);
>
> if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm &&
> !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
> TTM_PAGE_FLAG_SWAPPED))) {
> list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]);
> - kref_get(&bo->list_kref);
> }
> }
>
> -static void ttm_bo_ref_bug(struct kref *list_kref)
> -{
> - BUG();
> -}
> -
> static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
> {
> struct ttm_bo_device *bdev = bo->bdev;
> @@ -203,12 +176,10 @@ static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
>
> if (!list_empty(&bo->swap)) {
> list_del_init(&bo->swap);
> - kref_put(&bo->list_kref, ttm_bo_ref_bug);
> notify = true;
> }
> if (!list_empty(&bo->lru)) {
> list_del_init(&bo->lru);
> - kref_put(&bo->list_kref, ttm_bo_ref_bug);
> notify = true;
> }
>
> @@ -446,74 +417,17 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
> dma_fence_enable_sw_signaling(fence);
>
> for (i = 0; fobj && i < fobj->shared_count; ++i) {
> - fence = rcu_dereference_protected(fobj->shared[i],
> - dma_resv_held(bo->base.resv));
> + fence = rcu_dereference_protected(fobj->shared[i], true);
>
> if (!fence->ops->signaled)
> dma_fence_enable_sw_signaling(fence);
> }
> }
>
> -static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
> -{
> - struct ttm_bo_device *bdev = bo->bdev;
> - int ret;
> -
> - ret = ttm_bo_individualize_resv(bo);
> - if (ret) {
> - /* Last resort, if we fail to allocate memory for the
> - * fences block for the BO to become idle
> - */
> - dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
> - 30 * HZ);
> - spin_lock(&ttm_bo_glob.lru_lock);
> - goto error;
> - }
> -
> - spin_lock(&ttm_bo_glob.lru_lock);
> - ret = dma_resv_trylock(bo->base.resv) ? 0 : -EBUSY;
> - if (!ret) {
> - if (dma_resv_test_signaled_rcu(&bo->base._resv, true)) {
> - ttm_bo_del_from_lru(bo);
> - spin_unlock(&ttm_bo_glob.lru_lock);
> - if (bo->base.resv != &bo->base._resv)
> - dma_resv_unlock(&bo->base._resv);
> -
> - ttm_bo_cleanup_memtype_use(bo);
> - dma_resv_unlock(bo->base.resv);
> - return;
> - }
> -
> - ttm_bo_flush_all_fences(bo);
> -
> - /*
> - * Make NO_EVICT bos immediately available to
> - * shrinkers, now that they are queued for
> - * destruction.
> - */
> - if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
> - bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
> - ttm_bo_move_to_lru_tail(bo, NULL);
> - }
> -
> - dma_resv_unlock(bo->base.resv);
> - }
> - if (bo->base.resv != &bo->base._resv)
> - dma_resv_unlock(&bo->base._resv);
> -
> -error:
> - kref_get(&bo->list_kref);
> - list_add_tail(&bo->ddestroy, &bdev->ddestroy);
> - spin_unlock(&ttm_bo_glob.lru_lock);
> -
> - schedule_delayed_work(&bdev->wq,
> - ((HZ / 100) < 1) ? 1 : HZ / 100);
> -}
> -
> /**
> * function ttm_bo_cleanup_refs
> - * If bo idle, remove from delayed- and lru lists, and unref.
> - * If not idle, do nothing.
> + * If bo idle, remove from lru lists, and unref.
> + * If not idle, block if possible.
> *
> * Must be called with lru_lock and reservation held, this function
> * will drop the lru lock and optionally the reservation lock before returning.
> @@ -575,14 +489,14 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
>
> ttm_bo_del_from_lru(bo);
> list_del_init(&bo->ddestroy);
> - kref_put(&bo->list_kref, ttm_bo_ref_bug);
> -
> spin_unlock(&ttm_bo_glob.lru_lock);
> ttm_bo_cleanup_memtype_use(bo);
>
> if (unlock_resv)
> dma_resv_unlock(bo->base.resv);
>
> + ttm_bo_put(bo);
> +
> return 0;
> }
>
> @@ -604,8 +518,9 @@ static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
>
> bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
> ddestroy);
> - kref_get(&bo->list_kref);
> list_move_tail(&bo->ddestroy, &removed);
> + if (!ttm_bo_get_unless_zero(bo))
> + continue;
>
> if (remove_all || bo->base.resv != &bo->base._resv) {
> spin_unlock(&glob->lru_lock);
> @@ -620,7 +535,7 @@ static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
> spin_unlock(&glob->lru_lock);
> }
>
> - kref_put(&bo->list_kref, ttm_bo_release_list);
> + ttm_bo_put(bo);
> spin_lock(&glob->lru_lock);
> }
> list_splice_tail(&removed, &bdev->ddestroy);
> @@ -646,16 +561,68 @@ static void ttm_bo_release(struct kref *kref)
> container_of(kref, struct ttm_buffer_object, kref);
> struct ttm_bo_device *bdev = bo->bdev;
> struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
> + size_t acc_size = bo->acc_size;
> + int ret;
>
> - if (bo->bdev->driver->release_notify)
> - bo->bdev->driver->release_notify(bo);
> + if (!bo->deleted) {
> + if (bo->bdev->driver->release_notify)
> + bo->bdev->driver->release_notify(bo);
>
> - drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
> - ttm_mem_io_lock(man, false);
> - ttm_mem_io_free_vm(bo);
> - ttm_mem_io_unlock(man);
> - ttm_bo_cleanup_refs_or_queue(bo);
> - kref_put(&bo->list_kref, ttm_bo_release_list);
> + drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
> + ttm_mem_io_lock(man, false);
> + ttm_mem_io_free_vm(bo);
> + ttm_mem_io_unlock(man);
> +
> + ret = ttm_bo_individualize_resv(bo);
> + if (ret) {
> + /* Last resort, if we fail to allocate memory for the
> + * fences block for the BO to become idle
> + */
> + dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
> + 30 * HZ);
> + }
> + }
> +
> + if (!dma_resv_test_signaled_rcu(bo->base.resv, true)) {
> + /* The BO is not idle, resurrect it for delayed destroy */
> + ttm_bo_flush_all_fences(bo);
> + bo->deleted = true;
> +
> + /*
> + * Make NO_EVICT bos immediately available to
> + * shrinkers, now that they are queued for
> + * destruction.
> + */
> + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
> + bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
> + ttm_bo_move_to_lru_tail(bo, NULL);
> + }
> +
> + spin_lock(&ttm_bo_glob.lru_lock);
> + kref_init(&bo->kref);
> + list_add_tail(&bo->ddestroy, &bdev->ddestroy);
> + spin_unlock(&ttm_bo_glob.lru_lock);
> +
> + schedule_delayed_work(&bdev->wq,
> + ((HZ / 100) < 1) ? 1 : HZ / 100);
> + return;
> + }
> +
> + spin_lock(&ttm_bo_glob.lru_lock);
> + ttm_bo_del_from_lru(bo);
> + list_del(&bo->ddestroy);
> + spin_unlock(&ttm_bo_glob.lru_lock);
> +
> + ttm_bo_cleanup_memtype_use(bo);
> +
> + BUG_ON(bo->mem.mm_node != NULL);
> + ttm_tt_destroy(bo->ttm);
> + atomic_dec(&ttm_bo_glob.bo_count);
> + dma_fence_put(bo->moving);
> + if (!ttm_bo_uses_embedded_gem_object(bo))
> + dma_resv_fini(&bo->base._resv);
> + bo->destroy(bo);
> + ttm_mem_global_free(&ttm_mem_glob, acc_size);
> }
>
> void ttm_bo_put(struct ttm_buffer_object *bo)
> @@ -758,8 +725,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
>
> if (bo->base.resv == ctx->resv) {
> dma_resv_assert_held(bo->base.resv);
> - if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
> - || !list_empty(&bo->ddestroy))
> + if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT || bo->deleted)
> ret = true;
> *locked = false;
> if (busy)
> @@ -840,6 +806,11 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
> dma_resv_unlock(bo->base.resv);
> continue;
> }
> + if (!ttm_bo_get_unless_zero(bo)) {
> + if (locked)
> + dma_resv_unlock(bo->base.resv);
> + continue;
> + }
> break;
> }
>
> @@ -851,21 +822,19 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
> }
>
> if (!bo) {
> - if (busy_bo)
> - kref_get(&busy_bo->list_kref);
> + if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
> + busy_bo = NULL;
> spin_unlock(&ttm_bo_glob.lru_lock);
> ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
> if (busy_bo)
> - kref_put(&busy_bo->list_kref, ttm_bo_release_list);
> + ttm_bo_put(busy_bo);
> return ret;
> }
>
> - kref_get(&bo->list_kref);
> -
> - if (!list_empty(&bo->ddestroy)) {
> + if (bo->deleted) {
> ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
> ctx->no_wait_gpu, locked);
> - kref_put(&bo->list_kref, ttm_bo_release_list);
> + ttm_bo_put(bo);
> return ret;
> }
>
> @@ -875,7 +844,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
> if (locked)
> ttm_bo_unreserve(bo);
>
> - kref_put(&bo->list_kref, ttm_bo_release_list);
> + ttm_bo_put(bo);
> return ret;
> }
>
> @@ -1279,7 +1248,6 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
> bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
>
> kref_init(&bo->kref);
> - kref_init(&bo->list_kref);
> INIT_LIST_HEAD(&bo->lru);
> INIT_LIST_HEAD(&bo->ddestroy);
> INIT_LIST_HEAD(&bo->swap);
> @@ -1799,11 +1767,18 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
> spin_lock(&glob->lru_lock);
> for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
> list_for_each_entry(bo, &glob->swap_lru[i], swap) {
> - if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
> - NULL)) {
> - ret = 0;
> - break;
> + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
> + NULL))
> + continue;
> +
> + if (!ttm_bo_get_unless_zero(bo)) {
> + if (locked)
> + dma_resv_unlock(bo->base.resv);
> + continue;
> }
> +
> + ret = 0;
> + break;
> }
> if (!ret)
> break;
> @@ -1814,11 +1789,9 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
> return ret;
> }
>
> - kref_get(&bo->list_kref);
> -
> - if (!list_empty(&bo->ddestroy)) {
> + if (bo->deleted) {
> ret = ttm_bo_cleanup_refs(bo, false, false, locked);
> - kref_put(&bo->list_kref, ttm_bo_release_list);
> + ttm_bo_put(bo);
> return ret;
> }
>
> @@ -1872,7 +1845,7 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
> */
> if (locked)
> dma_resv_unlock(bo->base.resv);
> - kref_put(&bo->list_kref, ttm_bo_release_list);
> + ttm_bo_put(bo);
> return ret;
> }
> EXPORT_SYMBOL(ttm_bo_swapout);
> diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
> index 86d152472f38..c8e359ded1df 100644
> --- a/drivers/gpu/drm/ttm/ttm_bo_util.c
> +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
> @@ -507,7 +507,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
> fbo->base.moving = NULL;
> drm_vma_node_reset(&fbo->base.base.vma_node);
>
> - kref_init(&fbo->base.list_kref);
> kref_init(&fbo->base.kref);
> fbo->base.destroy = &ttm_transfered_destroy;
> fbo->base.acc_size = 0;
> diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
> index 66ca49db9633..b9bc1b00142e 100644
> --- a/include/drm/ttm/ttm_bo_api.h
> +++ b/include/drm/ttm/ttm_bo_api.h
> @@ -135,18 +135,14 @@ struct ttm_tt;
> * @num_pages: Actual number of pages.
> * @acc_size: Accounted size for this object.
> * @kref: Reference count of this buffer object. When this refcount reaches
> - * zero, the object is put on the delayed delete list.
> - * @list_kref: List reference count of this buffer object. This member is
> - * used to avoid destruction while the buffer object is still on a list.
> - * Lru lists may keep one refcount, the delayed delete list, and kref != 0
> - * keeps one refcount. When this refcount reaches zero,
> - * the object is destroyed.
> + * zero, the object is destroyed or put on the delayed delete list.
> * @mem: structure describing current placement.
> * @persistent_swap_storage: Usually the swap storage is deleted for buffers
> * pinned in physical memory. If this behaviour is not desired, this member
> * holds a pointer to a persistent shmem object.
> * @ttm: TTM structure holding system pages.
> * @evicted: Whether the object was evicted without user-space knowing.
> + * @deleted: True if the object is only a zombie and already deleted.
> * @lru: List head for the lru list.
> * @ddestroy: List head for the delayed destroy list.
> * @swap: List head for swap LRU list.
> @@ -183,9 +179,7 @@ struct ttm_buffer_object {
> /**
> * Members not needing protection.
> */
> -
> struct kref kref;
> - struct kref list_kref;
>
> /**
> * Members protected by the bo::resv::reserved lock.
> @@ -195,6 +189,7 @@ struct ttm_buffer_object {
> struct file *persistent_swap_storage;
> struct ttm_tt *ttm;
> bool evicted;
> + bool deleted;
>
> /**
> * Members protected by the bdev::lru_lock.
More information about the amd-gfx
mailing list