[PATCH] vmaref
Chris Wilson
chris at chris-wilson.co.uk
Sat May 23 14:44:02 UTC 2020
---
drivers/gpu/drm/i915/display/intel_display.c | 33 ++--
.../drm/i915/display/intel_display_types.h | 2 +-
drivers/gpu/drm/i915/display/intel_dsb.c | 2 +-
drivers/gpu/drm/i915/display/intel_overlay.c | 29 ++-
.../gpu/drm/i915/gem/i915_gem_client_blt.c | 1 +
drivers/gpu/drm/i915/gem/i915_gem_context.c | 50 ++---
drivers/gpu/drm/i915/gem/i915_gem_domain.c | 4 +-
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 48 ++---
drivers/gpu/drm/i915/gem/i915_gem_mman.c | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_object.c | 27 +--
drivers/gpu/drm/i915/gem/i915_gem_object.h | 1 -
.../gpu/drm/i915/gem/i915_gem_object_blt.c | 17 +-
.../gpu/drm/i915/gem/selftests/huge_pages.c | 19 +-
.../i915/gem/selftests/i915_gem_client_blt.c | 3 +-
.../i915/gem/selftests/i915_gem_coherency.c | 5 +
.../drm/i915/gem/selftests/i915_gem_context.c | 6 +-
.../i915/gem/selftests/i915_gem_execbuffer.c | 4 +-
.../drm/i915/gem/selftests/i915_gem_mman.c | 66 ++++---
.../drm/i915/gem/selftests/igt_gem_utils.c | 5 +-
drivers/gpu/drm/i915/gt/gen6_ppgtt.c | 2 +-
drivers/gpu/drm/i915/gt/intel_engine_cs.c | 11 +-
drivers/gpu/drm/i915/gt/intel_gt.c | 30 ++-
drivers/gpu/drm/i915/gt/intel_gt_pm.c | 1 -
drivers/gpu/drm/i915/gt/intel_gt_types.h | 5 +-
drivers/gpu/drm/i915/gt/intel_gtt.c | 54 +++---
drivers/gpu/drm/i915/gt/intel_gtt.h | 20 +-
drivers/gpu/drm/i915/gt/intel_lrc.c | 36 ++--
drivers/gpu/drm/i915/gt/intel_renderstate.c | 13 +-
drivers/gpu/drm/i915/gt/intel_ring.c | 7 +-
.../gpu/drm/i915/gt/intel_ring_submission.c | 27 +--
drivers/gpu/drm/i915/gt/intel_timeline.c | 3 +-
drivers/gpu/drm/i915/gt/intel_workarounds.c | 14 +-
drivers/gpu/drm/i915/gt/selftest_engine_cs.c | 48 ++---
drivers/gpu/drm/i915/gt/selftest_hangcheck.c | 34 ++--
drivers/gpu/drm/i915/gt/selftest_lrc.c | 53 +++---
drivers/gpu/drm/i915/gt/selftest_mocs.c | 7 +-
.../drm/i915/gt/selftest_ring_submission.c | 11 +-
drivers/gpu/drm/i915/gt/selftest_rps.c | 7 +-
.../gpu/drm/i915/gt/selftest_workarounds.c | 13 +-
drivers/gpu/drm/i915/gt/uc/intel_guc.c | 11 +-
drivers/gpu/drm/i915/gvt/scheduler.c | 4 +-
drivers/gpu/drm/i915/i915_debugfs.c | 2 +
drivers/gpu/drm/i915/i915_drv.c | 1 -
drivers/gpu/drm/i915/i915_gem.c | 84 ++++-----
drivers/gpu/drm/i915/i915_perf.c | 28 +--
drivers/gpu/drm/i915/i915_vma.c | 175 +++++++++++-------
drivers/gpu/drm/i915/i915_vma.h | 38 ++--
.../gpu/drm/i915/selftests/i915_gem_evict.c | 26 ++-
drivers/gpu/drm/i915/selftests/i915_vma.c | 4 +
49 files changed, 544 insertions(+), 549 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 890d7fd907aa..62bb12939928 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -2291,7 +2291,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
*/
ret = i915_vma_pin_fence(vma);
if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
- i915_gem_object_unpin_from_display_plane(vma);
+ i915_vma_unpin_and_release(&vma, 0);
vma = ERR_PTR(ret);
goto err;
}
@@ -2300,7 +2300,6 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
*out_flags |= PLANE_HAS_FENCE;
}
- i915_vma_get(vma);
err:
atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
@@ -2309,13 +2308,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
{
- i915_gem_object_lock(vma->obj);
- if (flags & PLANE_HAS_FENCE)
- i915_vma_unpin_fence(vma);
- i915_gem_object_unpin_from_display_plane(vma);
- i915_gem_object_unlock(vma->obj);
-
- i915_vma_put(vma);
+ i915_vma_unpin_and_release(&vma, flags);
}
static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
@@ -3442,25 +3435,27 @@ initial_plane_vma(struct drm_i915_private *i915,
break;
default:
MISSING_CASE(plane_config->tiling);
- goto err_obj;
+ i915_gem_object_put(obj);
+ return NULL;
}
vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
+ i915_gem_object_put(obj);
if (IS_ERR(vma))
- goto err_obj;
+ return vma;
- if (i915_ggtt_pin(vma, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base))
- goto err_obj;
+ if (i915_ggtt_pin(vma, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base)) {
+ i915_vma_put(vma);
+ return NULL;
+ }
if (i915_gem_object_is_tiled(obj) &&
- !i915_vma_is_map_and_fenceable(vma))
- goto err_obj;
+ !i915_vma_is_map_and_fenceable(vma)) {
+ i915_vma_put(vma);
+ return NULL;
+ }
return vma;
-
-err_obj:
- i915_gem_object_put(obj);
- return NULL;
}
static bool
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 83e2f0bd2387..dca677746f93 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -523,7 +523,7 @@ struct intel_plane_state {
struct i915_ggtt_view view;
struct i915_vma *vma;
unsigned long flags;
-#define PLANE_HAS_FENCE BIT(0)
+#define PLANE_HAS_FENCE I915_VMA_RELEASE_FENCE
struct {
u32 offset;
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index 29fec6a92d17..c8958e5005a9 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -120,9 +120,9 @@ intel_dsb_get(struct intel_crtc *crtc)
}
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
drm_err(&i915->drm, "Vma creation failed\n");
- i915_gem_object_put(obj);
goto out;
}
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 52b4f6193b4c..ffacbe0b599b 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -190,7 +190,7 @@ struct intel_overlay {
u32 brightness, contrast, saturation;
u32 old_xscale, old_yscale;
/* register access */
- struct drm_i915_gem_object *reg_bo;
+ struct i915_vma *reg_vma;
struct overlay_registers __iomem *regs;
u32 flip_addr;
/* flip handling */
@@ -302,10 +302,7 @@ static void intel_overlay_flip_prepare(struct intel_overlay *overlay,
INTEL_FRONTBUFFER_OVERLAY(pipe));
overlay->old_vma = overlay->vma;
- if (vma)
- overlay->vma = i915_vma_get(vma);
- else
- overlay->vma = NULL;
+ overlay->vma = vma;
}
/* overlay needs to be enabled in OCMD reg */
@@ -359,8 +356,7 @@ static void intel_overlay_release_old_vma(struct intel_overlay *overlay)
intel_frontbuffer_flip_complete(overlay->i915,
INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe));
- i915_gem_object_unpin_from_display_plane(vma);
- i915_vma_put(vma);
+ i915_vma_unpin_and_release(&vma, 0);
}
static void
@@ -860,7 +856,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
return 0;
out_unpin:
- i915_gem_object_unpin_from_display_plane(vma);
+ i915_vma_unpin_and_release(&vma, 0);
out_pin_section:
atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
@@ -1322,10 +1318,9 @@ static int get_registers(struct intel_overlay *overlay, bool use_phys)
return PTR_ERR(obj);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_put_bo;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
if (use_phys)
overlay->flip_addr = sg_dma_address(obj->mm.pages->sgl);
@@ -1336,14 +1331,14 @@ static int get_registers(struct intel_overlay *overlay, bool use_phys)
if (IS_ERR(overlay->regs)) {
err = PTR_ERR(overlay->regs);
- goto err_put_bo;
+ goto err_vma;
}
- overlay->reg_bo = obj;
+ overlay->reg_vma = vma;
return 0;
-err_put_bo:
- i915_gem_object_put(obj);
+err_vma:
+ i915_vma_put(vma);
return err;
}
@@ -1408,7 +1403,7 @@ void intel_overlay_cleanup(struct drm_i915_private *dev_priv)
*/
drm_WARN_ON(&dev_priv->drm, overlay->active);
- i915_gem_object_put(overlay->reg_bo);
+ i915_vma_put(overlay->reg_vma);
i915_active_fini(&overlay->last_flip);
kfree(overlay);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
index d3a86a4d5c04..5905d24a527d 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
@@ -86,6 +86,7 @@ static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
static void destroy_sleeve(struct i915_sleeve *sleeve)
{
+ i915_vma_put(sleeve->vma);
kfree(sleeve);
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index f5d59d18cd5b..e84c69ffccce 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -106,35 +106,39 @@ static void lut_close(struct i915_gem_context *ctx)
rcu_read_lock();
radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) {
struct i915_vma *vma = rcu_dereference_raw(*slot);
- struct drm_i915_gem_object *obj = vma->obj;
struct i915_lut_handle *lut;
- if (!kref_get_unless_zero(&obj->base.refcount))
+ vma = i915_vma_tryget(vma);
+ if (!vma)
continue;
- rcu_read_unlock();
- i915_gem_object_lock(obj);
- list_for_each_entry(lut, &obj->lut_list, obj_link) {
- if (lut->ctx != ctx)
- continue;
-
- if (lut->handle != iter.index)
- continue;
-
- list_del(&lut->obj_link);
- break;
- }
- i915_gem_object_unlock(obj);
- rcu_read_lock();
-
- if (&lut->obj_link != &obj->lut_list) {
- i915_lut_handle_free(lut);
- radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
- i915_vma_close(vma);
- i915_gem_object_put(obj);
+ if (vma->obj) {
+ struct drm_i915_gem_object *obj = vma->obj;
+
+ rcu_read_unlock();
+ i915_gem_object_lock(obj);
+ list_for_each_entry(lut, &obj->lut_list, obj_link) {
+ if (lut->ctx != ctx)
+ continue;
+
+ if (lut->handle != iter.index)
+ continue;
+
+ list_del(&lut->obj_link);
+ break;
+ }
+ i915_gem_object_unlock(obj);
+ rcu_read_lock();
+
+ if (&lut->obj_link != &obj->lut_list) {
+ i915_lut_handle_free(lut);
+ radix_tree_iter_delete(&ctx->handles_vma,
+ &iter, slot);
+ i915_vma_put(vma);
+ }
}
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
}
rcu_read_unlock();
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index 7f76fc68f498..4ea9bb88089a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -399,10 +399,8 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
void
i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
{
- /* Bump the LRU to try and avoid premature eviction whilst flipping */
- i915_gem_object_bump_inactive_ggtt(vma->obj);
-
i915_vma_unpin(vma);
+ i915_vma_put(vma);
}
/**
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index e4fb6c372537..da420ea4cdbb 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -800,9 +800,7 @@ static int __eb_add_lut(struct i915_execbuffer *eb,
if (unlikely(!lut))
return -ENOMEM;
- i915_vma_get(vma);
- if (!atomic_fetch_inc(&vma->open_count))
- i915_vma_reopen(vma);
+ i915_vma_open(vma);
lut->handle = handle;
lut->ctx = ctx;
@@ -826,15 +824,10 @@ static int __eb_add_lut(struct i915_execbuffer *eb,
}
mutex_unlock(&ctx->mutex);
}
- if (unlikely(err))
- goto err;
-
- return 0;
-
-err:
- i915_vma_close(vma);
- i915_vma_put(vma);
- i915_lut_handle_free(lut);
+ if (unlikely(err)) {
+ i915_vma_close(vma);
+ i915_lut_handle_free(lut);
+ }
return err;
}
@@ -858,16 +851,15 @@ static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle)
return ERR_PTR(-ENOENT);
vma = i915_vma_instance(obj, eb->context->vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = __eb_add_lut(eb, handle, vma);
if (likely(!err))
return vma;
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
if (err != -EEXIST)
return ERR_PTR(err);
} while (1);
@@ -999,7 +991,7 @@ static int reloc_gpu_chain(struct reloc_cache *cache)
err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
if (err)
- goto out_pool;
+ goto out_batch;
GEM_BUG_ON(cache->rq_size + RELOC_TAIL > PAGE_SIZE / sizeof(u32));
cmd = cache->rq_cmd + cache->rq_size;
@@ -1026,7 +1018,7 @@ static int reloc_gpu_chain(struct reloc_cache *cache)
}
i915_vma_unpin(batch);
if (err)
- goto out_pool;
+ goto out_batch;
cmd = i915_gem_object_pin_map(batch->obj,
cache->has_llc ?
@@ -1042,6 +1034,8 @@ static int reloc_gpu_chain(struct reloc_cache *cache)
cache->rq_size = 0;
cache->rq_vma = batch;
+out_batch:
+ i915_vma_put(batch);
out_pool:
intel_gt_buffer_pool_put(pool);
return err;
@@ -1117,7 +1111,11 @@ static void reloc_cache_reset(struct reloc_cache *cache)
drm_mm_remove_node(&cache->node);
mutex_unlock(&ggtt->vm.mutex);
} else {
- i915_vma_unpin((struct i915_vma *)cache->node.mm);
+ struct i915_vma *vma =
+ (struct i915_vma *)cache->node.mm;
+
+ i915_vma_unpin(vma);
+ i915_vma_put(vma);
}
}
@@ -1316,7 +1314,7 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
if (err)
- goto err_unmap;
+ goto err_batch;
if (engine == eb->context->engine) {
rq = i915_request_create(eb->context);
@@ -1369,6 +1367,8 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
i915_request_add(rq);
err_unpin:
i915_vma_unpin(batch);
+err_batch:
+ i915_vma_put(batch);
err_unmap:
i915_gem_object_unpin_map(pool->obj);
out_pool:
@@ -1944,8 +1944,10 @@ shadow_batch_pin(struct drm_i915_gem_object *obj,
return vma;
err = i915_vma_pin(vma, 0, 0, flags);
- if (err)
+ if (err) {
+ i915_vma_put(vma);
return ERR_PTR(err);
+ }
return vma;
}
@@ -2721,7 +2723,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
* batch" bit. Hence we need to pin secure batches into the global gtt.
* hsw should have this fixed, but bdw mucks it up again. */
- batch = eb.batch->vma;
+ batch = i915_vma_get(eb.batch->vma);
if (eb.batch_flags & I915_DISPATCH_SECURE) {
struct i915_vma *vma;
@@ -2741,6 +2743,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
goto err_parse;
}
+ i915_vma_put(batch);
batch = vma;
}
@@ -2819,6 +2822,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
err_parse:
if (batch->private)
intel_gt_buffer_pool_put(batch->private);
+ i915_vma_put(batch);
err_vma:
if (eb.trampoline)
i915_vma_unpin(eb.trampoline);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 9d306dc9849d..0ef363d11b08 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -386,7 +386,7 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)
err_fence:
i915_vma_unpin_fence(vma);
err_unpin:
- __i915_vma_unpin(vma);
+ i915_vma_unpin_and_release(&vma, 0);
err_reset:
intel_gt_reset_unlock(ggtt->vm.gt, srcu);
err_rpm:
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 99356c00c19e..672b8a7c0a98 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -134,14 +134,13 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
if (vma) {
GEM_BUG_ON(vma->obj != obj);
- GEM_BUG_ON(!atomic_read(&vma->open_count));
i915_vma_close(vma);
}
mutex_unlock(&ctx->mutex);
i915_gem_context_put(lut->ctx);
i915_lut_handle_free(lut);
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
}
}
@@ -168,31 +167,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
trace_i915_gem_object_destroy(obj);
- if (!list_empty(&obj->vma.list)) {
- struct i915_vma *vma;
-
- /*
- * Note that the vma keeps an object reference while
- * it is active, so it *should* not sleep while we
- * destroy it. Our debug code errs insits it *might*.
- * For the moment, play along.
- */
- spin_lock(&obj->vma.lock);
- while ((vma = list_first_entry_or_null(&obj->vma.list,
- struct i915_vma,
- obj_link))) {
- GEM_BUG_ON(vma->obj != obj);
- spin_unlock(&obj->vma.lock);
-
- __i915_vma_put(vma);
-
- spin_lock(&obj->vma.lock);
- }
- spin_unlock(&obj->vma.lock);
- }
-
i915_gem_object_release_mmap(obj);
-
rbtree_postorder_for_each_entry_safe(mmo, mn,
&obj->mmo.offsets,
offset) {
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 2faa481cc18f..7351e18bad3f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -445,7 +445,6 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
u32 alignment,
const struct i915_ggtt_view *view,
unsigned int flags);
-void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma);
void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj);
void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c
index f457d7130491..7b2ab0586122 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c
@@ -123,6 +123,7 @@ void intel_emit_vma_release(struct intel_context *ce, struct i915_vma *vma)
{
i915_vma_unpin(vma);
intel_gt_buffer_pool_put(vma->private);
+ i915_vma_put(vma);
intel_engine_pm_put(ce->engine);
}
@@ -141,7 +142,7 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj,
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (unlikely(err))
- return err;
+ goto out_vma;
if (obj->cache_dirty & ~obj->cache_coherent) {
i915_gem_object_lock(obj);
@@ -195,6 +196,8 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj,
intel_emit_vma_release(ce, batch);
out_unpin:
i915_vma_unpin(vma);
+out_vma:
+ i915_vma_put(vma);
return err;
}
@@ -305,11 +308,13 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce,
err = i915_vma_pin(batch, 0, 0, PIN_USER);
if (unlikely(err))
- goto out_put;
+ goto out_batch;
batch->private = pool;
return batch;
+out_batch:
+ i915_vma_put(batch);
out_put:
intel_gt_buffer_pool_put(pool);
out_pm:
@@ -344,7 +349,7 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src,
err = i915_vma_pin(vma[0], 0, 0, PIN_USER);
if (unlikely(err))
- return err;
+ goto out_src;
vma[1] = i915_vma_instance(dst, vm, NULL);
if (IS_ERR(vma[1]))
@@ -352,7 +357,7 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src,
err = i915_vma_pin(vma[1], 0, 0, PIN_USER);
if (unlikely(err))
- goto out_unpin_src;
+ goto out_dst;
batch = intel_emit_vma_copy_blt(ce, vma[0], vma[1]);
if (IS_ERR(batch)) {
@@ -408,8 +413,12 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src,
intel_emit_vma_release(ce, batch);
out_unpin_dst:
i915_vma_unpin(vma[1]);
+out_dst:
+ i915_vma_put(vma[1]);
out_unpin_src:
i915_vma_unpin(vma[0]);
+out_src:
+ i915_vma_put(vma[0]);
return err;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index a0ed2fab0ff3..e792305b3b1a 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -409,11 +409,13 @@ static int igt_mock_exhaust_device_supported_pages(void *arg)
if (obj->base.size != combination) {
pr_err("obj->base.size=%zu, expected=%u\n",
obj->base.size, combination);
+ i915_gem_object_put(obj);
err = -EINVAL;
- goto out_put;
+ goto out_device;
}
vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
goto out_put;
@@ -432,7 +434,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg)
}
i915_vma_unpin(vma);
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
if (err)
goto out_device;
@@ -442,7 +444,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg)
goto out_device;
out_put:
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
out_device:
mkwrite_device_info(i915)->page_sizes = saved_mask;
@@ -456,7 +458,6 @@ static int igt_mock_memory_region_huge_pages(void *arg)
struct drm_i915_private *i915 = ppgtt->vm.i915;
unsigned long supported = INTEL_INFO(i915)->page_sizes;
struct intel_memory_region *mem;
- struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int bit;
int err = 0;
@@ -473,6 +474,8 @@ static int igt_mock_memory_region_huge_pages(void *arg)
int i;
for (i = 0; i < ARRAY_SIZE(flags); ++i) {
+ struct drm_i915_gem_object *obj;
+
obj = i915_gem_object_create_region(mem, page_size,
flags[i]);
if (IS_ERR(obj)) {
@@ -481,9 +484,10 @@ static int igt_mock_memory_region_huge_pages(void *arg)
}
vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
- goto out_put;
+ goto out_region;
}
err = i915_vma_pin(vma, 0, 0, PIN_USER);
@@ -521,7 +525,7 @@ static int igt_mock_memory_region_huge_pages(void *arg)
out_unpin:
i915_vma_unpin(vma);
out_put:
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
out_region:
intel_memory_region_put(mem);
return err;
@@ -533,6 +537,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg)
struct drm_i915_private *i915 = ppgtt->vm.i915;
unsigned long supported = INTEL_INFO(i915)->page_sizes;
struct drm_i915_gem_object *obj;
+ struct i915_vma *vma;
int bit;
int err;
@@ -552,7 +557,6 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg)
unsigned int offset;
unsigned int size =
round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
- struct i915_vma *vma;
obj = fake_huge_pages_object(i915, size, true);
if (IS_ERR(obj))
@@ -561,6 +565,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg)
if (obj->base.size != size) {
pr_err("obj->base.size=%zu, expected=%u\n",
obj->base.size, size);
+ i915_gem_object_put(obj);
err = -EINVAL;
goto out_put;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
index 8fe3ad2ee34e..e307f6cbea18 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
@@ -243,8 +243,7 @@ __create_vma(struct tiled_blits *t, size_t size, bool lmem)
return ERR_CAST(obj);
vma = i915_vma_instance(obj, t->ce->vm, NULL);
- if (IS_ERR(vma))
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
return vma;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c
index 87d7d8aa080f..e5a6368a126d 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c
@@ -106,6 +106,7 @@ static int gtt_set(struct context *ctx, unsigned long offset, u32 v)
out_rpm:
intel_gt_pm_put(vma->vm->gt);
+ i915_vma_put(vma);
return err;
}
@@ -139,6 +140,7 @@ static int gtt_get(struct context *ctx, unsigned long offset, u32 *v)
out_rpm:
intel_gt_pm_put(vma->vm->gt);
+ i915_vma_put(vma);
return err;
}
@@ -206,6 +208,7 @@ static int gpu_set(struct context *ctx, unsigned long offset, u32 v)
rq = intel_engine_create_kernel_request(ctx->engine);
if (IS_ERR(rq)) {
i915_vma_unpin(vma);
+ i915_vma_put(vma);
return PTR_ERR(rq);
}
@@ -213,6 +216,7 @@ static int gpu_set(struct context *ctx, unsigned long offset, u32 v)
if (IS_ERR(cs)) {
i915_request_add(rq);
i915_vma_unpin(vma);
+ i915_vma_put(vma);
return PTR_ERR(cs);
}
@@ -240,6 +244,7 @@ static int gpu_set(struct context *ctx, unsigned long offset, u32 v)
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
i915_vma_unlock(vma);
i915_vma_unpin(vma);
+ i915_vma_put(vma);
i915_request_add(rq);
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
index b81978890641..4e64f7f512d7 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c
@@ -435,7 +435,7 @@ static int gpu_fill(struct intel_context *ce,
err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
if (err)
- return err;
+ goto out_vma;
/*
* Within the GTT the huge objects maps every page onto
@@ -450,8 +450,10 @@ static int gpu_fill(struct intel_context *ce,
(dw * sizeof(u32)),
real_page_count(obj),
dw);
- i915_vma_unpin(vma);
+ i915_vma_unpin(vma);
+out_vma:
+ i915_vma_put(vma);
return err;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c
index a49016f8ee0d..68a0d8c68b9f 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c
@@ -34,7 +34,7 @@ static int __igt_gpu_reloc(struct i915_execbuffer *eb,
err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
if (err)
- return err;
+ goto out_vma;
/* 8-Byte aligned */
if (!__reloc_entry_gpu(eb, vma,
@@ -102,6 +102,8 @@ static int __igt_gpu_reloc(struct i915_execbuffer *eb,
i915_request_put(rq);
unpin_vma:
i915_vma_unpin(vma);
+out_vma:
+ i915_vma_put(vma);
return err;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
index 9c7402ce5bf9..250664283614 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
@@ -164,7 +164,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj,
kunmap(p);
out:
- __i915_vma_put(vma);
+ i915_vma_put(vma);
return err;
}
@@ -223,6 +223,7 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj,
if (IS_ERR(io)) {
pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
page, (int)PTR_ERR(io));
+ i915_vma_put(vma);
return PTR_ERR(io);
}
@@ -230,36 +231,35 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj,
i915_vma_unpin_iomap(vma);
offset = tiled_offset(tile, page << PAGE_SHIFT);
- if (offset >= obj->base.size)
- continue;
-
- intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt);
-
- p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
- cpu = kmap(p) + offset_in_page(offset);
- drm_clflush_virt_range(cpu, sizeof(*cpu));
- if (*cpu != (u32)page) {
- pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
- page, n,
- view.partial.offset,
- view.partial.size,
- vma->size >> PAGE_SHIFT,
- tile->tiling ? tile_row_pages(obj) : 0,
- vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
- offset >> PAGE_SHIFT,
- (unsigned int)offset_in_page(offset),
- offset,
- (u32)page, *cpu);
- err = -EINVAL;
+ if (offset < obj->base.size) {
+ intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt);
+
+ p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
+ cpu = kmap(p) + offset_in_page(offset);
+ drm_clflush_virt_range(cpu, sizeof(*cpu));
+ if (*cpu != (u32)page) {
+ pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
+ page, n,
+ view.partial.offset,
+ view.partial.size,
+ vma->size >> PAGE_SHIFT,
+ tile->tiling ? tile_row_pages(obj) : 0,
+ vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
+ offset >> PAGE_SHIFT,
+ (unsigned int)offset_in_page(offset),
+ offset,
+ (u32)page, *cpu);
+ err = -EINVAL;
+ }
+ *cpu = 0;
+ drm_clflush_virt_range(cpu, sizeof(*cpu));
+ kunmap(p);
}
- *cpu = 0;
- drm_clflush_virt_range(cpu, sizeof(*cpu));
- kunmap(p);
+
+ i915_vma_put(vma);
if (err)
return err;
- __i915_vma_put(vma);
-
if (igt_timeout(end_time,
"%s: timed out after tiling=%d stride=%d\n",
__func__, tile->tiling, tile->stride))
@@ -535,12 +535,15 @@ static int make_obj_busy(struct drm_i915_gem_object *obj)
return PTR_ERR(vma);
err = i915_vma_pin(vma, 0, 0, PIN_USER);
- if (err)
+ if (err) {
+ i915_vma_put(vma);
return err;
+ }
rq = intel_engine_create_kernel_request(engine);
if (IS_ERR(rq)) {
i915_vma_unpin(vma);
+ i915_vma_put(vma);
return PTR_ERR(rq);
}
@@ -553,6 +556,7 @@ static int make_obj_busy(struct drm_i915_gem_object *obj)
i915_request_add(rq);
i915_vma_unpin(vma);
+ i915_vma_put(vma);
if (err)
return err;
}
@@ -749,6 +753,7 @@ static int gtt_set(struct drm_i915_gem_object *obj)
out:
intel_gt_pm_put(vma->vm->gt);
+ i915_vma_put(vma);
return err;
}
@@ -779,6 +784,7 @@ static int gtt_check(struct drm_i915_gem_object *obj)
out:
intel_gt_pm_put(vma->vm->gt);
+ i915_vma_put(vma);
return err;
}
@@ -1132,7 +1138,7 @@ static int __igt_mmap_gpu(struct drm_i915_private *i915,
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (err)
- goto out_unmap;
+ goto out_vma;
rq = i915_request_create(engine->kernel_context);
if (IS_ERR(rq)) {
@@ -1166,6 +1172,8 @@ static int __igt_mmap_gpu(struct drm_i915_private *i915,
out_unpin:
i915_vma_unpin(vma);
+out_vma:
+ i915_vma_put(vma);
if (err)
goto out_unmap;
}
diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
index e21b5023ca7d..8e57dd5f4bdf 100644
--- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
+++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c
@@ -97,10 +97,13 @@ igt_emit_store_dw(struct i915_vma *vma,
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (err)
- goto err;
+ goto err_vma;
+ i915_gem_object_put(obj);
return vma;
+err_vma:
+ i915_vma_put(vma);
err:
i915_gem_object_put(obj);
return ERR_PTR(err);
diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
index f4fec7eb4064..34cfb79bc0ad 100644
--- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
@@ -276,7 +276,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
- __i915_vma_put(ppgtt->vma);
+ i915_vma_put(ppgtt->vma);
gen6_ppgtt_free_pd(ppgtt);
free_scratch(vm);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index da5b61085257..b1ab897788bf 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -535,7 +535,7 @@ static void cleanup_status_page(struct intel_engine_cs *engine)
i915_vma_unpin(vma);
i915_gem_object_unpin_map(vma->obj);
- i915_gem_object_put(vma->obj);
+ i915_vma_put(vma);
}
static int pin_ggtt_status_page(struct intel_engine_cs *engine,
@@ -586,10 +586,9 @@ static int init_status_page(struct intel_engine_cs *engine)
i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
if (IS_ERR(vaddr)) {
@@ -611,7 +610,7 @@ static int init_status_page(struct intel_engine_cs *engine)
err_unpin:
i915_gem_object_unpin_map(obj);
err:
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ret;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index f069551e412f..dac254673c2e 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -26,9 +26,6 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
spin_lock_init(>->irq_lock);
- INIT_LIST_HEAD(>->closed_vma);
- spin_lock_init(>->closed_lock);
-
intel_gt_init_buffer_pool(gt);
intel_gt_init_reset(gt);
intel_gt_init_requests(gt);
@@ -37,6 +34,8 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
intel_rps_init_early(>->rps);
intel_uc_init_early(>->uc);
+
+ i915_vma_clock_init_early(>->vma_clock);
}
void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt)
@@ -332,7 +331,7 @@ static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
struct drm_i915_private *i915 = gt->i915;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
- int ret;
+ int err;
obj = i915_gem_object_create_stolen(i915, size);
if (IS_ERR(obj))
@@ -343,22 +342,18 @@ static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
}
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err_unref;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
- ret = i915_ggtt_pin(vma, 0, PIN_HIGH);
- if (ret)
- goto err_unref;
+ err = i915_ggtt_pin(vma, 0, PIN_HIGH);
+ if (err) {
+ i915_vma_put(vma);
+ return err;
+ }
gt->scratch = i915_vma_make_unshrinkable(vma);
-
return 0;
-
-err_unref:
- i915_gem_object_put(obj);
- return ret;
}
static void intel_gt_fini_scratch(struct intel_gt *gt)
@@ -595,6 +590,7 @@ int intel_gt_init(struct intel_gt *gt)
void intel_gt_driver_remove(struct intel_gt *gt)
{
__intel_gt_disable(gt);
+ i915_vma_clock_flush(>->vma_clock);
intel_uc_driver_remove(>->uc);
@@ -628,6 +624,8 @@ void intel_gt_driver_release(struct intel_gt *gt)
void intel_gt_driver_late_release(struct intel_gt *gt)
{
+ i915_vma_clock_flush(>->vma_clock);
+
/* We need to wait for inflight RCU frees to release their grip */
rcu_barrier();
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
index 6bdb434a442d..f0809578f894 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
@@ -81,7 +81,6 @@ static int __gt_park(struct intel_wakeref *wf)
intel_gt_park_requests(gt);
- i915_vma_parked(gt);
i915_pmu_gt_parked(i915);
intel_rps_park(>->rps);
intel_rc6_park(>->rc6);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h
index 0cc1d6b185dc..1681e952d186 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h
@@ -59,9 +59,6 @@ struct intel_gt {
struct intel_wakeref wakeref;
atomic_t user_wakeref;
- struct list_head closed_vma;
- spinlock_t closed_lock; /* guards the list of closed_vma */
-
ktime_t last_init_time;
struct intel_reset reset;
@@ -108,6 +105,8 @@ struct intel_gt {
*/
struct intel_gt_buffer_pool buffer_pool;
+ struct i915_vma_clock vma_clock;
+
struct i915_vma *scratch;
};
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c
index 2a72cce63fd9..2186b51c6a5e 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.c
@@ -167,31 +167,6 @@ static void vm_free_page(struct i915_address_space *vm, struct page *page)
spin_unlock(&vm->free_pages.lock);
}
-void __i915_vm_close(struct i915_address_space *vm)
-{
- struct i915_vma *vma, *vn;
-
- if (!atomic_dec_and_mutex_lock(&vm->open, &vm->mutex))
- return;
-
- list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
- struct drm_i915_gem_object *obj = vma->obj;
-
- /* Keep the obj (and hence the vma) alive as _we_ destroy it */
- if (!kref_get_unless_zero(&obj->base.refcount))
- continue;
-
- atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
- WARN_ON(__i915_vma_unbind(vma));
- __i915_vma_put(vma);
-
- i915_gem_object_put(obj);
- }
- GEM_BUG_ON(!list_empty(&vm->bound_list));
-
- mutex_unlock(&vm->mutex);
-}
-
void i915_address_space_fini(struct i915_address_space *vm)
{
spin_lock(&vm->free_pages.lock);
@@ -227,12 +202,41 @@ void i915_vm_release(struct kref *kref)
queue_rcu_work(vm->i915->wq, &vm->rcu);
}
+static void i915_vm_release_vma(struct work_struct *wrk)
+{
+ struct i915_address_space *vm =
+ container_of(wrk, typeof(*vm), release.work);
+ struct i915_vma *vma, *next;
+ LIST_HEAD(list);
+
+ spin_lock_irq(&vm->release.lock);
+ list_splice_init(&vm->release.list, &list);
+ spin_unlock_irq(&vm->release.lock);
+
+ mutex_lock(&vm->mutex);
+ list_for_each_entry(vma, &list, closed_link) {
+ if (drm_mm_node_allocated(&vma->node)) {
+ atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
+ __i915_vma_unbind(vma);
+ GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+ }
+ }
+ mutex_unlock(&vm->mutex);
+
+ list_for_each_entry_safe(vma, next, &list, closed_link)
+ i915_vma_release(&vma->ref);
+}
+
void i915_address_space_init(struct i915_address_space *vm, int subclass)
{
kref_init(&vm->ref);
INIT_RCU_WORK(&vm->rcu, __i915_vm_release);
atomic_set(&vm->open, 1);
+ spin_lock_init(&vm->release.lock);
+ INIT_LIST_HEAD(&vm->release.list);
+ INIT_WORK(&vm->release.work, i915_vm_release_vma);
+
/*
* The vm->mutex must be reclaim safe (for use in the shrinker).
* Do a dummy acquire now under fs_reclaim so that any allocation
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h
index d93ebdf3fa0e..259a8945a014 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.h
@@ -263,6 +263,12 @@ struct i915_address_space {
*/
struct list_head bound_list;
+ struct {
+ spinlock_t lock;
+ struct list_head list;
+ struct work_struct work;
+ } release;
+
struct pagestash free_pages;
/* Global GTT */
@@ -415,23 +421,11 @@ i915_vm_open(struct i915_address_space *vm)
return i915_vm_get(vm);
}
-static inline bool
-i915_vm_tryopen(struct i915_address_space *vm)
-{
- if (atomic_add_unless(&vm->open, 1, 0))
- return i915_vm_get(vm);
-
- return false;
-}
-
-void __i915_vm_close(struct i915_address_space *vm);
-
static inline void
i915_vm_close(struct i915_address_space *vm)
{
GEM_BUG_ON(!atomic_read(&vm->open));
- __i915_vm_close(vm);
-
+ atomic_dec(&vm->open);
i915_vm_put(vm);
}
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index de5be57ed6d2..7c61f8d9085f 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -3834,21 +3834,18 @@ static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
return PTR_ERR(obj);
vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
err = i915_ggtt_pin(vma, 0, PIN_HIGH);
- if (err)
- goto err;
+ if (err) {
+ i915_vma_put(vma);
+ return err;
+ }
engine->wa_ctx.vma = vma;
return 0;
-
-err:
- i915_gem_object_put(obj);
- return err;
}
static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine)
@@ -5264,10 +5261,9 @@ static int __execlists_context_alloc(struct intel_context *ce,
return PTR_ERR(ctx_obj);
vma = i915_vma_instance(ctx_obj, &engine->gt->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto error_deref_obj;
- }
+ i915_gem_object_put(ctx_obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
if (!ce->timeline) {
struct intel_timeline *tl;
@@ -5284,7 +5280,7 @@ static int __execlists_context_alloc(struct intel_context *ce,
tl = intel_timeline_create(engine->gt, hwsp);
if (IS_ERR(tl)) {
ret = PTR_ERR(tl);
- goto error_deref_obj;
+ goto err_vma;
}
ce->timeline = tl;
@@ -5293,14 +5289,14 @@ static int __execlists_context_alloc(struct intel_context *ce,
ring = intel_engine_create_ring(engine, (unsigned long)ce->ring);
if (IS_ERR(ring)) {
ret = PTR_ERR(ring);
- goto error_deref_obj;
+ goto err_vma;
}
ret = populate_lr_context(ce, ctx_obj, engine, ring);
if (ret) {
drm_dbg(&engine->i915->drm,
"Failed to populate LRC: %d\n", ret);
- goto error_ring_free;
+ goto err_ring;
}
ce->ring = ring;
@@ -5308,10 +5304,10 @@ static int __execlists_context_alloc(struct intel_context *ce,
return 0;
-error_ring_free:
+err_ring:
intel_ring_put(ring);
-error_deref_obj:
- i915_gem_object_put(ctx_obj);
+err_vma:
+ i915_vma_put(vma);
return ret;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.c b/drivers/gpu/drm/i915/gt/intel_renderstate.c
index f59e7875cc5e..167d23e74e78 100644
--- a/drivers/gpu/drm/i915/gt/intel_renderstate.c
+++ b/drivers/gpu/drm/i915/gt/intel_renderstate.c
@@ -187,14 +187,13 @@ int intel_renderstate_init(struct intel_renderstate *so,
return PTR_ERR(obj);
so->vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
- if (IS_ERR(so->vma)) {
- err = PTR_ERR(so->vma);
- goto err_obj;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(so->vma))
+ return PTR_ERR(so->vma);
err = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
if (err)
- goto err_obj;
+ goto err_vma;
err = render_state_setup(so, engine->i915);
if (err)
@@ -204,8 +203,8 @@ int intel_renderstate_init(struct intel_renderstate *so,
err_unpin:
i915_vma_unpin(so->vma);
-err_obj:
- i915_gem_object_put(obj);
+err_vma:
+ i915_vma_put(so->vma);
so->vma = NULL;
return err;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c
index 8cda1b7e17ba..4965b0b7c79b 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring.c
@@ -117,13 +117,8 @@ static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
i915_gem_object_set_readonly(obj);
vma = i915_vma_instance(obj, vm, NULL);
- if (IS_ERR(vma))
- goto err;
-
- return vma;
-
-err:
i915_gem_object_put(obj);
+
return vma;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index ca7286e58409..803e9938de34 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -1217,7 +1217,6 @@ alloc_context_vma(struct intel_engine_cs *engine)
struct drm_i915_private *i915 = engine->i915;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
- int err;
obj = i915_gem_object_create_shmem(i915, engine->context_size);
if (IS_ERR(obj))
@@ -1246,8 +1245,8 @@ alloc_context_vma(struct intel_engine_cs *engine)
vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
if (IS_ERR(vaddr)) {
- err = PTR_ERR(vaddr);
- goto err_obj;
+ i915_gem_object_put(obj);
+ return ERR_CAST(vaddr);
}
shmem_read(engine->default_state, 0,
@@ -1258,16 +1257,9 @@ alloc_context_vma(struct intel_engine_cs *engine)
}
vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
- }
+ i915_gem_object_put(obj);
return vma;
-
-err_obj:
- i915_gem_object_put(obj);
- return ERR_PTR(err);
}
static int ring_context_alloc(struct intel_context *ce)
@@ -1995,15 +1987,14 @@ static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
return PTR_ERR(obj);
vma = i915_vma_instance(obj, engine->gt->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
vma->private = intel_context_create(engine); /* dummy residuals */
if (IS_ERR(vma->private)) {
err = PTR_ERR(vma->private);
- goto err_obj;
+ goto err_vma;
}
err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
@@ -2025,8 +2016,8 @@ static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
i915_vma_unpin(vma);
err_private:
intel_context_put(vma->private);
-err_obj:
- i915_gem_object_put(obj);
+err_vma:
+ i915_vma_put(vma);
return err;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c
index 4546284fede1..ee4a11ebca98 100644
--- a/drivers/gpu/drm/i915/gt/intel_timeline.c
+++ b/drivers/gpu/drm/i915/gt/intel_timeline.c
@@ -39,8 +39,7 @@ static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma))
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
return vma;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
index fa1e15657663..08015a2d29b6 100644
--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
@@ -1643,21 +1643,19 @@ create_scratch(struct i915_address_space *vm, int count)
i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
vma = i915_vma_instance(obj, vm, NULL);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
+ return vma;
}
err = i915_vma_pin(vma, 0, 0,
i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
- if (err)
- goto err_obj;
+ if (err) {
+ i915_vma_put(vma);
+ return ERR_PTR(err);
+ }
return vma;
-
-err_obj:
- i915_gem_object_put(obj);
- return ERR_PTR(err);
}
static const struct {
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_cs.c b/drivers/gpu/drm/i915/gt/selftest_engine_cs.c
index f88e445a1cae..301f683b5fe3 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_cs.c
@@ -74,8 +74,8 @@ static struct i915_vma *create_empty_batch(struct intel_context *ce)
cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
if (IS_ERR(cs)) {
- err = PTR_ERR(cs);
- goto err_put;
+ i915_gem_object_put(obj);
+ return ERR_CAST(cs);
}
cs[0] = MI_BATCH_BUFFER_END;
@@ -83,23 +83,18 @@ static struct i915_vma *create_empty_batch(struct intel_context *ce)
i915_gem_object_flush_map(obj);
vma = i915_vma_instance(obj, ce->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_unpin;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return vma;
err = i915_vma_pin(vma, 0, 0, PIN_USER);
- if (err)
- goto err_unpin;
+ if (err) {
+ i915_vma_put(vma);
+ return ERR_PTR(err);
+ }
i915_gem_object_unpin_map(obj);
return vma;
-
-err_unpin:
- i915_gem_object_unpin_map(obj);
-err_put:
- i915_gem_object_put(obj);
- return ERR_PTR(err);
}
static u32 trifilter(u32 *a)
@@ -209,10 +204,8 @@ static struct i915_vma *create_nop_batch(struct intel_context *ce)
return ERR_CAST(obj);
cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
- if (IS_ERR(cs)) {
- err = PTR_ERR(cs);
- goto err_put;
- }
+ if (IS_ERR(cs))
+ return ERR_CAST(cs);
memset(cs, 0, SZ_64K);
cs[SZ_64K / sizeof(*cs) - 1] = MI_BATCH_BUFFER_END;
@@ -220,23 +213,18 @@ static struct i915_vma *create_nop_batch(struct intel_context *ce)
i915_gem_object_flush_map(obj);
vma = i915_vma_instance(obj, ce->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_unpin;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return vma;
err = i915_vma_pin(vma, 0, 0, PIN_USER);
- if (err)
- goto err_unpin;
+ if (err) {
+ i915_vma_put(vma);
+ return ERR_PTR(err);
+ }
i915_gem_object_unpin_map(obj);
return vma;
-
-err_unpin:
- i915_gem_object_unpin_map(obj);
-err_put:
- i915_gem_object_put(obj);
- return ERR_PTR(err);
}
static int perf_mi_noop(void *arg)
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index 4aa4cc917d8b..4133208916be 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -162,22 +162,20 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine)
h->batch = vaddr;
vma = i915_vma_instance(h->obj, vm, NULL);
- if (IS_ERR(vma)) {
- i915_vm_put(vm);
+ i915_gem_object_put(obj);
+ i915_vm_put(vm);
+ if (IS_ERR(vma))
return ERR_CAST(vma);
- }
hws = i915_vma_instance(h->hws, vm, NULL);
if (IS_ERR(hws)) {
- i915_vm_put(vm);
+ i915_vma_put(vma);
return ERR_CAST(hws);
}
err = i915_vma_pin(vma, 0, 0, PIN_USER);
- if (err) {
- i915_vm_put(vm);
- return ERR_PTR(err);
- }
+ if (err)
+ goto put_hws;
err = i915_vma_pin(hws, 0, 0, PIN_USER);
if (err)
@@ -275,7 +273,9 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine)
i915_vma_unpin(hws);
unpin_vma:
i915_vma_unpin(vma);
- i915_vm_put(vm);
+put_hws:
+ i915_vma_put(hws);
+ i915_vma_put(vma);
return err ? ERR_PTR(err) : rq;
}
@@ -1176,20 +1176,22 @@ static int __igt_reset_evict_vma(struct intel_gt *gt,
err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);
if (err) {
pr_err("Invalid X-tiling settings; err:%d\n", err);
- goto out_obj;
+ i915_gem_object_put(obj);
+ goto fini;
}
}
arg.vma = i915_vma_instance(obj, vm, NULL);
+ i915_gem_object_put(obj);
if (IS_ERR(arg.vma)) {
err = PTR_ERR(arg.vma);
- goto out_obj;
+ goto fini;
}
rq = hang_create_request(&h, engine);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
- goto out_obj;
+ goto out_vma;
}
pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER;
@@ -1200,7 +1202,7 @@ static int __igt_reset_evict_vma(struct intel_gt *gt,
err = i915_vma_pin(arg.vma, 0, 0, pin_flags);
if (err) {
i915_request_add(rq);
- goto out_obj;
+ goto out_vma;
}
if (flags & EXEC_OBJECT_NEEDS_FENCE) {
@@ -1209,7 +1211,7 @@ static int __igt_reset_evict_vma(struct intel_gt *gt,
pr_err("Unable to pin X-tiled fence; err:%d\n", err);
i915_vma_unpin(arg.vma);
i915_request_add(rq);
- goto out_obj;
+ goto out_vma;
}
}
@@ -1279,8 +1281,8 @@ static int __igt_reset_evict_vma(struct intel_gt *gt,
out_rq:
i915_request_put(rq);
-out_obj:
- i915_gem_object_put(obj);
+out_vma:
+ i915_vma_put(arg.vma);
fini:
hang_fini(&h);
if (intel_gt_is_wedged(gt))
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index 66f710b1b61e..d2653986ffe2 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -37,14 +37,13 @@ static struct i915_vma *create_scratch(struct intel_gt *gt)
i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
if (err) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_PTR(err);
}
@@ -866,15 +865,14 @@ static int live_timeslice_preempt(void *arg)
return PTR_ERR(obj);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
if (IS_ERR(vaddr)) {
err = PTR_ERR(vaddr);
- goto err_obj;
+ goto err_vma;
}
err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
@@ -912,8 +910,8 @@ static int live_timeslice_preempt(void *arg)
i915_vma_unpin(vma);
err_map:
i915_gem_object_unpin_map(obj);
-err_obj:
- i915_gem_object_put(obj);
+err_vma:
+ i915_vma_put(vma);
return err;
}
@@ -1170,15 +1168,14 @@ static int live_timeslice_queue(void *arg)
return PTR_ERR(obj);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
if (IS_ERR(vaddr)) {
err = PTR_ERR(vaddr);
- goto err_obj;
+ goto err_vma;
}
err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
@@ -1284,8 +1281,8 @@ static int live_timeslice_queue(void *arg)
i915_vma_unpin(vma);
err_map:
i915_gem_object_unpin_map(obj);
-err_obj:
- i915_gem_object_put(obj);
+err_vma:
+ i915_vma_put(vma);
return err;
}
@@ -2869,10 +2866,9 @@ static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
return ERR_CAST(obj);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = i915_ggtt_pin(vma, 0, 0);
if (err) {
@@ -2905,12 +2901,12 @@ create_gpr_client(struct intel_engine_cs *engine,
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (err)
- goto out_ce;
+ goto out_vma;
batch = create_gpr_user(engine, vma, offset);
if (IS_ERR(batch)) {
err = PTR_ERR(batch);
- goto out_vma;
+ goto out_unpin;
}
rq = intel_context_create_request(ce);
@@ -2943,8 +2939,10 @@ create_gpr_client(struct intel_engine_cs *engine,
out_batch:
i915_vma_put(batch);
-out_vma:
+out_unpin:
i915_vma_unpin(vma);
+out_vma:
+ i915_vma_put(vma);
out_ce:
intel_context_put(ce);
return err ? ERR_PTR(err) : rq;
@@ -5317,14 +5315,13 @@ create_user_vma(struct i915_address_space *vm, unsigned long size)
return ERR_CAST(obj);
vma = i915_vma_instance(obj, vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (err) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_PTR(err);
}
diff --git a/drivers/gpu/drm/i915/gt/selftest_mocs.c b/drivers/gpu/drm/i915/gt/selftest_mocs.c
index 8831ffee2061..2fa6cae9f374 100644
--- a/drivers/gpu/drm/i915/gt/selftest_mocs.c
+++ b/drivers/gpu/drm/i915/gt/selftest_mocs.c
@@ -55,14 +55,13 @@ static struct i915_vma *create_scratch(struct intel_gt *gt)
i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
if (err) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_PTR(err);
}
diff --git a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
index 3350e7c995bc..a72bd4ea8cd7 100644
--- a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
@@ -18,26 +18,25 @@ static struct i915_vma *create_wally(struct intel_engine_cs *engine)
return ERR_CAST(obj);
vma = i915_vma_instance(obj, engine->gt->vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
if (err) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_PTR(err);
}
err = i915_vma_sync(vma);
if (err) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_PTR(err);
}
cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
if (IS_ERR(cs)) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_CAST(cs);
}
diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c
index 5049c3dd08a6..cce48e1aec7b 100644
--- a/drivers/gpu/drm/i915/gt/selftest_rps.c
+++ b/drivers/gpu/drm/i915/gt/selftest_rps.c
@@ -91,10 +91,9 @@ create_spin_counter(struct intel_engine_cs *engine,
end = obj->base.size / sizeof(u32) - 1;
vma = i915_vma_instance(obj, vm, NULL);
- if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
return vma;
- }
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (err) {
@@ -104,7 +103,7 @@ create_spin_counter(struct intel_engine_cs *engine,
base = i915_gem_object_pin_map(obj, I915_MAP_WC);
if (IS_ERR(base)) {
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
return ERR_CAST(base);
}
cs = base;
diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c
index 5ed323254ee1..0ffea179dc0b 100644
--- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c
@@ -370,19 +370,18 @@ static struct i915_vma *create_batch(struct i915_address_space *vm)
return ERR_CAST(obj);
vma = i915_vma_instance(obj, vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
- }
+ i915_gem_object_put(obj);
+ if (IS_ERR(vma))
+ return vma;
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (err)
- goto err_obj;
+ goto err_vma;
return vma;
-err_obj:
- i915_gem_object_put(obj);
+err_vma:
+ i915_vma_put(vma);
return ERR_PTR(err);
}
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
index 861657897c0f..2d2719915b26 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
@@ -673,21 +673,18 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size)
return ERR_CAST(obj);
vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
+ i915_gem_object_put(obj);
if (IS_ERR(vma))
- goto err;
+ return vma;
flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
ret = i915_ggtt_pin(vma, 0, flags);
if (ret) {
- vma = ERR_PTR(ret);
- goto err;
+ i915_vma_put(vma);
+ return ERR_PTR(ret);
}
return i915_vma_make_unshrinkable(vma);
-
-err:
- i915_gem_object_put(obj);
- return vma;
}
/**
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 0fb1df71c637..1193906804ca 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -636,8 +636,10 @@ static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
if (bb->va && !IS_ERR(bb->va))
i915_gem_object_unpin_map(bb->obj);
- if (bb->vma && !IS_ERR(bb->vma))
+ if (bb->vma && !IS_ERR(bb->vma)) {
i915_vma_unpin(bb->vma);
+ i915_vma_put(bb->vma);
+ }
i915_gem_object_put(bb->obj);
}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index bca036ac6621..de48387dc409 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1479,6 +1479,8 @@ gt_drop_caches(struct intel_gt *gt, u64 val)
ret = intel_gt_pm_wait_for_idle(gt);
if (ret)
return ret;
+
+ i915_vma_clock_flush(>->vma_clock);
}
if (val & DROP_RESET_ACTIVE && intel_gt_terminally_wedged(gt))
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 34ee12f3f02d..aeb6d285e595 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -456,7 +456,6 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
intel_wopcm_init_early(&dev_priv->wopcm);
intel_gt_init_early(&dev_priv->gt, dev_priv);
-
i915_gem_init_early(dev_priv);
/* This must be called before any calls to HAS_PCH_* */
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 0cbcb9f54e7d..30133277f8a6 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -112,32 +112,16 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
unsigned long flags)
{
- struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
- LIST_HEAD(still_in_list);
- intel_wakeref_t wakeref;
- struct i915_vma *vma;
+ struct i915_vma *vma, *vn;
int ret;
if (list_empty(&obj->vma.list))
return 0;
- /*
- * As some machines use ACPI to handle runtime-resume callbacks, and
- * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
- * as they are required by the shrinker. Ergo, we wake the device up
- * first just in case.
- */
- wakeref = intel_runtime_pm_get(rpm);
-
try_again:
ret = 0;
spin_lock(&obj->vma.lock);
- while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
- struct i915_vma,
- obj_link))) {
- struct i915_address_space *vm = vma->vm;
-
- list_move_tail(&vma->obj_link, &still_in_list);
+ list_for_each_entry_safe(vma, vn, &obj->vma.list, obj_link) {
if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
continue;
@@ -146,27 +130,26 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
break;
}
- ret = -EAGAIN;
- if (!i915_vm_tryopen(vm))
+ vma = i915_vma_tryget(vma);
+ if (!vma) {
+ ret = -EAGAIN;
break;
-
- /* Prevent vma being freed by i915_vma_parked as we unbind */
- vma = __i915_vma_get(vma);
+ }
spin_unlock(&obj->vma.lock);
- if (vma) {
- ret = -EBUSY;
- if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
- !i915_vma_is_active(vma))
- ret = i915_vma_unbind(vma);
-
- __i915_vma_put(vma);
- }
+ ret = -EBUSY;
+ if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
+ !i915_vma_is_active(vma))
+ ret = i915_vma_unbind(vma);
+ i915_vma_put(vma);
- i915_vm_close(vm);
spin_lock(&obj->vma.lock);
+ if (ret)
+ break;
+
+ /* Reset iteration after dropping lock */
+ vn = list_first_entry(&obj->vma.list, typeof(*vn), obj_link);
}
- list_splice_init(&still_in_list, &obj->vma.list);
spin_unlock(&obj->vma.lock);
if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
@@ -174,8 +157,6 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
goto try_again;
}
- intel_runtime_pm_put(rpm, wakeref);
-
return ret;
}
@@ -477,7 +458,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
remove_mappable_node(ggtt, &node);
} else {
- i915_vma_unpin(vma);
+ i915_vma_unpin_and_release(&vma, 0);
}
out_rpm:
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
@@ -688,7 +669,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
remove_mappable_node(ggtt, &node);
} else {
- i915_vma_unpin(vma);
+ i915_vma_unpin_and_release(&vma, 0);
}
out_rpm:
intel_runtime_pm_put(rpm, wakeref);
@@ -985,22 +966,27 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
if (i915_vma_misplaced(vma, size, alignment, flags)) {
if (flags & PIN_NONBLOCK) {
- if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
- return ERR_PTR(-ENOSPC);
+ if (i915_vma_is_pinned(vma) ||
+ i915_vma_is_active(vma)) {
+ ret = -ENOSPC;
+ goto err_put;
+ }
if (flags & PIN_MAPPABLE &&
- vma->fence_size > ggtt->mappable_end / 2)
- return ERR_PTR(-ENOSPC);
+ vma->fence_size > ggtt->mappable_end / 2) {
+ ret = -ENOSPC;
+ goto err_put;
+ }
}
ret = i915_vma_unbind(vma);
if (ret)
- return ERR_PTR(ret);
+ goto err_put;
}
ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
if (ret)
- return ERR_PTR(ret);
+ goto err_put;
if (vma->fence && !i915_gem_object_is_tiled(obj)) {
mutex_lock(&ggtt->vm.mutex);
@@ -1009,12 +995,16 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
}
ret = i915_vma_wait_for_bind(vma);
- if (ret) {
- i915_vma_unpin(vma);
- return ERR_PTR(ret);
- }
+ if (ret)
+ goto err_unpin;
return vma;
+
+err_unpin:
+ i915_vma_unpin(vma);
+err_put:
+ i915_vma_put(vma);
+ return ERR_PTR(ret);
}
int
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index f35712d04ba4..27284503a5f2 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1548,23 +1548,21 @@ static int alloc_oa_buffer(struct i915_perf_stream *stream)
BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE);
- if (IS_ERR(bo)) {
- drm_err(&i915->drm, "Failed to allocate OA buffer\n");
+ if (IS_ERR(bo))
return PTR_ERR(bo);
- }
i915_gem_object_set_cache_coherency(bo, I915_CACHE_LLC);
/* PreHSW required 512K alignment, HSW requires 16M */
vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0);
+ i915_gem_object_put(bo);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
- goto err_unref;
+ goto err;
}
stream->oa_buffer.vma = vma;
- stream->oa_buffer.vaddr =
- i915_gem_object_pin_map(bo, I915_MAP_WB);
+ stream->oa_buffer.vaddr = i915_gem_object_pin_map(bo, I915_MAP_WB);
if (IS_ERR(stream->oa_buffer.vaddr)) {
ret = PTR_ERR(stream->oa_buffer.vaddr);
goto err_unpin;
@@ -1573,14 +1571,10 @@ static int alloc_oa_buffer(struct i915_perf_stream *stream)
return 0;
err_unpin:
- __i915_vma_unpin(vma);
-
-err_unref:
- i915_gem_object_put(bo);
-
+ i915_vma_unpin(vma);
+err:
stream->oa_buffer.vaddr = NULL;
stream->oa_buffer.vma = NULL;
-
return ret;
}
@@ -1639,10 +1633,9 @@ static int alloc_noa_wait(struct i915_perf_stream *stream)
* needs to be fixed during the lifetime of the i915/perf stream.
*/
vma = i915_gem_object_ggtt_pin(bo, NULL, 0, 0, PIN_HIGH);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err_unref;
- }
+ i915_gem_object_put(bo);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
batch = cs = i915_gem_object_pin_map(bo, I915_MAP_WB);
if (IS_ERR(batch)) {
@@ -1779,8 +1772,6 @@ static int alloc_noa_wait(struct i915_perf_stream *stream)
err_unpin:
i915_vma_unpin_and_release(&vma, 0);
-err_unref:
- i915_gem_object_put(bo);
return ret;
}
@@ -1879,6 +1870,7 @@ alloc_oa_config_buffer(struct i915_perf_stream *stream,
oa_bo->oa_config = i915_oa_config_get(oa_config);
llist_add(&oa_bo->node, &stream->oa_config_bos);
+ i915_gem_object_put(obj);
return oa_bo;
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 22198b758459..a87057f7877e 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -90,7 +90,8 @@ static inline struct i915_vma *active_to_vma(struct i915_active *ref)
static int __i915_vma_active(struct i915_active *ref)
{
- return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT;
+ i915_vma_get(active_to_vma(ref));
+ return 0;
}
__i915_active_call
@@ -118,7 +119,7 @@ vma_create(struct drm_i915_gem_object *obj,
mutex_init(&vma->pages_mutex);
vma->vm = i915_vm_get(vm);
vma->ops = &vm->vma_ops;
- vma->obj = obj;
+ vma->obj = i915_gem_object_get(obj);
vma->resv = obj->base.resv;
vma->size = obj->base.size;
vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
@@ -282,6 +283,8 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
spin_lock(&obj->vma.lock);
vma = vma_lookup(obj, vm, view);
+ if (vma)
+ vma = i915_vma_tryget(vma);
spin_unlock(&obj->vma.lock);
/* vma_create() will resolve the race if another creates the vma */
@@ -510,21 +513,20 @@ void i915_vma_unpin_iomap(struct i915_vma *vma)
void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
{
struct i915_vma *vma;
- struct drm_i915_gem_object *obj;
vma = fetch_and_zero(p_vma);
if (!vma)
return;
- obj = vma->obj;
- GEM_BUG_ON(!obj);
+ if (flags & I915_VMA_RELEASE_FENCE)
+ i915_vma_unpin_fence(vma);
i915_vma_unpin(vma);
if (flags & I915_VMA_RELEASE_MAP)
- i915_gem_object_unpin_map(obj);
+ i915_gem_object_unpin_map(vma->obj);
- i915_gem_object_put(obj);
+ i915_vma_put(vma);
}
bool i915_vma_misplaced(const struct i915_vma *vma,
@@ -1019,8 +1021,31 @@ int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags)
} while (1);
}
-static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
+void i915_vma_open(struct i915_vma *vma)
+{
+ struct i915_vma_clock *clock = &vma->vm->gt->vma_clock;
+ bool closed = false;
+
+ if (!atomic_add_unless(&vma->open_count, 1, 0)) {
+ spin_lock(&clock->lock);
+ if (!atomic_add_unless(&vma->open_count, 1, 0)) {
+ if (!list_empty(&vma->closed_link)) {
+ list_del_init(&vma->closed_link);
+ closed = true;
+ }
+ atomic_set(&vma->open_count, 1);
+ }
+ spin_unlock(&clock->lock);
+ }
+
+ if (!closed)
+ i915_vma_get(vma);
+}
+
+void i915_vma_close(struct i915_vma *vma)
{
+ struct i915_vma_clock *clock = &vma->vm->gt->vma_clock;
+
/*
* We defer actually closing, unbinding and destroying the VMA until
* the next idle point, or if the object is freed in the meantime. By
@@ -1033,40 +1058,34 @@ static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
* causing us to rebind the VMA once more. This ends up being a lot
* of wasted work for the steady state.
*/
- GEM_BUG_ON(i915_vma_is_closed(vma));
- list_add(&vma->closed_link, >->closed_vma);
-}
-
-void i915_vma_close(struct i915_vma *vma)
-{
- struct intel_gt *gt = vma->vm->gt;
- unsigned long flags;
-
- if (i915_vma_is_ggtt(vma))
- return;
GEM_BUG_ON(!atomic_read(&vma->open_count));
- if (atomic_dec_and_lock_irqsave(&vma->open_count,
- >->closed_lock,
- flags)) {
- __vma_close(vma, gt);
- spin_unlock_irqrestore(>->closed_lock, flags);
- }
-}
-
-static void __i915_vma_remove_closed(struct i915_vma *vma)
-{
- struct intel_gt *gt = vma->vm->gt;
+ if (atomic_dec_and_lock(&vma->open_count, &clock->lock)) {
+ GEM_BUG_ON(i915_vma_is_closed(vma));
+ if (vma->obj->mm.madv == I915_MADV_WILLNEED) {
+ list_add(&vma->closed_link, &clock->age[0]);
+ i915_vma_get(vma);
+ }
+ spin_unlock(&clock->lock);
- spin_lock_irq(>->closed_lock);
- list_del_init(&vma->closed_link);
- spin_unlock_irq(>->closed_lock);
+ if (!list_empty(&vma->closed_link))
+ schedule_delayed_work(&clock->work,
+ round_jiffies_up_relative(HZ));
+ }
}
-void i915_vma_reopen(struct i915_vma *vma)
+static void __release_unbind(struct i915_vma *vma)
{
- if (i915_vma_is_closed(vma))
- __i915_vma_remove_closed(vma);
+ struct i915_address_space *vm = vma->vm;
+ unsigned long flags;
+ bool first;
+
+ spin_lock_irqsave(&vm->release.lock, flags);
+ list_add_tail(&vma->closed_link, &vm->release.list);
+ first = list_is_first(&vma->closed_link, &vm->release.list);
+ spin_unlock_irqrestore(&vm->release.lock, flags);
+ if (first)
+ schedule_work(&vm->release.work);
}
void i915_vma_release(struct kref *ref)
@@ -1074,11 +1093,9 @@ void i915_vma_release(struct kref *ref)
struct i915_vma *vma = container_of(ref, typeof(*vma), ref);
if (drm_mm_node_allocated(&vma->node)) {
- mutex_lock(&vma->vm->mutex);
- atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
- WARN_ON(__i915_vma_unbind(vma));
- mutex_unlock(&vma->vm->mutex);
- GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+ GEM_BUG_ON(i915_vma_is_closed(vma));
+ __release_unbind(vma);
+ return;
}
GEM_BUG_ON(i915_vma_is_active(vma));
@@ -1089,50 +1106,63 @@ void i915_vma_release(struct kref *ref)
list_del(&vma->obj_link);
rb_erase(&vma->obj_node, &obj->vma.tree);
spin_unlock(&obj->vma.lock);
+
+ i915_gem_object_put(obj);
}
- __i915_vma_remove_closed(vma);
i915_vm_put(vma->vm);
i915_active_fini(&vma->active);
i915_vma_free(vma);
}
-void i915_vma_parked(struct intel_gt *gt)
+static void i915_vma_clock(struct work_struct *w)
{
+ struct i915_vma_clock *clock =
+ container_of(w, typeof(*clock), work.work);
struct i915_vma *vma, *next;
LIST_HEAD(closed);
- spin_lock_irq(>->closed_lock);
- list_for_each_entry_safe(vma, next, >->closed_vma, closed_link) {
- struct drm_i915_gem_object *obj = vma->obj;
- struct i915_address_space *vm = vma->vm;
+ /*
+ * A very simple clock aging algorithm: we keep the user's closed
+ * vma alive for a couple of timer ticks before destroying them.
+ * This serves a shortlived cache so that frequently reused VMA
+ * are kept alive between frames and we skip having to rebing them.
+ *
+ * When closed, we insert the vma into age[0]. Upon completion of
+ * a timer tick, it is moved to age[1]. At the start of each timer
+ * tick, we destroy all the old vma that were accumulated into age[1]
+ * and have not been reused. All destroyed vma have therefore been
+ * unused for more than 1 tick (at least a second), and at most 2
+ * ticks (we expect the average to be 1.5 ticks).
+ */
- /* XXX All to avoid keeping a reference on i915_vma itself */
+ spin_lock(&clock->lock);
- if (!kref_get_unless_zero(&obj->base.refcount))
- continue;
+ list_for_each_entry_safe(vma, next, &clock->age[1], closed_link) {
+ list_del_init(&vma->closed_link);
- if (!i915_vm_tryopen(vm)) {
- i915_gem_object_put(obj);
+ if (i915_vma_is_active(vma)) {
+ list_add_tail(&vma->closed_link, &clock->age[0]);
continue;
}
- list_move(&vma->closed_link, &closed);
- }
- spin_unlock_irq(>->closed_lock);
-
- /* As the GT is held idle, no vma can be reopened as we destroy them */
- list_for_each_entry_safe(vma, next, &closed, closed_link) {
- struct drm_i915_gem_object *obj = vma->obj;
- struct i915_address_space *vm = vma->vm;
+ spin_unlock(&clock->lock);
- INIT_LIST_HEAD(&vma->closed_link);
- __i915_vma_put(vma);
+ i915_vma_put(vma);
- i915_gem_object_put(obj);
- i915_vm_close(vm);
+ /* Restart after dropping lock */
+ spin_lock(&clock->lock);
+ next = list_first_entry(&clock->age[1],
+ typeof(*next), closed_link);
}
+ list_splice_tail_init(&clock->age[0], &clock->age[1]);
+
+ if (!list_empty(&clock->age[1]))
+ schedule_delayed_work(&clock->work,
+ round_jiffies_up_relative(HZ));
+
+ spin_unlock(&clock->lock);
}
static void __i915_vma_iounmap(struct i915_vma *vma)
@@ -1349,6 +1379,23 @@ void i915_vma_make_purgeable(struct i915_vma *vma)
i915_gem_object_make_purgeable(vma->obj);
}
+void i915_vma_clock_init_early(struct i915_vma_clock *clock)
+{
+ spin_lock_init(&clock->lock);
+ INIT_LIST_HEAD(&clock->age[0]);
+ INIT_LIST_HEAD(&clock->age[1]);
+
+ INIT_DELAYED_WORK(&clock->work, i915_vma_clock);
+}
+
+void i915_vma_clock_flush(struct i915_vma_clock *clock)
+{
+ do {
+ if (cancel_delayed_work_sync(&clock->work))
+ i915_vma_clock(&clock->work.work);
+ } while (delayed_work_pending(&clock->work));
+}
+
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/i915_vma.c"
#endif
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 8ad1daabcd58..531d948fdb8d 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -45,7 +45,8 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
const struct i915_ggtt_view *view);
void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);
-#define I915_VMA_RELEASE_MAP BIT(0)
+#define I915_VMA_RELEASE_FENCE BIT(0)
+#define I915_VMA_RELEASE_MAP BIT(1)
static inline bool i915_vma_is_active(const struct i915_vma *vma)
{
@@ -124,15 +125,17 @@ static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma)
return i915_vm_to_ggtt(vma->vm)->pin_bias;
}
+void i915_vma_release(struct kref *ref);
+
static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
{
- i915_gem_object_get(vma->obj);
+ kref_get(&vma->ref);
return vma;
}
static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma)
{
- if (likely(kref_get_unless_zero(&vma->obj->base.refcount)))
+ if (kref_get_unless_zero(&vma->ref))
return vma;
return NULL;
@@ -140,7 +143,7 @@ static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma)
static inline void i915_vma_put(struct i915_vma *vma)
{
- i915_gem_object_put(vma->obj);
+ kref_put(&vma->ref, i915_vma_release);
}
static __always_inline ptrdiff_t ptrdiff(const void *a, const void *b)
@@ -206,22 +209,8 @@ void i915_vma_revoke_mmap(struct i915_vma *vma);
int __i915_vma_unbind(struct i915_vma *vma);
int __must_check i915_vma_unbind(struct i915_vma *vma);
void i915_vma_unlink_ctx(struct i915_vma *vma);
+void i915_vma_open(struct i915_vma *vma);
void i915_vma_close(struct i915_vma *vma);
-void i915_vma_reopen(struct i915_vma *vma);
-
-static inline struct i915_vma *__i915_vma_get(struct i915_vma *vma)
-{
- if (kref_get_unless_zero(&vma->ref))
- return vma;
-
- return NULL;
-}
-
-void i915_vma_release(struct kref *ref);
-static inline void __i915_vma_put(struct i915_vma *vma)
-{
- kref_put(&vma->ref, i915_vma_release);
-}
#define assert_vma_held(vma) dma_resv_assert_held((vma)->resv)
@@ -351,8 +340,6 @@ i915_vma_unpin_fence(struct i915_vma *vma)
__i915_vma_unpin_fence(vma);
}
-void i915_vma_parked(struct intel_gt *gt);
-
#define for_each_until(cond) if (cond) break; else
/**
@@ -383,4 +370,13 @@ static inline int i915_vma_sync(struct i915_vma *vma)
return i915_active_wait(&vma->active);
}
+struct i915_vma_clock {
+ spinlock_t lock;
+ struct list_head age[2];
+ struct delayed_work work;
+};
+
+void i915_vma_clock_init_early(struct i915_vma_clock *clock);
+void i915_vma_clock_flush(struct i915_vma_clock *clock);
+
#endif
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index 028baae9631f..f1e5fe0604ed 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -58,8 +58,8 @@ static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects)
return PTR_ERR(obj);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
- i915_gem_object_put(obj);
if (vma == ERR_PTR(-ENOSPC))
break;
@@ -82,11 +82,14 @@ static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects)
static void unpin_ggtt(struct i915_ggtt *ggtt)
{
- struct i915_vma *vma;
+ struct i915_vma *vma, *vn;
- list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
- if (vma->obj->mm.quirked)
- i915_vma_unpin(vma);
+ list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
+ if (!vma->obj->mm.quirked)
+ continue;
+
+ i915_vma_unpin(vma);
+ }
}
static void cleanup_objects(struct i915_ggtt *ggtt, struct list_head *list)
@@ -171,15 +174,16 @@ static int igt_overcommit(void *arg)
goto cleanup;
}
- quirk_add(obj, &objects);
-
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+ i915_gem_object_put(obj);
if (vma != ERR_PTR(-ENOSPC)) {
pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR_OR_ZERO(vma));
err = -EINVAL;
goto cleanup;
}
+ quirk_add(obj, &objects);
+
cleanup:
cleanup_objects(ggtt, &objects);
return err;
@@ -266,33 +270,37 @@ static int igt_evict_for_cache_color(void *arg)
goto cleanup;
}
i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
- quirk_add(obj, &objects);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
I915_GTT_PAGE_SIZE | flags);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
pr_err("[0]i915_gem_object_ggtt_pin failed\n");
err = PTR_ERR(vma);
goto cleanup;
}
+ quirk_add(obj, &objects);
+
obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
if (IS_ERR(obj)) {
err = PTR_ERR(obj);
goto cleanup;
}
i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
- quirk_add(obj, &objects);
/* Neighbouring; same colour - should fit */
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
(I915_GTT_PAGE_SIZE * 2) | flags);
+ i915_gem_object_put(obj);
if (IS_ERR(vma)) {
pr_err("[1]i915_gem_object_ggtt_pin failed\n");
err = PTR_ERR(vma);
goto cleanup;
}
+ quirk_add(obj, &objects);
+
i915_vma_unpin(vma);
/* Remove just the second vma */
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index af89c7fc8f59..576f3d2fa72b 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -910,6 +910,7 @@ static int igt_vma_remapped_gtt(void *arg)
i915_vma_unpin(vma);
if (IS_ERR(map)) {
err = PTR_ERR(map);
+ i915_vma_put(vma);
goto out;
}
@@ -928,6 +929,7 @@ static int igt_vma_remapped_gtt(void *arg)
}
i915_vma_unpin_iomap(vma);
+ i915_vma_put(vma);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
if (IS_ERR(vma)) {
@@ -941,6 +943,7 @@ static int igt_vma_remapped_gtt(void *arg)
i915_vma_unpin(vma);
if (IS_ERR(map)) {
err = PTR_ERR(map);
+ i915_vma_put(vma);
goto out;
}
@@ -967,6 +970,7 @@ static int igt_vma_remapped_gtt(void *arg)
}
}
i915_vma_unpin_iomap(vma);
+ i915_vma_put(vma);
cond_resched();
}
--
2.20.1
More information about the Intel-gfx-trybot
mailing list