[PATCH 04/12] drm/i915: Shrink pages around failure to dma map
Chris Wilson
chris at chris-wilson.co.uk
Sat Aug 6 14:11:19 UTC 2016
Similar to how we handle resource allocation failure of both physical
memory and GGTT mmap space, if we fail to allocate our DMAR remapping,
shrink some of our other objects and try again.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_debugfs.c | 16 +-
drivers/gpu/drm/i915/i915_drv.h | 82 ++++--
drivers/gpu/drm/i915/i915_gem.c | 209 ++++++++-------
drivers/gpu/drm/i915/i915_gem_evict.c | 7 +-
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 20 +-
drivers/gpu/drm/i915/i915_gem_fence.c | 401 +++++++++++------------------
drivers/gpu/drm/i915/i915_gem_gtt.c | 37 ++-
drivers/gpu/drm/i915/i915_gem_gtt.h | 11 +-
drivers/gpu/drm/i915/i915_gem_tiling.c | 67 +++--
drivers/gpu/drm/i915/i915_gem_userptr.c | 17 +-
drivers/gpu/drm/i915/i915_gpu_error.c | 2 +-
drivers/gpu/drm/i915/intel_display.c | 57 ++--
drivers/gpu/drm/i915/intel_fbc.c | 15 +-
drivers/gpu/drm/i915/intel_overlay.c | 2 +-
14 files changed, 462 insertions(+), 481 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index c569c858f2f1..6b5b4b8ecaa8 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -145,11 +145,9 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
seq_printf(m, "%x ",
i915_gem_active_get_seqno(&obj->last_read[id],
&obj->base.dev->struct_mutex));
- seq_printf(m, "] %x %x%s%s%s",
+ seq_printf(m, "] %x %s%s%s",
i915_gem_active_get_seqno(&obj->last_write,
&obj->base.dev->struct_mutex),
- i915_gem_active_get_seqno(&obj->last_fence,
- &obj->base.dev->struct_mutex),
i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
obj->dirty ? " dirty" : "",
obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
@@ -162,8 +160,6 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
seq_printf(m, " (pinned x %d)", pin_count);
if (obj->pin_display)
seq_printf(m, " (display)");
- if (obj->fence_reg != I915_FENCE_REG_NONE)
- seq_printf(m, " (fence: %d)", obj->fence_reg);
list_for_each_entry(vma, &obj->vma_list, obj_link) {
if (!drm_mm_node_allocated(&vma->node))
continue;
@@ -173,6 +169,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
vma->node.start, vma->node.size);
if (i915_vma_is_ggtt(vma))
seq_printf(m, ", type: %u", vma->ggtt_view.type);
+ if (vma->fence)
+ seq_printf(m, " , fence: %d%s",
+ vma->fence->id,
+ i915_gem_active_isset(&vma->last_fence) ? "*" : "");
seq_puts(m, ")");
}
if (obj->stolen)
@@ -1025,14 +1025,14 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
for (i = 0; i < dev_priv->num_fence_regs; i++) {
- struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
+ struct i915_vma *vma = dev_priv->fence_regs[i].vma;
seq_printf(m, "Fence %d, pin count = %d, object = ",
i, dev_priv->fence_regs[i].pin_count);
- if (obj == NULL)
+ if (!vma)
seq_puts(m, "unused");
else
- describe_obj(m, obj);
+ describe_obj(m, vma->obj);
seq_putc(m, '\n');
}
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index c83c779c3dd1..d0734fc8c560 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -455,15 +455,21 @@ struct intel_opregion {
struct intel_overlay;
struct intel_overlay_error_state;
-#define I915_FENCE_REG_NONE -1
-#define I915_MAX_NUM_FENCES 32
-/* 32 fences + sign bit for FENCE_REG_NONE */
-#define I915_MAX_NUM_FENCE_BITS 6
-
struct drm_i915_fence_reg {
struct list_head lru_list;
- struct drm_i915_gem_object *obj;
+ struct drm_i915_private *i915;
+ struct i915_vma *vma;
int pin_count;
+ int id;
+ /**
+ * Whether the tiling parameters for the currently
+ * associated fence register have changed. Note that
+ * for the purposes of tracking tiling changes we also
+ * treat the unfenced register, the register slot that
+ * the object occupies whilst it executes a fenced
+ * command (such as BLT on gen2/3), as a "fence".
+ */
+ bool dirty;
};
struct sdvo_device_mapping {
@@ -2177,27 +2183,11 @@ struct drm_i915_gem_object {
unsigned int dirty:1;
/**
- * Fence register bits (if any) for this object. Will be set
- * as needed when mapped into the GTT.
- * Protected by dev->struct_mutex.
- */
- signed int fence_reg:I915_MAX_NUM_FENCE_BITS;
-
- /**
* Advice: are the backing pages purgeable?
*/
unsigned int madv:2;
/**
- * Whether the tiling parameters for the currently associated fence
- * register have changed. Note that for the purposes of tracking
- * tiling changes we also treat the unfenced register, the register
- * slot that the object occupies whilst it executes a fenced
- * command (such as BLT on gen2/3), as a "fence".
- */
- unsigned int fence_dirty:1;
-
- /**
* Whether the current gtt mapping needs to be mappable (and isn't just
* mappable by accident). Track pin and fault separate for a more
* accurate mappable working set.
@@ -2245,7 +2235,6 @@ struct drm_i915_gem_object {
*/
struct i915_gem_active last_read[I915_NUM_ENGINES];
struct i915_gem_active last_write;
- struct i915_gem_active last_fence;
/** References from framebuffers, locks out tiling changes. */
unsigned long framebuffer_references;
@@ -3321,11 +3310,50 @@ i915_gem_object_ggtt_offset(struct drm_i915_gem_object *o,
}
/* i915_gem_fence.c */
-int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj);
-int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
+int __must_check i915_vma_get_fence(struct i915_vma *vma);
+int __must_check i915_vma_put_fence(struct i915_vma *vma);
+
+/**
+ * i915_vma_pin_fence - pin fencing state
+ * @vma: vma to pin fencing for
+ *
+ * This pins the fencing state (whether tiled or untiled) to make sure the
+ * vma (and its object) is ready to be used as a scanout target. Fencing
+ * status must be synchronize first by calling i915_vma_get_fence():
+ *
+ * The resulting fence pin reference must be released again with
+ * i915_vma_unpin_fence().
+ *
+ * Returns:
+ *
+ * True if the vma has a fence, false otherwise.
+ */
+static inline bool
+i915_vma_pin_fence(struct i915_vma *vma)
+{
+ if (vma->fence) {
+ vma->fence->pin_count++;
+ return true;
+ } else
+ return false;
+}
-bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj);
-void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj);
+/**
+ * i915_vma_unpin_fence - unpin fencing state
+ * @vma: vma to unpin fencing for
+ *
+ * This releases the fence pin reference acquired through
+ * i915_vma_pin_fence. It will handle both objects with and without an
+ * attached fence correctly, callers do not need to distinguish this.
+ */
+static inline void
+i915_vma_unpin_fence(struct i915_vma *vma)
+{
+ if (vma->fence) {
+ GEM_BUG_ON(vma->fence->pin_count <= 0);
+ vma->fence->pin_count--;
+ }
+}
void i915_gem_restore_fences(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 800533b58344..ce8e67009c35 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -832,7 +832,7 @@ i915_gem_gtt_pread(struct drm_device *dev,
} else {
node.start = vma->node.start;
node.allocated = false;
- ret = i915_gem_object_put_fence(obj);
+ ret = i915_vma_put_fence(vma);
if (ret)
goto out_unpin;
}
@@ -1131,15 +1131,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
} else {
node.start = vma->node.start;
node.allocated = false;
- ret = i915_gem_object_put_fence(obj);
+ ret = i915_vma_put_fence(vma);
if (ret)
goto out_unpin;
}
- ret = i915_gem_object_set_to_gtt_domain(obj, true);
- if (ret)
- goto out_unpin;
-
intel_fb_obj_invalidate(obj, ORIGIN_GTT);
obj->dirty = true;
@@ -1654,6 +1650,16 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
return 0;
}
+static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
+{
+ u64 size;
+
+ size = i915_gem_object_get_stride(obj);
+ size *= i915_gem_object_get_tiling(obj) == I915_TILING_Y ? 32 : 8;
+
+ return size >> PAGE_SHIFT;
+}
+
/**
* i915_gem_fault - fault a page into the GTT
* @vma: VMA in question
@@ -1675,11 +1681,11 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = to_i915(dev);
- struct i915_ggtt_view view = i915_ggtt_view_normal;
bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
struct i915_vma *ggtt;
pgoff_t page_offset;
unsigned long pfn;
+ unsigned int flags;
int ret;
/* We don't use vmf->pgoff since that has the fake offset */
@@ -1709,23 +1715,31 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
goto err_unlock;
}
- /* Use a partial view if the object is bigger than the aperture. */
- if (obj->base.size >= dev_priv->ggtt.mappable_end &&
- !i915_gem_object_is_tiled(obj)) {
- static const unsigned int chunk_size = 256; // 1 MiB
-
- memset(&view, 0, sizeof(view));
- view.type = I915_GGTT_VIEW_PARTIAL;
- view.params.partial.offset = rounddown(page_offset, chunk_size);
- view.params.partial.size =
+ flags = PIN_MAPPABLE;
+ if (obj->base.size > 2 << 20)
+ flags |= PIN_NONBLOCK | PIN_NOFAULT;
+ ggtt = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
+ if (IS_ERR(ggtt)) {
+ struct i915_ggtt_view partial;
+ unsigned int chunk_size;
+
+ chunk_size = 256; /* 1 MiB */
+ if (i915_gem_object_is_tiled(obj))
+ chunk_size = max(chunk_size, tile_row_pages(obj));
+
+ memset(&partial, 0, sizeof(partial));
+ partial.type = I915_GGTT_VIEW_PARTIAL;
+ partial.params.partial.offset =
+ rounddown(page_offset, chunk_size);
+ partial.params.partial.size =
min_t(unsigned int,
chunk_size,
(vma->vm_end - vma->vm_start)/PAGE_SIZE -
- view.params.partial.offset);
- }
+ partial.params.partial.offset);
- /* Now pin it into the GTT if needed */
- ggtt = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
+ ggtt = i915_gem_object_ggtt_pin(obj, &partial, 0, 0,
+ PIN_MAPPABLE);
+ }
if (IS_ERR(ggtt)) {
ret = PTR_ERR(ggtt);
goto err_unlock;
@@ -1735,7 +1749,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
if (ret)
goto err_unpin;
- ret = i915_gem_object_get_fence(obj);
+ ret = i915_vma_get_fence(ggtt);
if (ret)
goto err_unpin;
@@ -1743,24 +1757,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
pfn = dev_priv->ggtt.mappable_base + ggtt->node.start;
pfn >>= PAGE_SHIFT;
- if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
- /* Overriding existing pages in partial view does not cause
- * us any trouble as TLBs are still valid because the fault
- * is due to userspace losing part of the mapping or never
- * having accessed it before (at this partials' range).
- */
- unsigned long base = vma->vm_start +
- (view.params.partial.offset << PAGE_SHIFT);
- unsigned int i;
-
- for (i = 0; i < view.params.partial.size; i++) {
- ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
- if (ret)
- break;
- }
-
- obj->fault_mappable = true;
- } else {
+ if (ggtt->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
if (!obj->fault_mappable) {
unsigned long size = min_t(unsigned long,
vma->vm_end - vma->vm_start,
@@ -1774,13 +1771,29 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
if (ret)
break;
}
-
- obj->fault_mappable = true;
} else
ret = vm_insert_pfn(vma,
(unsigned long)vmf->virtual_address,
pfn + page_offset);
+ } else {
+ /* Overriding existing pages in partial view does not cause
+ * us any trouble as TLBs are still valid because the fault
+ * is due to userspace losing part of the mapping or never
+ * having accessed it before (at this partials' range).
+ */
+ const struct i915_ggtt_view *view = &ggtt->ggtt_view;
+ unsigned long base = vma->vm_start +
+ (view->params.partial.offset << PAGE_SHIFT);
+ unsigned int i;
+
+ for (i = 0; i < view->params.partial.size; i++) {
+ ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
+ if (ret)
+ break;
+ }
}
+
+ obj->fault_mappable = true;
err_unpin:
__i915_vma_unpin(ggtt);
err_unlock:
@@ -2765,27 +2778,6 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj,
return 0;
}
-static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
-{
- u32 old_write_domain, old_read_domains;
-
- /* Force a pagefault for domain tracking on next user access */
- i915_gem_release_mmap(obj);
-
- if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
- return;
-
- old_read_domains = obj->base.read_domains;
- old_write_domain = obj->base.write_domain;
-
- obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
- obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
-
- trace_i915_gem_object_change_domain(obj,
- old_read_domains,
- old_write_domain);
-}
-
static void __i915_vma_iounmap(struct i915_vma *vma)
{
GEM_BUG_ON(i915_vma_is_pinned(vma));
@@ -2841,13 +2833,14 @@ int i915_vma_unbind(struct i915_vma *vma)
GEM_BUG_ON(!obj->pages);
if (i915_vma_is_map_and_fenceable(vma)) {
- i915_gem_object_finish_gtt(obj);
-
/* release the fence reg _after_ flushing */
- ret = i915_gem_object_put_fence(obj);
+ ret = i915_vma_put_fence(vma);
if (ret)
return ret;
+ /* Force a pagefault for domain tracking on next user access */
+ i915_gem_release_mmap(obj);
+
__i915_vma_iounmap(vma);
vma->flags &= ~I915_VMA_CAN_FENCE;
}
@@ -2959,7 +2952,6 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
struct drm_i915_gem_object *obj = vma->obj;
u64 start, end;
- u64 min_alignment;
int ret;
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
@@ -2970,17 +2962,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
size = i915_gem_get_ggtt_size(dev_priv, size,
i915_gem_object_get_tiling(obj));
- min_alignment =
- i915_gem_get_ggtt_alignment(dev_priv, size,
- i915_gem_object_get_tiling(obj),
- flags & PIN_MAPPABLE);
- if (alignment == 0)
- alignment = min_alignment;
- if (alignment & (min_alignment - 1)) {
- DRM_DEBUG("Invalid object alignment requested %llu, minimum %llu\n",
- alignment, min_alignment);
- return -EINVAL;
- }
+ alignment = max(max(alignment, vma->display_alignment),
+ i915_gem_get_ggtt_alignment(dev_priv, size,
+ i915_gem_object_get_tiling(obj),
+ flags & PIN_MAPPABLE));
start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
@@ -3170,6 +3155,24 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
I915_GEM_DOMAIN_CPU);
}
+static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
+{
+ struct i915_vma *vma;
+
+ list_for_each_entry(vma, &obj->vma_list, obj_link) {
+ if (!i915_vma_is_ggtt(vma))
+ continue;
+
+ if (i915_vma_is_active(vma))
+ continue;
+
+ if (!drm_mm_node_allocated(&vma->node))
+ continue;
+
+ list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+ }
+}
+
/**
* Moves a single object to the GTT read, and possibly write domain.
* @obj: object to act on
@@ -3182,7 +3185,6 @@ int
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
{
uint32_t old_write_domain, old_read_domains;
- struct i915_vma *vma;
int ret;
ret = i915_gem_object_wait_rendering(obj, !write);
@@ -3232,11 +3234,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
old_write_domain);
/* And bump the LRU for this access */
- vma = i915_gem_object_to_ggtt(obj, NULL);
- if (vma &&
- drm_mm_node_allocated(&vma->node) &&
- !i915_vma_is_active(vma))
- list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+ i915_gem_object_bump_inactive_ggtt(obj);
return 0;
}
@@ -3327,9 +3325,11 @@ restart:
* dropped the fence as all snoopable access is
* supposed to be linear.
*/
- ret = i915_gem_object_put_fence(obj);
- if (ret)
- return ret;
+ list_for_each_entry(vma, &obj->vma_list, obj_link) {
+ ret = i915_vma_put_fence(vma);
+ if (ret)
+ return ret;
+ }
} else {
/* We either have incoherent backing store and
* so no GTT access or the architecture is fully
@@ -3488,14 +3488,22 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
/* As the user may map the buffer once pinned in the display plane
* (e.g. libkms for the bootup splash), we have to ensure that we
- * always use map_and_fenceable for all scanout buffers.
+ * always use map_and_fenceable for all scanout buffers. However,
+ * it may simply be too big to fit into mappable, in which case
+ * put it anyway and hope that userspace can cope (but always first
+ * try to preserve the existing ABI).
*/
- vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
- view->type == I915_GGTT_VIEW_NORMAL ?
- PIN_MAPPABLE : 0);
+ vma = ERR_PTR(-ENOSPC);
+ if (view->type == I915_GGTT_VIEW_NORMAL)
+ vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
+ PIN_MAPPABLE | PIN_NONBLOCK);
+ if (IS_ERR(vma))
+ vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 0);
if (IS_ERR(vma))
goto err_unpin_display;
+ vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
+
WARN_ON(obj->pin_display > i915_vma_pin_count(vma));
i915_gem_object_flush_cpu_write_domain(obj);
@@ -3526,7 +3534,12 @@ i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
if (WARN_ON(vma->obj->pin_display == 0))
return;
- vma->obj->pin_display--;
+ if (--vma->obj->pin_display == 0)
+ vma->display_alignment = 0;
+
+ /* Bump the LRU to try and avoid premature eviction whilst flipping */
+ if (!i915_vma_is_active(vma))
+ list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
i915_vma_unpin(vma);
WARN_ON(vma->obj->pin_display > i915_vma_pin_count(vma));
@@ -3968,14 +3981,12 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
i915_gem_object_retire__read);
init_request_active(&obj->last_write,
i915_gem_object_retire__write);
- init_request_active(&obj->last_fence, NULL);
INIT_LIST_HEAD(&obj->obj_exec_link);
INIT_LIST_HEAD(&obj->vma_list);
INIT_LIST_HEAD(&obj->batch_pool_link);
obj->ops = ops;
- obj->fence_reg = I915_FENCE_REG_NONE;
obj->madv = I915_MADV_WILLNEED;
i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
@@ -4405,6 +4416,7 @@ void
i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
{
struct drm_device *dev = &dev_priv->drm;
+ int i;
if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
!IS_CHERRYVIEW(dev_priv))
@@ -4420,6 +4432,13 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
I915_READ(vgtif_reg(avail_rs.fence_num));
/* Initialize fence registers to zero */
+ for (i = 0; i < dev_priv->num_fence_regs; i++) {
+ struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
+
+ fence->i915 = dev_priv;
+ fence->id = i;
+ list_add_tail(&fence->lru_list, &dev_priv->mm.fence_list);
+ }
i915_gem_restore_fences(dev);
i915_gem_detect_bit_6_swizzle(dev);
@@ -4455,8 +4474,6 @@ i915_gem_load_init(struct drm_device *dev)
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
for (i = 0; i < I915_NUM_ENGINES; i++)
init_engine_lists(&dev_priv->engine[i]);
- for (i = 0; i < I915_MAX_NUM_FENCES; i++)
- INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
i915_gem_retire_work_handler);
INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
@@ -4466,8 +4483,6 @@ i915_gem_load_init(struct drm_device *dev)
dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
- INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-
init_waitqueue_head(&dev_priv->pending_flip_queue);
dev_priv->mm.interruptible = true;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index f76c06e92677..fde61d7adf4d 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -47,7 +47,7 @@ gpu_is_idle(struct drm_i915_private *dev_priv)
}
static bool
-mark_free(struct i915_vma *vma, struct list_head *unwind)
+mark_free(struct i915_vma *vma, unsigned int flags, struct list_head *unwind)
{
if (i915_vma_is_pinned(vma))
return false;
@@ -55,6 +55,9 @@ mark_free(struct i915_vma *vma, struct list_head *unwind)
if (WARN_ON(!list_empty(&vma->exec_list)))
return false;
+ if (flags & PIN_NOFAULT && vma->obj->fault_mappable)
+ return false;
+
list_add(&vma->exec_list, unwind);
return drm_mm_scan_add_block(&vma->node);
}
@@ -129,7 +132,7 @@ search_again:
phase = phases;
do {
list_for_each_entry(vma, *phase, vm_link)
- if (mark_free(vma, &eviction_list))
+ if (mark_free(vma, flags, &eviction_list))
goto found;
} while (*++phase);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index a93fa0ca81d6..896d88c8a9fa 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -250,7 +250,6 @@ static void
i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
{
struct drm_i915_gem_exec_object2 *entry;
- struct drm_i915_gem_object *obj = vma->obj;
if (!drm_mm_node_allocated(&vma->node))
return;
@@ -258,7 +257,7 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
entry = vma->exec_entry;
if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
- i915_gem_object_unpin_fence(obj);
+ i915_vma_unpin_fence(vma);
if (entry->flags & __EXEC_OBJECT_HAS_PIN)
__i915_vma_unpin(vma);
@@ -451,7 +450,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
if (ret)
return ERR_PTR(ret);
} else {
- ret = i915_gem_object_put_fence(obj);
+ ret = i915_vma_put_fence(vma);
if (ret) {
i915_vma_unpin(vma);
return ERR_PTR(ret);
@@ -807,11 +806,11 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
entry->flags |= __EXEC_OBJECT_HAS_PIN;
if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
- ret = i915_gem_object_get_fence(obj);
+ ret = i915_vma_get_fence(vma);
if (ret)
return ret;
- if (i915_gem_object_pin_fence(obj))
+ if (i915_vma_pin_fence(vma))
entry->flags |= __EXEC_OBJECT_HAS_FENCE;
}
@@ -1308,15 +1307,8 @@ void i915_vma_move_to_active(struct i915_vma *vma,
obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
}
- if (flags & EXEC_OBJECT_NEEDS_FENCE) {
- i915_gem_active_set(&obj->last_fence, req);
- if (flags & __EXEC_OBJECT_HAS_FENCE) {
- struct drm_i915_private *dev_priv = req->i915;
-
- list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
- &dev_priv->mm.fence_list);
- }
- }
+ if (flags & EXEC_OBJECT_NEEDS_FENCE)
+ i915_gem_active_set(&vma->last_fence, req);
i915_vma_set_active(vma, idx);
i915_gem_active_set(&vma->last_read[idx], req);
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c
index 57b7834ac8f9..2d6d8c4ac8ec 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence.c
@@ -55,71 +55,70 @@
* CPU ptes into GTT mmaps (not the GTT ptes themselves) as needed.
*/
-static void i965_write_fence_reg(struct drm_device *dev, int reg,
- struct drm_i915_gem_object *obj)
+static void i965_write_fence_reg(struct drm_i915_fence_reg *fence,
+ struct i915_vma *vma)
{
- struct drm_i915_private *dev_priv = to_i915(dev);
i915_reg_t fence_reg_lo, fence_reg_hi;
int fence_pitch_shift;
+ u64 val;
- if (INTEL_INFO(dev)->gen >= 6) {
- fence_reg_lo = FENCE_REG_GEN6_LO(reg);
- fence_reg_hi = FENCE_REG_GEN6_HI(reg);
+ if (INTEL_INFO(fence->i915)->gen >= 6) {
+ fence_reg_lo = FENCE_REG_GEN6_LO(fence->id);
+ fence_reg_hi = FENCE_REG_GEN6_HI(fence->id);
fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT;
+
} else {
- fence_reg_lo = FENCE_REG_965_LO(reg);
- fence_reg_hi = FENCE_REG_965_HI(reg);
+ fence_reg_lo = FENCE_REG_965_LO(fence->id);
+ fence_reg_hi = FENCE_REG_965_HI(fence->id);
fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
}
- /* To w/a incoherency with non-atomic 64-bit register updates,
- * we split the 64-bit update into two 32-bit writes. In order
- * for a partial fence not to be evaluated between writes, we
- * precede the update with write to turn off the fence register,
- * and only enable the fence as the last step.
- *
- * For extra levels of paranoia, we make sure each step lands
- * before applying the next step.
- */
- I915_WRITE(fence_reg_lo, 0);
- POSTING_READ(fence_reg_lo);
-
- if (obj) {
- unsigned int tiling = i915_gem_object_get_tiling(obj);
- unsigned int stride = i915_gem_object_get_stride(obj);
- struct i915_vma *vma = i915_gem_object_to_ggtt(obj, NULL);
- u32 row_size = stride * (tiling == I915_TILING_Y ? 32 : 8);
+ if (vma) {
+ unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
+ unsigned int tiling_y = tiling == I915_TILING_Y;
+ unsigned int stride = i915_gem_object_get_stride(vma->obj);
+ u32 row_size = stride * (tiling_y ? 32 : 8);
u32 size = (u32)vma->node.size / row_size * row_size;
- u64 val;
val = ((vma->node.start + size - 4096) & 0xfffff000) << 32;
val |= vma->node.start & 0xfffff000;
val |= (u64)((stride / 128) - 1) << fence_pitch_shift;
- if (tiling == I915_TILING_Y)
+ if (tiling_y)
val |= 1 << I965_FENCE_TILING_Y_SHIFT;
val |= I965_FENCE_REG_VALID;
+ } else
+ val = 0;
- I915_WRITE(fence_reg_hi, val >> 32);
- POSTING_READ(fence_reg_hi);
+ if (1) {
+ struct drm_i915_private *dev_priv = fence->i915;
- I915_WRITE(fence_reg_lo, val);
+ /* To w/a incoherency with non-atomic 64-bit register updates,
+ * we split the 64-bit update into two 32-bit writes. In order
+ * for a partial fence not to be evaluated between writes, we
+ * precede the update with write to turn off the fence register,
+ * and only enable the fence as the last step.
+ *
+ * For extra levels of paranoia, we make sure each step lands
+ * before applying the next step.
+ */
+ I915_WRITE(fence_reg_lo, 0);
+ POSTING_READ(fence_reg_lo);
+
+ I915_WRITE(fence_reg_hi, upper_32_bits(val));
+ I915_WRITE(fence_reg_lo, lower_32_bits(val));
POSTING_READ(fence_reg_lo);
- } else {
- I915_WRITE(fence_reg_hi, 0);
- POSTING_READ(fence_reg_hi);
}
}
-static void i915_write_fence_reg(struct drm_device *dev, int reg,
- struct drm_i915_gem_object *obj)
+static void i915_write_fence_reg(struct drm_i915_fence_reg *fence,
+ struct i915_vma *vma)
{
- struct drm_i915_private *dev_priv = to_i915(dev);
u32 val;
- if (obj) {
- struct i915_vma *vma = i915_gem_object_to_ggtt(obj, NULL);
- unsigned int tiling = i915_gem_object_get_tiling(obj);
- unsigned int stride = i915_gem_object_get_stride(obj);
+ if (vma) {
+ unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
+ unsigned int tiling_y = tiling == I915_TILING_Y;
+ unsigned int stride = i915_gem_object_get_stride(vma->obj);
int pitch_val;
int tile_width;
@@ -131,7 +130,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
i915_vma_is_map_and_fenceable(vma),
(long)vma->node.size);
- if (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
+ if (tiling_y && HAS_128_BYTE_Y_TILING(fence->i915))
tile_width = 128;
else
tile_width = 512;
@@ -141,7 +140,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
pitch_val = ffs(pitch_val) - 1;
val = vma->node.start;
- if (tiling == I915_TILING_Y)
+ if (tiling_y)
val |= 1 << I830_FENCE_TILING_Y_SHIFT;
val |= I915_FENCE_SIZE_BITS(vma->node.size);
val |= pitch_val << I830_FENCE_PITCH_SHIFT;
@@ -149,20 +148,23 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
} else
val = 0;
- I915_WRITE(FENCE_REG(reg), val);
- POSTING_READ(FENCE_REG(reg));
+ if (1) {
+ struct drm_i915_private *dev_priv = fence->i915;
+ i915_reg_t reg = FENCE_REG(fence->id);
+
+ I915_WRITE(reg, val);
+ POSTING_READ(reg);
+ }
}
-static void i830_write_fence_reg(struct drm_device *dev, int reg,
- struct drm_i915_gem_object *obj)
+static void i830_write_fence_reg(struct drm_i915_fence_reg *fence,
+ struct i915_vma *vma)
{
- struct drm_i915_private *dev_priv = to_i915(dev);
- uint32_t val;
+ u32 val;
- if (obj) {
- struct i915_vma *vma = i915_gem_object_to_ggtt(obj, NULL);
- unsigned int tiling = i915_gem_object_get_tiling(obj);
- unsigned int stride = i915_gem_object_get_stride(obj);
+ if (vma) {
+ unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
+ unsigned int stride = i915_gem_object_get_stride(vma->obj);
uint32_t pitch_val;
WARN((vma->node.start & ~I830_FENCE_START_MASK) ||
@@ -183,96 +185,95 @@ static void i830_write_fence_reg(struct drm_device *dev, int reg,
} else
val = 0;
- I915_WRITE(FENCE_REG(reg), val);
- POSTING_READ(FENCE_REG(reg));
-}
+ if (1) {
+ struct drm_i915_private *dev_priv = fence->i915;
+ i915_reg_t reg = FENCE_REG(fence->id);
-inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
-{
- return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
+ I915_WRITE(reg, val);
+ POSTING_READ(reg);
+ }
}
-static void i915_gem_write_fence(struct drm_device *dev, int reg,
- struct drm_i915_gem_object *obj)
+static void fence_write(struct drm_i915_fence_reg *fence,
+ struct i915_vma *vma)
{
- struct drm_i915_private *dev_priv = to_i915(dev);
-
- /* Ensure that all CPU reads are completed before installing a fence
- * and all writes before removing the fence.
+ /* Previous access through the fence register is marshalled by
+ * the mb() inside the fault handlers (i915_gem_release_mmaps)
+ * and explicitly managed for internal users.
*/
- if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
- mb();
-
- WARN(obj &&
- (!i915_gem_object_get_stride(obj) ||
- !i915_gem_object_get_tiling(obj)),
- "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
- i915_gem_object_get_stride(obj),
- i915_gem_object_get_tiling(obj));
-
- if (IS_GEN2(dev))
- i830_write_fence_reg(dev, reg, obj);
- else if (IS_GEN3(dev))
- i915_write_fence_reg(dev, reg, obj);
- else if (INTEL_INFO(dev)->gen >= 4)
- i965_write_fence_reg(dev, reg, obj);
-
- /* And similarly be paranoid that no direct access to this region
- * is reordered to before the fence is installed.
+
+ if (IS_GEN2(fence->i915))
+ i830_write_fence_reg(fence, vma);
+ else if (IS_GEN3(fence->i915))
+ i915_write_fence_reg(fence, vma);
+ else
+ i965_write_fence_reg(fence, vma);
+
+ /* Access through the fenced region afterwards is
+ * ordered by the posting reads whilst writing the registers.
*/
- if (i915_gem_object_needs_mb(obj))
- mb();
-}
-static inline int fence_number(struct drm_i915_private *dev_priv,
- struct drm_i915_fence_reg *fence)
-{
- return fence - dev_priv->fence_regs;
+ fence->dirty = false;
}
-static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
- struct drm_i915_fence_reg *fence,
- bool enable)
+static int fence_update(struct drm_i915_fence_reg *fence,
+ struct i915_vma *vma)
{
- struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
- int reg = fence_number(dev_priv, fence);
+ int ret;
- i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
+ if (vma) {
+ if (!i915_vma_is_map_and_fenceable(vma))
+ return -EINVAL;
- if (enable) {
- obj->fence_reg = reg;
- fence->obj = obj;
- list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
- } else {
- obj->fence_reg = I915_FENCE_REG_NONE;
- fence->obj = NULL;
- list_del_init(&fence->lru_list);
+ if (WARN(!i915_gem_object_get_stride(vma->obj) ||
+ !i915_gem_object_get_tiling(vma->obj),
+ "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
+ i915_gem_object_get_stride(vma->obj),
+ i915_gem_object_get_tiling(vma->obj)))
+ return -EINVAL;
+
+ ret = i915_gem_active_retire(&vma->last_fence,
+ &vma->obj->base.dev->struct_mutex);
+ if (ret)
+ return ret;
}
- obj->fence_dirty = false;
-}
-static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
-{
- if (i915_gem_object_is_tiled(obj))
- i915_gem_release_mmap(obj);
+ if (fence->vma) {
+ ret = i915_gem_active_retire(&fence->vma->last_fence,
+ &fence->vma->obj->base.dev->struct_mutex);
+ if (ret)
+ return ret;
+ }
- /* As we do not have an associated fence register, we will force
- * a tiling change if we ever need to acquire one.
- */
- obj->fence_dirty = false;
- obj->fence_reg = I915_FENCE_REG_NONE;
-}
+ if (fence->vma && fence->vma != vma) {
+ /* Ensure that all userspace CPU access is completed before
+ * stealing the fence.
+ */
+ i915_gem_release_mmap(fence->vma->obj);
-static int
-i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
-{
- return i915_gem_active_retire(&obj->last_fence,
- &obj->base.dev->struct_mutex);
+ fence->vma->fence = NULL;
+ fence->vma = NULL;
+
+ list_move(&fence->lru_list, &fence->i915->mm.fence_list);
+ }
+
+ fence_write(fence, vma);
+
+ if (vma) {
+ if (fence->vma != vma) {
+ vma->fence = fence;
+ fence->vma = vma;
+ }
+
+ list_move_tail(&fence->lru_list, &fence->i915->mm.fence_list);
+ }
+
+ return 0;
}
/**
- * i915_gem_object_put_fence - force-remove fence for an object
- * @obj: object to map through a fence reg
+ * i915_vma_put_fence - force-remove fence for a VMA
+ * @vma: vma to map linearly (not through a fence reg)
*
* This function force-removes any fence from the given object, which is useful
* if the kernel wants to do untiled GTT access.
@@ -282,70 +283,40 @@ i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
* 0 on success, negative error code on failure.
*/
int
-i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
+i915_vma_put_fence(struct i915_vma *vma)
{
- struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
- struct drm_i915_fence_reg *fence;
- int ret;
+ struct drm_i915_fence_reg *fence = vma->fence;
- ret = i915_gem_object_wait_fence(obj);
- if (ret)
- return ret;
-
- if (obj->fence_reg == I915_FENCE_REG_NONE)
+ if (!fence)
return 0;
- fence = &dev_priv->fence_regs[obj->fence_reg];
-
if (WARN_ON(fence->pin_count))
return -EBUSY;
- i915_gem_object_fence_lost(obj);
- i915_gem_object_update_fence(obj, fence, false);
-
- return 0;
+ return fence_update(fence, NULL);
}
-static struct drm_i915_fence_reg *
-i915_find_fence_reg(struct drm_device *dev)
+static struct drm_i915_fence_reg *fence_find(struct drm_i915_private *dev_priv)
{
- struct drm_i915_private *dev_priv = to_i915(dev);
- struct drm_i915_fence_reg *reg, *avail;
- int i;
-
- /* First try to find a free reg */
- avail = NULL;
- for (i = 0; i < dev_priv->num_fence_regs; i++) {
- reg = &dev_priv->fence_regs[i];
- if (!reg->obj)
- return reg;
-
- if (!reg->pin_count)
- avail = reg;
- }
-
- if (avail == NULL)
- goto deadlock;
+ struct drm_i915_fence_reg *fence;
- /* None available, try to steal one or wait for a user to finish */
- list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
- if (reg->pin_count)
+ list_for_each_entry(fence, &dev_priv->mm.fence_list, lru_list) {
+ if (fence->pin_count)
continue;
- return reg;
+ return fence;
}
-deadlock:
/* Wait for completion of pending flips which consume fences */
- if (intel_has_pending_fb_unpin(dev))
+ if (intel_has_pending_fb_unpin(&dev_priv->drm))
return ERR_PTR(-EAGAIN);
return ERR_PTR(-EDEADLK);
}
/**
- * i915_gem_object_get_fence - set up fencing for an object
- * @obj: object to map through a fence reg
+ * i915_vma_get_fence - set up fencing for a vma
+ * @vma: vma to map through a fence reg
*
* When mapping objects through the GTT, userspace wants to be able to write
* to them without having to worry about swizzling if the object is tiled.
@@ -362,94 +333,27 @@ deadlock:
* 0 on success, negative error code on failure.
*/
int
-i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
+i915_vma_get_fence(struct i915_vma *vma)
{
- struct drm_device *dev = obj->base.dev;
- struct drm_i915_private *dev_priv = to_i915(dev);
- bool enable = i915_gem_object_is_tiled(obj);
- struct drm_i915_fence_reg *reg;
- int ret;
-
- /* Have we updated the tiling parameters upon the object and so
- * will need to serialise the write to the associated fence register?
- */
- if (obj->fence_dirty) {
- ret = i915_gem_object_wait_fence(obj);
- if (ret)
- return ret;
- }
+ struct drm_i915_fence_reg *fence;
+ struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
/* Just update our place in the LRU if our fence is getting reused. */
- if (obj->fence_reg != I915_FENCE_REG_NONE) {
- reg = &dev_priv->fence_regs[obj->fence_reg];
- if (!obj->fence_dirty) {
- list_move_tail(®->lru_list,
- &dev_priv->mm.fence_list);
+ if (vma->fence) {
+ fence = vma->fence;
+ if (!fence->dirty) {
+ list_move_tail(&fence->lru_list,
+ &fence->i915->mm.fence_list);
return 0;
}
- } else if (enable) {
- reg = i915_find_fence_reg(dev);
- if (IS_ERR(reg))
- return PTR_ERR(reg);
-
- if (reg->obj) {
- struct drm_i915_gem_object *old = reg->obj;
-
- ret = i915_gem_object_wait_fence(old);
- if (ret)
- return ret;
-
- i915_gem_object_fence_lost(old);
- }
+ } else if (set) {
+ fence = fence_find(to_i915(vma->vm->dev));
+ if (IS_ERR(fence))
+ return PTR_ERR(fence);
} else
return 0;
- i915_gem_object_update_fence(obj, reg, enable);
-
- return 0;
-}
-
-/**
- * i915_gem_object_pin_fence - pin fencing state
- * @obj: object to pin fencing for
- *
- * This pins the fencing state (whether tiled or untiled) to make sure the
- * object is ready to be used as a scanout target. Fencing status must be
- * synchronize first by calling i915_gem_object_get_fence():
- *
- * The resulting fence pin reference must be released again with
- * i915_gem_object_unpin_fence().
- *
- * Returns:
- *
- * True if the object has a fence, false otherwise.
- */
-bool
-i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
-{
- if (obj->fence_reg != I915_FENCE_REG_NONE) {
- to_i915(obj->base.dev)->fence_regs[obj->fence_reg].pin_count++;
- return true;
- } else
- return false;
-}
-
-/**
- * i915_gem_object_unpin_fence - unpin fencing state
- * @obj: object to unpin fencing for
- *
- * This releases the fence pin reference acquired through
- * i915_gem_object_pin_fence. It will handle both objects with and without an
- * attached fence correctly, callers do not need to distinguish this.
- */
-void
-i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
-{
- if (obj->fence_reg != I915_FENCE_REG_NONE) {
- struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
- WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
- dev_priv->fence_regs[obj->fence_reg].pin_count--;
- }
+ return fence_update(fence, set);
}
/**
@@ -471,12 +375,7 @@ void i915_gem_restore_fences(struct drm_device *dev)
* Commit delayed tiling changes if we have an object still
* attached to the fence, otherwise just clear the fence.
*/
- if (reg->obj) {
- i915_gem_object_update_fence(reg->obj, reg,
- i915_gem_object_get_tiling(reg->obj));
- } else {
- i915_gem_write_fence(dev, i, NULL);
- }
+ fence_write(reg, reg->vma);
}
}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 0e0ff9fdceda..bcd21af1471e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -2299,12 +2299,37 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
{
- if (!dma_map_sg(&obj->base.dev->pdev->dev,
- obj->pages->sgl, obj->pages->nents,
- PCI_DMA_BIDIRECTIONAL))
- return -ENOSPC;
+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
+ struct device *dma_device = &obj->base.dev->pdev->dev;
- return 0;
+ if (dma_map_sg(dma_device,
+ obj->pages->sgl, obj->pages->nents,
+ PCI_DMA_BIDIRECTIONAL))
+ return 0;
+
+ /* If we fail here, it is likely we couldn't allocate the map, or
+ * we have exhausted the available DMAR space. First throw out
+ * objects to make enough rooom for the mapping and try again,
+ * though this doesn't take fragmentation into account. So if we
+ * still fail, throw out everything we can and start afresh.
+ */
+ i915_gem_shrink(i915,
+ obj->base.size >> PAGE_SHIFT,
+ I915_SHRINK_BOUND |
+ I915_SHRINK_UNBOUND |
+ I915_SHRINK_PURGEABLE);
+ if (dma_map_sg(dma_device,
+ obj->pages->sgl, obj->pages->nents,
+ PCI_DMA_BIDIRECTIONAL))
+ return 0;
+
+ i915_gem_shrink_all(i915);
+ if (dma_map_sg(dma_device,
+ obj->pages->sgl, obj->pages->nents,
+ PCI_DMA_BIDIRECTIONAL))
+ return 0;
+
+ return -ENOSPC;
}
static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
@@ -3333,6 +3358,7 @@ void i915_vma_destroy(struct i915_vma *vma)
GEM_BUG_ON(vma->node.allocated);
GEM_BUG_ON(i915_vma_is_active(vma));
GEM_BUG_ON(!i915_vma_is_closed(vma));
+ GEM_BUG_ON(vma->fence);
list_del(&vma->vm_link);
if (!i915_vma_is_ggtt(vma))
@@ -3368,6 +3394,7 @@ __i915_gem_vma_create(struct drm_i915_gem_object *obj,
INIT_LIST_HEAD(&vma->exec_list);
for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
init_request_active(&vma->last_read[i], i915_vma_retire);
+ init_request_active(&vma->last_fence, NULL);
list_add(&vma->vm_link, &vm->unbound_list);
vma->vm = vm;
vma->obj = obj;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 01bcba9c5cc4..cae8b67b99c4 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -38,7 +38,13 @@
#include "i915_gem_request.h"
+#define I915_FENCE_REG_NONE -1
+#define I915_MAX_NUM_FENCES 32
+/* 32 fences + sign bit for FENCE_REG_NONE */
+#define I915_MAX_NUM_FENCE_BITS 6
+
struct drm_i915_file_private;
+struct drm_i915_fence_reg;
typedef uint32_t gen6_pte_t;
typedef uint64_t gen8_pte_t;
@@ -179,8 +185,9 @@ struct i915_vma {
struct drm_mm_node node;
struct drm_i915_gem_object *obj;
struct i915_address_space *vm;
+ struct drm_i915_fence_reg *fence;
void __iomem *iomap;
- u64 size;
+ u64 size, display_alignment;
unsigned int flags;
/**
@@ -207,6 +214,7 @@ struct i915_vma {
unsigned int active;
struct i915_gem_active last_read[I915_NUM_ENGINES];
+ struct i915_gem_active last_fence;
/**
* Support different GGTT views into the same object.
@@ -620,6 +628,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
#define PIN_NONBLOCK BIT(0)
#define PIN_MAPPABLE BIT(1)
#define PIN_ZONE_4G BIT(2)
+#define PIN_NOFAULT BIT(3)
#define PIN_MBZ BIT(5) /* I915_VMA_PIN_OVERFLOW */
#define PIN_GLOBAL BIT(6) /* I915_VMA_GLOBAL_BIND */
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index bd2df61b7b95..567e0e6ba392 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -116,13 +116,39 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
return true;
}
+static bool i915_vma_fence_ok(struct i915_vma *vma, int tiling_mode)
+{
+ struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
+ u32 size;
+
+ if (!i915_vma_is_map_and_fenceable(vma))
+ return true;
+
+ if (INTEL_GEN(dev_priv) == 3) {
+ if (vma->node.start & ~I915_FENCE_START_MASK)
+ return false;
+ } else {
+ if (vma->node.start & ~I830_FENCE_START_MASK)
+ return false;
+ }
+
+ size = i915_gem_get_ggtt_size(dev_priv, vma->size, tiling_mode);
+ if (vma->node.size < size)
+ return false;
+
+ if (vma->node.start & (size - 1))
+ return false;
+
+ return true;
+}
+
/* Is the current GTT allocation valid for the change in tiling? */
static int
i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
{
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct i915_vma *vma;
- u32 size;
+ int ret;
if (tiling_mode == I915_TILING_NONE)
return 0;
@@ -130,32 +156,16 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
if (INTEL_GEN(dev_priv) >= 4)
return 0;
- vma = i915_gem_object_to_ggtt(obj, NULL);
- if (!vma)
- return 0;
-
- if (!i915_vma_is_map_and_fenceable(vma))
- return 0;
+ list_for_each_entry(vma, &obj->vma_list, obj_link) {
+ if (i915_vma_fence_ok(vma, tiling_mode))
+ continue;
- if (IS_GEN3(dev_priv)) {
- if (vma->node.start & ~I915_FENCE_START_MASK)
- goto bad;
- } else {
- if (vma->node.start & ~I830_FENCE_START_MASK)
- goto bad;
+ ret = i915_vma_unbind(vma);
+ if (ret)
+ return ret;
}
- size = i915_gem_get_ggtt_size(dev_priv, vma->size, tiling_mode);
- if (vma->node.size < size)
- goto bad;
-
- if (vma->node.start & (size - 1))
- goto bad;
-
return 0;
-
-bad:
- return i915_vma_unbind(vma);
}
/**
@@ -247,6 +257,8 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
*/
ret = i915_gem_object_fence_ok(obj, args->tiling_mode);
if (ret == 0) {
+ struct i915_vma *vma;
+
if (obj->pages &&
obj->madv == I915_MADV_WILLNEED &&
dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
@@ -256,11 +268,12 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
i915_gem_object_pin_pages(obj);
}
- obj->fence_dirty =
- !i915_gem_active_is_idle(&obj->last_fence,
- &dev->struct_mutex) ||
- obj->fence_reg != I915_FENCE_REG_NONE;
+ list_for_each_entry(vma, &obj->vma_list, obj_link) {
+ if (!vma->fence)
+ continue;
+ vma->fence->dirty = true;
+ }
obj->tiling_and_stride =
args->stride | args->tiling_mode;
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 57218cca7e05..be54825ef3e8 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -542,8 +542,6 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
}
}
obj->userptr.work = ERR_PTR(ret);
- if (ret)
- __i915_gem_userptr_set_active(obj, false);
}
obj->userptr.workers--;
@@ -628,15 +626,14 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
* to the vma (discard or cloning) which should prevent the more
* egregious cases from causing harm.
*/
- if (IS_ERR(obj->userptr.work)) {
- /* active flag will have been dropped already by the worker */
- ret = PTR_ERR(obj->userptr.work);
- obj->userptr.work = NULL;
- return ret;
- }
- if (obj->userptr.work)
+
+ if (obj->userptr.work) {
/* active flag should still be held for the pending work */
- return -EAGAIN;
+ if (IS_ERR(obj->userptr.work))
+ return PTR_ERR(obj->userptr.work);
+ else
+ return -EAGAIN;
+ }
/* Let the mmu-notifier know that we have begun and need cancellation */
ret = __i915_gem_userptr_set_active(obj, true);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 24ee630ddba6..e9631a582878 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -761,7 +761,7 @@ static void capture_bo(struct drm_i915_error_buffer *err,
err->gtt_offset = vma->node.start;
err->read_domains = obj->base.read_domains;
err->write_domain = obj->base.write_domain;
- err->fence_reg = obj->fence_reg;
+ err->fence_reg = vma->fence ? vma->fence->id : -1;
err->tiling = i915_gem_object_get_tiling(obj);
err->dirty = obj->dirty;
err->purgeable = obj->madv != I915_MADV_WILLNEED;
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index ae362ca8cfea..e24591e1d092 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -2215,7 +2215,6 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
struct i915_ggtt_view view;
struct i915_vma *vma;
u32 alignment;
- int ret;
WARN_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -2241,43 +2240,33 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
intel_runtime_pm_get(dev_priv);
vma = i915_gem_object_pin_to_display_plane(obj, alignment, &view);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err_pm;
- }
+ if (IS_ERR(vma))
+ goto err;
- /* Install a fence for tiled scan-out. Pre-i965 always needs a
- * fence, whereas 965+ only requires a fence if using
- * framebuffer compression. For simplicity, we always install
- * a fence as the cost is not that onerous.
- */
if (i915_vma_is_map_and_fenceable(vma)) {
- ret = i915_gem_object_get_fence(obj);
- if (ret == -EDEADLK) {
- /*
- * -EDEADLK means there are no free fences
- * no pending flips.
- *
- * This is propagated to atomic, but it uses
- * -EDEADLK to force a locking recovery, so
- * change the returned error to -EBUSY.
- */
- ret = -EBUSY;
- goto err_unpin;
- } else if (ret)
- goto err_unpin;
-
- i915_gem_object_pin_fence(obj);
+ /* Install a fence for tiled scan-out. Pre-i965 always needs a
+ * fence, whereas 965+ only requires a fence if using
+ * framebuffer compression. For simplicity, we always, when
+ * possible, install a fence as the cost is not that onerous.
+ *
+ * If we fail to fence the tiled scanout, then either the
+ * modeset will reject the change (which is highly unlikely as
+ * the affected systems, all but one, do not have unmappable
+ * space) or we will not be able to enable full powersaving
+ * techniques (also likely not to apply due to various limits
+ * FBC and the like impose on the size of the buffer, which
+ * presumably we violated anyway with this unmappable buffer).
+ * Anyway, it is presumably better to stumble onwards with
+ * something and try to run the system in a "less than optimal"
+ * mode that matches the user configuration.
+ */
+ if (i915_vma_get_fence(vma) == 0)
+ i915_vma_pin_fence(vma);
}
+err:
intel_runtime_pm_put(dev_priv);
return vma;
-
-err_unpin:
- i915_gem_object_unpin_from_display_plane(vma);
-err_pm:
- intel_runtime_pm_put(dev_priv);
- return ERR_PTR(ret);
}
void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
@@ -2291,9 +2280,7 @@ void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
intel_fill_fb_ggtt_view(&view, fb, rotation);
vma = i915_gem_object_to_ggtt(obj, &view);
- if (i915_vma_is_map_and_fenceable(vma))
- i915_gem_object_unpin_fence(obj);
-
+ i915_vma_unpin_fence(vma);
i915_gem_object_unpin_from_display_plane(vma);
}
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index c2ef71a1215e..9e713a836fb6 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -244,7 +244,6 @@ static void ilk_fbc_activate(struct drm_i915_private *dev_priv)
dpfc_ctl |= DPFC_CTL_LIMIT_1X;
break;
}
- dpfc_ctl |= DPFC_CTL_FENCE_EN;
if (IS_GEN5(dev_priv))
dpfc_ctl |= params->fb.fence_reg;
@@ -709,6 +708,14 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
return effective_w <= max_w && effective_h <= max_h;
}
+/* XXX replace me when we have VMA tracking for intel_plane_state */
+static int get_fence_id(struct drm_framebuffer *fb)
+{
+ struct i915_vma *vma = i915_gem_object_to_ggtt(intel_fb_obj(fb), NULL);
+
+ return vma->fence ? vma->fence->id : I915_FENCE_REG_NONE;
+}
+
static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
struct intel_crtc_state *crtc_state,
struct intel_plane_state *plane_state)
@@ -740,7 +747,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
cache->fb.ilk_ggtt_offset = i915_gem_object_ggtt_offset(obj, NULL);
cache->fb.pixel_format = fb->pixel_format;
cache->fb.stride = fb->pitches[0];
- cache->fb.fence_reg = obj->fence_reg;
+ cache->fb.fence_reg = get_fence_id(fb);
cache->fb.tiling_mode = i915_gem_object_get_tiling(obj);
}
@@ -768,6 +775,10 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
/* The use of a CPU fence is mandatory in order to detect writes
* by the CPU to the scanout and trigger updates to the FBC.
+ *
+ * Note that is possible for a tiled surface to be unmappable (and
+ * so have no fence associated with it) due to aperture constaints
+ * at the time of pinning.
*/
if (cache->fb.tiling_mode != I915_TILING_X ||
cache->fb.fence_reg == I915_FENCE_REG_NONE) {
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 0753cb34527d..cd6cc5c52eaf 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -755,7 +755,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
if (IS_ERR(vma))
return PTR_ERR(vma);
- ret = i915_gem_object_put_fence(new_bo);
+ ret = i915_vma_put_fence(vma);
if (ret)
goto out_unpin;
--
2.8.1
More information about the Intel-gfx-trybot
mailing list