[PATCH 03/14] drm/i915: Cache kmap between relocations
Chris Wilson
chris at chris-wilson.co.uk
Sat Aug 6 13:33:57 UTC 2016
When doing relocations, we have to obtain a mapping to the page
containing the target address. This is either a kmap or iomap depending
on GPU and its cache coherency. Neighbouring relocation entries are
typically within the same page and so we can cache our kmapping between
them and avoid those pesky TLB flushes.
Note that there is some sleight-of-hand in how the slow relocate works
as the reloc_entry_cache implies pagefaults disabled (as we are inside a
kmap_atomic section). However, the slow relocate code is meant to be the
fallback from the atomic fast path failing. Fortunately it works as we
already have performed the copy_from_user for the relocation array (no
more pagefaults there) and the kmap_atomic cache is enabled after we
have waited upon an active buffer (so no more sleeping in atomic).
Magic!
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_cmd_parser.c | 2 +-
drivers/gpu/drm/i915/i915_debugfs.c | 46 ++--
drivers/gpu/drm/i915/i915_drv.h | 12 +-
drivers/gpu/drm/i915/i915_gem.c | 208 +++++++++-------
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 370 +++++++++++++++++++----------
drivers/gpu/drm/i915/i915_gem_fence.c | 5 +-
drivers/gpu/drm/i915/i915_gem_gtt.c | 2 +-
drivers/gpu/drm/i915/i915_gem_gtt.h | 10 +-
drivers/gpu/drm/i915/i915_gem_tiling.c | 4 +-
drivers/gpu/drm/i915/intel_display.c | 6 +-
10 files changed, 404 insertions(+), 261 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 1db829c8b912..c21cab7d1d61 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -973,7 +973,7 @@ static u32 *copy_batch(struct drm_i915_gem_object *dest_obj,
u32 batch_start_offset,
u32 batch_len)
{
- int needs_clflush = 0;
+ unsigned int needs_clflush;
void *src_base, *src;
void *dst = NULL;
int ret;
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 17ca6d50d0fa..c569c858f2f1 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -119,19 +119,6 @@ static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
return obj->mapping ? 'M' : ' ';
}
-static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
-{
- u64 size = 0;
- struct i915_vma *vma;
-
- list_for_each_entry(vma, &obj->vma_list, obj_link) {
- if (i915_vma_is_ggtt(vma) && drm_mm_node_allocated(&vma->node))
- size += vma->node.size;
- }
-
- return size;
-}
-
static void
describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
{
@@ -321,17 +308,6 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
return 0;
}
-#define count_objects(list, member) do { \
- list_for_each_entry(obj, list, member) { \
- size += i915_gem_obj_total_ggtt_size(obj); \
- ++count; \
- if (obj->map_and_fenceable) { \
- mappable_size += obj->base.size; \
- ++mappable_count; \
- } \
- } \
-} while (0)
-
struct file_stats {
struct drm_i915_file_private *file_priv;
unsigned long count;
@@ -450,7 +426,7 @@ static void print_context_stats(struct seq_file *m,
list_for_each_entry(vma, list, member) { \
size += vma->size; \
++count; \
- if (vma->obj->map_and_fenceable) { \
+ if (i915_vma_is_map_and_fenceable(vma)) { \
mappable_size += vma->size; \
++mappable_count; \
} \
@@ -481,7 +457,25 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
dev_priv->mm.object_memory);
size = count = mappable_size = mappable_count = 0;
- count_objects(&dev_priv->mm.bound_list, global_list);
+ list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+ bool allocated = false, mappable = false;
+
+ list_for_each_entry(vma, &obj->vma_list, obj_link) {
+ if (!i915_vma_is_ggtt(vma))
+ continue;
+
+ allocated = true;
+ size += vma->node.size;
+
+ if (i915_vma_is_map_and_fenceable(vma)) {
+ mappable = true;
+ mappable_size += vma->node.size;
+ }
+ }
+
+ count += allocated;
+ mappable_count += mappable;
+ }
seq_printf(m, "%u [%u] objects, %llu [%llu] bytes in gtt\n",
count, mappable_count, size, mappable_size);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 71292ed54cc5..c83c779c3dd1 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2198,12 +2198,6 @@ struct drm_i915_gem_object {
unsigned int fence_dirty:1;
/**
- * Is the object at the current location in the gtt mappable and
- * fenceable? Used to avoid costly recalculations.
- */
- unsigned int map_and_fenceable:1;
-
- /**
* Whether the current gtt mapping needs to be mappable (and isn't just
* mappable by accident). Track pin and fault separate for a more
* accurate mappable working set.
@@ -3092,7 +3086,11 @@ void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
- int *needs_clflush);
+ unsigned int *needs_clflush);
+int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
+ unsigned int *needs_clflush);
+#define CLFLUSH_BEFORE 0x1
+#define CLFLUSH_AFTER 0x2
int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 8783674e77a6..800533b58344 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -600,34 +600,106 @@ __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
* flush the object from the CPU cache.
*/
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
- int *needs_clflush)
+ unsigned int *needs_clflush)
{
int ret;
*needs_clflush = 0;
- if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
- return -EINVAL;
+ if (!i915_gem_object_has_struct_page(obj))
+ return -ENODEV;
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
return ret;
- if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
- /* If we're not in the cpu read domain, set ourself into the gtt
- * read domain and manually flush cachelines (if required). This
- * optimizes for the case when the gpu will dirty the data
- * anyway again before the next pread happens. */
+ ret = i915_gem_object_get_pages(obj);
+ if (ret)
+ return ret;
+
+ i915_gem_object_pin_pages(obj);
+
+ i915_gem_object_flush_gtt_write_domain(obj);
+
+ /* If we're not in the cpu read domain, set ourself into the gtt
+ * read domain and manually flush cachelines (if required). This
+ * optimizes for the case when the gpu will dirty the data
+ * anyway again before the next pread happens.
+ */
+ if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
obj->cache_level);
+
+ if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+ ret = i915_gem_object_set_to_cpu_domain(obj, false);
+ if (ret)
+ goto err_unpin;
+
+ *needs_clflush = 0;
}
+ /* return with the pages pinned */
+ return 0;
+
+err_unpin:
+ i915_gem_object_unpin_pages(obj);
+ return ret;
+}
+
+int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
+ unsigned int *needs_clflush)
+{
+ int ret;
+
+ *needs_clflush = 0;
+ if (!i915_gem_object_has_struct_page(obj))
+ return -ENODEV;
+
+ ret = i915_gem_object_wait_rendering(obj, false);
+ if (ret)
+ return ret;
+
ret = i915_gem_object_get_pages(obj);
if (ret)
return ret;
i915_gem_object_pin_pages(obj);
+ i915_gem_object_flush_gtt_write_domain(obj);
+
+ /* If we're not in the cpu write domain, set ourself into the
+ * gtt write domain and manually flush cachelines (as required).
+ * This optimizes for the case when the gpu will use the data
+ * right away and we therefore have to clflush anyway.
+ */
+ if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
+ *needs_clflush |= cpu_write_needs_clflush(obj) << 1;
+
+ /* Same trick applies to invalidate partially written cachelines read
+ * before writing.
+ */
+ if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
+ *needs_clflush |= !cpu_cache_is_coherent(obj->base.dev,
+ obj->cache_level);
+
+ if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+ ret = i915_gem_object_set_to_cpu_domain(obj, true);
+ if (ret)
+ goto err_unpin;
+
+ *needs_clflush = 0;
+ }
+
+ if ((*needs_clflush & CLFLUSH_AFTER) == 0)
+ obj->cache_dirty = true;
+
+ intel_fb_obj_invalidate(obj, ORIGIN_CPU);
+ obj->dirty = 1;
+ /* return with the pages pinned */
+ return 0;
+
+err_unpin:
+ i915_gem_object_unpin_pages(obj);
return ret;
}
@@ -860,19 +932,14 @@ i915_gem_shmem_pread(struct drm_device *dev,
int needs_clflush = 0;
struct sg_page_iter sg_iter;
- if (!i915_gem_object_has_struct_page(obj))
- return -ENODEV;
-
- user_data = u64_to_user_ptr(args->data_ptr);
- remain = args->size;
-
- obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-
ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
if (ret)
return ret;
+ obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+ user_data = u64_to_user_ptr(args->data_ptr);
offset = args->offset;
+ remain = args->size;
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
offset >> PAGE_SHIFT) {
@@ -1233,42 +1300,17 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
int shmem_page_offset, page_length, ret = 0;
int obj_do_bit17_swizzling, page_do_bit17_swizzling;
int hit_slowpath = 0;
- int needs_clflush_after = 0;
- int needs_clflush_before = 0;
+ unsigned int needs_clflush;
struct sg_page_iter sg_iter;
- user_data = u64_to_user_ptr(args->data_ptr);
- remain = args->size;
-
- obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-
- ret = i915_gem_object_wait_rendering(obj, false);
+ ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
if (ret)
return ret;
- if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
- /* If we're not in the cpu write domain, set ourself into the gtt
- * write domain and manually flush cachelines (if required). This
- * optimizes for the case when the gpu will use the data
- * right away and we therefore have to clflush anyway. */
- needs_clflush_after = cpu_write_needs_clflush(obj);
- }
- /* Same trick applies to invalidate partially written cachelines read
- * before writing. */
- if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
- needs_clflush_before =
- !cpu_cache_is_coherent(dev, obj->cache_level);
-
- ret = i915_gem_object_get_pages(obj);
- if (ret)
- return ret;
-
- intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-
- i915_gem_object_pin_pages(obj);
-
+ obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+ user_data = u64_to_user_ptr(args->data_ptr);
offset = args->offset;
- obj->dirty = 1;
+ remain = args->size;
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
offset >> PAGE_SHIFT) {
@@ -1292,7 +1334,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
/* If we don't overwrite a cacheline completely we need to be
* careful to have up-to-date data by first clflushing. Don't
* overcomplicate things and flush the entire patch. */
- partial_cacheline_write = needs_clflush_before &&
+ partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE &&
((shmem_page_offset | page_length)
& (boot_cpu_data.x86_clflush_size - 1));
@@ -1302,7 +1344,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
user_data, page_do_bit17_swizzling,
partial_cacheline_write,
- needs_clflush_after);
+ needs_clflush & CLFLUSH_AFTER);
if (ret == 0)
goto next_page;
@@ -1311,7 +1353,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
user_data, page_do_bit17_swizzling,
partial_cacheline_write,
- needs_clflush_after);
+ needs_clflush & CLFLUSH_AFTER);
mutex_lock(&dev->struct_mutex);
@@ -1333,17 +1375,15 @@ out:
* cachelines in-line while writing and the object moved
* out of the cpu write domain while we've dropped the lock.
*/
- if (!needs_clflush_after &&
+ if (!(needs_clflush & CLFLUSH_AFTER) &&
obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
if (i915_gem_clflush_object(obj, obj->pin_display))
- needs_clflush_after = true;
+ needs_clflush |= CLFLUSH_AFTER;
}
}
- if (needs_clflush_after)
+ if (needs_clflush & CLFLUSH_AFTER)
i915_gem_chipset_flush(to_i915(dev));
- else
- obj->cache_dirty = true;
intel_fb_obj_flush(obj, false, ORIGIN_CPU);
return ret;
@@ -1422,10 +1462,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
if (ret == -EFAULT || ret == -ENOSPC) {
if (obj->phys_handle)
ret = i915_gem_phys_pwrite(obj, args, file);
- else if (i915_gem_object_has_struct_page(obj))
- ret = i915_gem_shmem_pwrite(dev, obj, args, file);
else
- ret = -ENODEV;
+ ret = i915_gem_shmem_pwrite(dev, obj, args, file);
}
i915_gem_object_put(obj);
@@ -2802,8 +2840,7 @@ int i915_vma_unbind(struct i915_vma *vma)
GEM_BUG_ON(obj->bind_count == 0);
GEM_BUG_ON(!obj->pages);
- if (i915_vma_is_ggtt(vma) &&
- vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
+ if (i915_vma_is_map_and_fenceable(vma)) {
i915_gem_object_finish_gtt(obj);
/* release the fence reg _after_ flushing */
@@ -2812,6 +2849,7 @@ int i915_vma_unbind(struct i915_vma *vma)
return ret;
__i915_vma_iounmap(vma);
+ vma->flags &= ~I915_VMA_CAN_FENCE;
}
if (likely(!vma->vm->closed)) {
@@ -2823,10 +2861,8 @@ int i915_vma_unbind(struct i915_vma *vma)
drm_mm_remove_node(&vma->node);
list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
- if (i915_vma_is_ggtt(vma)) {
- if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
- obj->map_and_fenceable = false;
- } else if (vma->ggtt_view.pages) {
+ if (vma->ggtt_view.pages) {
+ if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) {
sg_free_table(vma->ggtt_view.pages);
kfree(vma->ggtt_view.pages);
}
@@ -3084,51 +3120,54 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
static void
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
{
- uint32_t old_write_domain;
+ struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
return;
/* No actual flushing is required for the GTT write domain. Writes
- * to it immediately go to main memory as far as we know, so there's
+ * to it "immediately" go to main memory as far as we know, so there's
* no chipset flush. It also doesn't land in render cache.
*
* However, we do have to enforce the order so that all writes through
* the GTT land before any writes to the device, such as updates to
* the GATT itself.
+ *
+ * We also have to wait a bit for the writes to land from the GTT.
+ * An uncached read (i.e. mmio) seems to be ideal for the round-trip
+ * timing. This issue has only been observed when switching quickly
+ * between GTT writes and CPU reads from inside the kernel on recent hw,
+ * and it appears to only affect discrete GTT blocks (i.e. on LLC
+ * system agents we cannot reproduce this behaviour).
*/
wmb();
-
- old_write_domain = obj->base.write_domain;
- obj->base.write_domain = 0;
+ if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv))
+ POSTING_READ(RING_ACTHD(dev_priv->engine[RCS].mmio_base));
intel_fb_obj_flush(obj, false, ORIGIN_GTT);
+ obj->base.write_domain = 0;
trace_i915_gem_object_change_domain(obj,
obj->base.read_domains,
- old_write_domain);
+ I915_GEM_DOMAIN_GTT);
}
/** Flushes the CPU write domain for the object if it's dirty. */
static void
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
{
- uint32_t old_write_domain;
-
if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
return;
if (i915_gem_clflush_object(obj, obj->pin_display))
i915_gem_chipset_flush(to_i915(obj->base.dev));
- old_write_domain = obj->base.write_domain;
- obj->base.write_domain = 0;
-
intel_fb_obj_flush(obj, false, ORIGIN_CPU);
+ obj->base.write_domain = 0;
trace_i915_gem_object_change_domain(obj,
obj->base.read_domains,
- old_write_domain);
+ I915_GEM_DOMAIN_CPU);
}
/**
@@ -3603,8 +3642,6 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
static bool
i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- struct drm_i915_gem_object *obj = vma->obj;
-
if (!drm_mm_node_allocated(&vma->node))
return false;
@@ -3614,7 +3651,7 @@ i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
if (alignment && vma->node.start & (alignment - 1))
return true;
- if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
+ if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
return true;
if (flags & PIN_OFFSET_BIAS &&
@@ -3636,10 +3673,10 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
u32 fence_size, fence_alignment;
fence_size = i915_gem_get_ggtt_size(dev_priv,
- obj->base.size,
+ vma->size,
i915_gem_object_get_tiling(obj));
fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
- obj->base.size,
+ vma->size,
i915_gem_object_get_tiling(obj),
true);
@@ -3649,7 +3686,10 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
mappable = (vma->node.start + fence_size <=
dev_priv->ggtt.mappable_end);
- obj->map_and_fenceable = mappable && fenceable;
+ if (mappable && fenceable)
+ vma->flags |= I915_VMA_CAN_FENCE;
+ else
+ vma->flags &= ~I915_VMA_CAN_FENCE;
}
int __i915_vma_do_pin(struct i915_vma *vma,
@@ -3709,13 +3749,13 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
WARN(i915_vma_is_pinned(vma),
"bo is already pinned in ggtt with incorrect alignment:"
- " offset=%08x %08x, req.alignment=%llx, req.map_and_fenceable=%d,"
- " obj->map_and_fenceable=%d\n",
+ " offset=%08x %08x, req.alignment=%llx,"
+ " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
upper_32_bits(vma->node.start),
lower_32_bits(vma->node.start),
alignment,
!!(flags & PIN_MAPPABLE),
- obj->map_and_fenceable);
+ i915_vma_is_map_and_fenceable(vma));
ret = i915_vma_unbind(vma);
if (ret)
return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 160867f48f91..a93fa0ca81d6 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -39,6 +39,8 @@
#include "intel_drv.h"
#include "intel_frontbuffer.h"
+#define DBG_USE_CPU_RELOC 0 /* force relocations to use the CPU write path */
+
#define __EXEC_OBJECT_HAS_PIN (1<<31)
#define __EXEC_OBJECT_HAS_FENCE (1<<30)
#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
@@ -59,6 +61,7 @@ struct i915_execbuffer_params {
};
struct eb_vmas {
+ struct drm_i915_private *i915;
struct list_head vmas;
int and;
union {
@@ -68,7 +71,8 @@ struct eb_vmas {
};
static struct eb_vmas *
-eb_create(struct drm_i915_gem_execbuffer2 *args)
+eb_create(struct drm_i915_private *i915,
+ struct drm_i915_gem_execbuffer2 *args)
{
struct eb_vmas *eb = NULL;
@@ -95,6 +99,7 @@ eb_create(struct drm_i915_gem_execbuffer2 *args)
} else
eb->and = -args->buffer_count;
+ eb->i915 = i915;
INIT_LIST_HEAD(&eb->vmas);
return eb;
}
@@ -278,6 +283,12 @@ static void eb_destroy(struct eb_vmas *eb)
static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
{
+ if (!i915_gem_object_has_struct_page(obj))
+ return false;
+
+ if (DBG_USE_CPU_RELOC)
+ return DBG_USE_CPU_RELOC > 0;
+
return (HAS_LLC(obj->base.dev) ||
obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
obj->cache_level != I915_CACHE_NONE);
@@ -302,142 +313,238 @@ static inline uint64_t gen8_noncanonical_addr(uint64_t address)
}
static inline uint64_t
-relocation_target(struct drm_i915_gem_relocation_entry *reloc,
+relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
uint64_t target_offset)
{
return gen8_canonical_addr((int)reloc->delta + target_offset);
}
-static int
-relocate_entry_cpu(struct drm_i915_gem_object *obj,
- struct drm_i915_gem_relocation_entry *reloc,
- uint64_t target_offset)
+struct reloc_cache {
+ struct drm_i915_private *i915;
+ struct drm_mm_node node;
+ unsigned long vaddr;
+ unsigned int page;
+ bool use_64bit_reloc;
+};
+
+static void reloc_cache_init(struct reloc_cache *cache,
+ struct drm_i915_private *i915)
{
- struct drm_device *dev = obj->base.dev;
- uint32_t page_offset = offset_in_page(reloc->offset);
- uint64_t delta = relocation_target(reloc, target_offset);
- char *vaddr;
- int ret;
+ cache->page = -1;
+ cache->vaddr = 0;
+ cache->i915 = i915;
+ cache->use_64bit_reloc = INTEL_GEN(cache->i915) >= 8;
+ cache->node.allocated = false;
+}
- ret = i915_gem_object_set_to_cpu_domain(obj, true);
- if (ret)
- return ret;
+static inline void *unmask_page(unsigned long p)
+{
+ return (void *)(uintptr_t)(p & PAGE_MASK);
+}
- vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
- reloc->offset >> PAGE_SHIFT));
- *(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
+static inline unsigned int unmask_flags(unsigned long p)
+{
+ return p & ~PAGE_MASK;
+}
- if (INTEL_INFO(dev)->gen >= 8) {
- page_offset = offset_in_page(page_offset + sizeof(uint32_t));
+#define KMAP 0x4
- if (page_offset == 0) {
- kunmap_atomic(vaddr);
- vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
- (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
- }
+static void reloc_cache_fini(struct reloc_cache *cache)
+{
+ void *vaddr;
- *(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
+ if (!cache->vaddr)
+ return;
+
+ vaddr = unmask_page(cache->vaddr);
+ if (cache->vaddr & KMAP) {
+ if (cache->vaddr & CLFLUSH_AFTER)
+ mb();
+
+ kunmap_atomic(vaddr);
+ i915_gem_object_unpin_pages((struct drm_i915_gem_object *)cache->node.mm);
+ } else {
+ wmb();
+ io_mapping_unmap_atomic(vaddr);
+ if (cache->node.allocated) {
+ struct i915_ggtt *ggtt = &cache->i915->ggtt;
+
+ ggtt->base.clear_range(&ggtt->base,
+ cache->node.start,
+ cache->node.size,
+ true);
+ drm_mm_remove_node(&cache->node);
+ } else
+ i915_vma_unpin((struct i915_vma *)cache->node.mm);
}
+}
- kunmap_atomic(vaddr);
+static void *reloc_kmap(struct drm_i915_gem_object *obj,
+ struct reloc_cache *cache,
+ int page)
+{
+ void *vaddr;
- return 0;
+ if (cache->vaddr) {
+ kunmap_atomic(unmask_page(cache->vaddr));
+ } else {
+ unsigned int flushes;
+ int ret;
+
+ ret = i915_gem_obj_prepare_shmem_write(obj, &flushes);
+ if (ret)
+ return ERR_PTR(ret);
+
+ cache->vaddr = flushes | KMAP;
+ cache->node.mm = (void *)obj;
+ if (flushes)
+ mb();
+ }
+
+ vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page));
+ cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
+ cache->page = page;
+
+ return vaddr;
}
-static int
-relocate_entry_gtt(struct drm_i915_gem_object *obj,
- struct drm_i915_gem_relocation_entry *reloc,
- uint64_t target_offset)
+static void *reloc_iomap(struct drm_i915_gem_object *obj,
+ struct reloc_cache *cache,
+ int page)
{
- struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
- struct i915_ggtt *ggtt = &dev_priv->ggtt;
- struct i915_vma *vma;
- uint64_t delta = relocation_target(reloc, target_offset);
- uint64_t offset;
- void __iomem *reloc_page;
- int ret;
+ struct i915_ggtt *ggtt = &cache->i915->ggtt;
+ unsigned long offset;
+ void *vaddr;
- vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
- if (IS_ERR(vma))
- return PTR_ERR(vma);
+ if (cache->node.allocated) {
+ wmb();
+ ggtt->base.insert_page(&ggtt->base,
+ i915_gem_object_get_dma_address(obj, page),
+ cache->node.start, I915_CACHE_NONE, 0);
+ cache->page = page;
+ return unmask_page(cache->vaddr);
+ }
- ret = i915_gem_object_set_to_gtt_domain(obj, true);
- if (ret)
- goto unpin;
+ if (cache->vaddr) {
+ io_mapping_unmap_atomic(unmask_page(cache->vaddr));
+ } else {
+ struct i915_vma *vma;
+ int ret;
- ret = i915_gem_object_put_fence(obj);
- if (ret)
- goto unpin;
-
- /* Map the page containing the relocation we're going to perform. */
- offset = vma->node.start;
- offset += reloc->offset;
- reloc_page = io_mapping_map_atomic_wc(ggtt->mappable,
- offset & PAGE_MASK);
- iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
-
- if (INTEL_GEN(dev_priv) >= 8) {
- offset += sizeof(uint32_t);
-
- if (offset_in_page(offset) == 0) {
- io_mapping_unmap_atomic(reloc_page);
- reloc_page =
- io_mapping_map_atomic_wc(ggtt->mappable,
- offset);
+ if (use_cpu_reloc(obj))
+ return NULL;
+
+ ret = i915_gem_object_set_to_gtt_domain(obj, true);
+ if (ret)
+ return ERR_PTR(ret);
+
+ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+ PIN_MAPPABLE | PIN_NONBLOCK);
+ if (IS_ERR(vma)) {
+ memset(&cache->node, 0, sizeof(cache->node));
+ ret = drm_mm_insert_node_in_range_generic
+ (&ggtt->base.mm, &cache->node,
+ 4096, 0, 0,
+ 0, ggtt->mappable_end,
+ DRM_MM_SEARCH_DEFAULT,
+ DRM_MM_CREATE_DEFAULT);
+ if (ret)
+ return ERR_PTR(ret);
+ } else {
+ ret = i915_gem_object_put_fence(obj);
+ if (ret) {
+ i915_vma_unpin(vma);
+ return ERR_PTR(ret);
+ }
+
+ cache->node.start = vma->node.start;
+ cache->node.mm = (void *)vma;
}
+ }
- iowrite32(upper_32_bits(delta),
- reloc_page + offset_in_page(offset));
+ offset = cache->node.start;
+ if (cache->node.allocated) {
+ ggtt->base.insert_page(&ggtt->base,
+ i915_gem_object_get_dma_address(obj, page),
+ offset, I915_CACHE_NONE, 0);
+ } else {
+ offset += page << PAGE_SHIFT;
}
- io_mapping_unmap_atomic(reloc_page);
+ vaddr = io_mapping_map_atomic_wc(cache->i915->ggtt.mappable, offset);
+ cache->page = page;
+ cache->vaddr = (unsigned long)vaddr;
-unpin:
- i915_vma_unpin(vma);
- return ret;
+ return vaddr;
}
-static void
-clflush_write32(void *addr, uint32_t value)
+static void *reloc_vaddr(struct drm_i915_gem_object *obj,
+ struct reloc_cache *cache,
+ int page)
{
- /* This is not a fast path, so KISS. */
- drm_clflush_virt_range(addr, sizeof(uint32_t));
- *(uint32_t *)addr = value;
- drm_clflush_virt_range(addr, sizeof(uint32_t));
+ void *vaddr;
+
+ if (cache->page == page) {
+ vaddr = unmask_page(cache->vaddr);
+ } else {
+ vaddr = NULL;
+ if ((cache->vaddr & KMAP) == 0)
+ vaddr = reloc_iomap(obj, cache, page);
+ if (!vaddr)
+ vaddr = reloc_kmap(obj, cache, page);
+ }
+
+ return vaddr;
}
-static int
-relocate_entry_clflush(struct drm_i915_gem_object *obj,
- struct drm_i915_gem_relocation_entry *reloc,
- uint64_t target_offset)
+static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
{
- struct drm_device *dev = obj->base.dev;
- uint32_t page_offset = offset_in_page(reloc->offset);
- uint64_t delta = relocation_target(reloc, target_offset);
- char *vaddr;
- int ret;
+ if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
+ if (flushes & CLFLUSH_BEFORE) {
+ clflushopt(addr);
+ mb();
+ }
- ret = i915_gem_object_set_to_gtt_domain(obj, true);
- if (ret)
- return ret;
+ *addr = value;
- vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
- reloc->offset >> PAGE_SHIFT));
- clflush_write32(vaddr + page_offset, lower_32_bits(delta));
+ /* Writes to the same cacheline are serialised by the CPU
+ * (including clflush). On the write path, we only require
+ * that it hits memory in an orderly fashion and place
+ * mb barriers at the start and end of the relocation phase
+ * to ensure ordering of clflush wrt to the system.
+ */
+ if (flushes & CLFLUSH_AFTER)
+ clflushopt(addr);
+ } else
+ *addr = value;
+}
- if (INTEL_INFO(dev)->gen >= 8) {
- page_offset = offset_in_page(page_offset + sizeof(uint32_t));
+static int
+relocate_entry(struct drm_i915_gem_object *obj,
+ const struct drm_i915_gem_relocation_entry *reloc,
+ struct reloc_cache *cache,
+ u64 target_offset)
+{
+ u64 offset = reloc->offset;
+ bool wide = cache->use_64bit_reloc;
+ void *vaddr;
- if (page_offset == 0) {
- kunmap_atomic(vaddr);
- vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
- (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
- }
+ target_offset = relocation_target(reloc, target_offset);
+repeat:
+ vaddr = reloc_vaddr(obj, cache, offset >> PAGE_SHIFT);
+ if (IS_ERR(vaddr))
+ return PTR_ERR(vaddr);
- clflush_write32(vaddr + page_offset, upper_32_bits(delta));
- }
+ clflush_write32(vaddr + offset_in_page(offset),
+ lower_32_bits(target_offset),
+ cache->vaddr);
- kunmap_atomic(vaddr);
+ if (wide) {
+ offset += sizeof(u32);
+ target_offset >>= 32;
+ wide = false;
+ goto repeat;
+ }
return 0;
}
@@ -459,7 +566,8 @@ static bool object_is_idle(struct drm_i915_gem_object *obj)
static int
i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
struct eb_vmas *eb,
- struct drm_i915_gem_relocation_entry *reloc)
+ struct drm_i915_gem_relocation_entry *reloc,
+ struct reloc_cache *cache)
{
struct drm_device *dev = obj->base.dev;
struct drm_gem_object *target_obj;
@@ -522,7 +630,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
/* Check that the relocation address is valid... */
if (unlikely(reloc->offset >
- obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
+ obj->base.size - (cache->use_64bit_reloc ? 8 : 4))) {
DRM_DEBUG("Relocation beyond object bounds: "
"obj %p target %d offset %d size %d.\n",
obj, reloc->target_handle,
@@ -542,23 +650,12 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
if (pagefault_disabled() && !object_is_idle(obj))
return -EFAULT;
- if (use_cpu_reloc(obj))
- ret = relocate_entry_cpu(obj, reloc, target_offset);
- else if (obj->map_and_fenceable)
- ret = relocate_entry_gtt(obj, reloc, target_offset);
- else if (static_cpu_has(X86_FEATURE_CLFLUSH))
- ret = relocate_entry_clflush(obj, reloc, target_offset);
- else {
- WARN_ONCE(1, "Impossible case in relocation handling\n");
- ret = -ENODEV;
- }
-
+ ret = relocate_entry(obj, reloc, cache, target_offset);
if (ret)
return ret;
/* and update the user's relocation entry */
reloc->presumed_offset = target_offset;
-
return 0;
}
@@ -570,9 +667,11 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
struct drm_i915_gem_relocation_entry __user *user_relocs;
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
- int remain, ret;
+ struct reloc_cache cache;
+ int remain, ret = 0;
user_relocs = u64_to_user_ptr(entry->relocs_ptr);
+ reloc_cache_init(&cache, eb->i915);
remain = entry->relocation_count;
while (remain) {
@@ -582,19 +681,23 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
count = ARRAY_SIZE(stack_reloc);
remain -= count;
- if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
- return -EFAULT;
+ if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]))) {
+ ret = -EFAULT;
+ goto out;
+ }
do {
u64 offset = r->presumed_offset;
- ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
+ ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r, &cache);
if (ret)
- return ret;
+ goto out;
if (r->presumed_offset != offset &&
- __put_user(r->presumed_offset, &user_relocs->presumed_offset)) {
- return -EFAULT;
+ __put_user(r->presumed_offset,
+ &user_relocs->presumed_offset)) {
+ ret = -EFAULT;
+ goto out;
}
user_relocs++;
@@ -602,7 +705,9 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
} while (--count);
}
- return 0;
+out:
+ reloc_cache_fini(&cache);
+ return ret;
#undef N_RELOC
}
@@ -612,15 +717,18 @@ i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
struct drm_i915_gem_relocation_entry *relocs)
{
const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
- int i, ret;
+ struct reloc_cache cache;
+ int i, ret = 0;
+ reloc_cache_init(&cache, eb->i915);
for (i = 0; i < entry->relocation_count; i++) {
- ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
+ ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i], &cache);
if (ret)
- return ret;
+ break;
}
+ reloc_cache_fini(&cache);
- return 0;
+ return ret;
}
static int
@@ -745,7 +853,6 @@ static bool
eb_vma_misplaced(struct i915_vma *vma)
{
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
- struct drm_i915_gem_object *obj = vma->obj;
WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
!i915_vma_is_ggtt(vma));
@@ -766,7 +873,8 @@ eb_vma_misplaced(struct i915_vma *vma)
return true;
/* avoid costly ping-pong once a batch bo ended up non-mappable */
- if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
+ if (entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
+ !i915_vma_is_map_and_fenceable(vma))
return !only_mappable_for_reloc(entry->flags);
if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
@@ -1038,8 +1146,8 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
if (flush_chipset)
i915_gem_chipset_flush(req->engine->i915);
- if (flush_domains & I915_GEM_DOMAIN_GTT)
- wmb();
+ /* Make sure (untracked) CPU relocs/parsing are flushed */
+ wmb();
/* Unconditionally invalidate GPU caches and TLBs. */
return req->engine->emit_flush(req, EMIT_INVALIDATE);
@@ -1600,7 +1708,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
memset(¶ms_master, 0x00, sizeof(params_master));
- eb = eb_create(args);
+ eb = eb_create(dev_priv, args);
if (eb == NULL) {
i915_gem_context_put(ctx);
mutex_unlock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c
index fdd7137a3ebf..57b7834ac8f9 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence.c
@@ -128,7 +128,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
(vma->node.start & (vma->node.size - 1)),
"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08lx) aligned\n",
(long)vma->node.start,
- obj->map_and_fenceable,
+ i915_vma_is_map_and_fenceable(vma),
(long)vma->node.size);
if (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
@@ -388,9 +388,6 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
return 0;
}
} else if (enable) {
- if (WARN_ON(!obj->map_and_fenceable))
- return -EINVAL;
-
reg = i915_find_fence_reg(dev);
if (IS_ERR(reg))
return PTR_ERR(reg);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 4ed7c700ff95..0e0ff9fdceda 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -3694,7 +3694,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
void __iomem *ptr;
lockdep_assert_held(&vma->vm->dev->struct_mutex);
- if (WARN_ON(!vma->obj->map_and_fenceable))
+ if (WARN_ON(!i915_vma_is_map_and_fenceable(vma)))
return IO_ERR_PTR(-ENODEV);
GEM_BUG_ON(!i915_vma_is_ggtt(vma));
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 010e10c0b62b..01bcba9c5cc4 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -201,8 +201,9 @@ struct i915_vma {
#define I915_VMA_LOCAL_BIND BIT(7)
#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
-#define I915_VMA_GGTT BIT(8)
-#define I915_VMA_CLOSED BIT(9)
+#define I915_VMA_GGTT BIT(8)
+#define I915_VMA_CAN_FENCE BIT(9)
+#define I915_VMA_CLOSED BIT(10)
unsigned int active;
struct i915_gem_active last_read[I915_NUM_ENGINES];
@@ -237,6 +238,11 @@ static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
return vma->flags & I915_VMA_GGTT;
}
+static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
+{
+ return vma->flags & I915_VMA_CAN_FENCE;
+}
+
static inline bool i915_vma_is_closed(const struct i915_vma *vma)
{
return vma->flags & I915_VMA_CLOSED;
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 6dabc97d2a81..bd2df61b7b95 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -134,7 +134,7 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
if (!vma)
return 0;
- if (!obj->map_and_fenceable)
+ if (!i915_vma_is_map_and_fenceable(vma))
return 0;
if (IS_GEN3(dev_priv)) {
@@ -145,7 +145,7 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
goto bad;
}
- size = i915_gem_get_ggtt_size(dev_priv, obj->base.size, tiling_mode);
+ size = i915_gem_get_ggtt_size(dev_priv, vma->size, tiling_mode);
if (vma->node.size < size)
goto bad;
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 15129c65d65d..ae362ca8cfea 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -2251,7 +2251,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
* framebuffer compression. For simplicity, we always install
* a fence as the cost is not that onerous.
*/
- if (view.type == I915_GGTT_VIEW_NORMAL) {
+ if (i915_vma_is_map_and_fenceable(vma)) {
ret = i915_gem_object_get_fence(obj);
if (ret == -EDEADLK) {
/*
@@ -2289,11 +2289,11 @@ void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
intel_fill_fb_ggtt_view(&view, fb, rotation);
+ vma = i915_gem_object_to_ggtt(obj, &view);
- if (view.type == I915_GGTT_VIEW_NORMAL)
+ if (i915_vma_is_map_and_fenceable(vma))
i915_gem_object_unpin_fence(obj);
- vma = i915_gem_object_to_ggtt(obj, &view);
i915_gem_object_unpin_from_display_plane(vma);
}
--
2.8.1
More information about the Intel-gfx-trybot
mailing list