[PATCH 5/9] drm/i915: Use SSE4.1 movntdqa to accelerate reads from WC memory
Chris Wilson
chris at chris-wilson.co.uk
Sat Aug 6 14:52:18 UTC 2016
This patch provides the infrastructure for performing a 16-byte aligned
read from WC memory using non-temporal instructions introduced with sse4.1.
Using movntdqa we can bypass the CPU caches and read directly from memory
and ignoring the page attributes set on the CPU PTE i.e. negating the
impact of an otherwise UC access. Copying using movntqda from WC is almost
as fast as reading from WB memory, modulo the possibility of both hitting
the CPU cache or leaving the data in the CPU cache for the next consumer.
(The CPU cache itself my be flushed for the region of the movntdqa and on
later access the movntdqa reads from a separate internal buffer for the
cacheline.) The write back to the memory is however cached.
This will be used in later patches to accelerate accessing WC memory.
v2: Report whether the accelerated copy is successful/possible.
v3: Function alignment override was only necessary when using the
function target("sse4.1") - which is not necessary for emitting movntdqa
from __asm__.
v4: Improve notes on CPU cache behaviour vs non-temporal stores.
v5: Fix byte offsets for unrolled moves.
v6: Find all remaining typos of movntqda, use kernel_fpu_begin.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Akash Goel <akash.goel at intel.com>
Cc: Damien Lespiau <damien.lespiau at intel.com>
Cc: Mika Kuoppala <mika.kuoppala at intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin at intel.com>
---
drivers/gpu/drm/Makefile | 2 +-
drivers/gpu/drm/drm_mm.c | 140 ++++++++++++++++++++++-----
drivers/gpu/drm/i915/Makefile | 8 +-
drivers/gpu/drm/i915/i915_drv.c | 2 +
drivers/gpu/drm/i915/i915_drv.h | 35 +++++++
drivers/gpu/drm/i915/i915_gem.c | 75 ++++++--------
drivers/gpu/drm/i915/i915_gem_batch_pool.c | 2 +-
drivers/gpu/drm/i915/i915_gem_context.c | 2 +-
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 +--
drivers/gpu/drm/i915/i915_gem_gtt.c | 99 ++++++++-----------
drivers/gpu/drm/i915/i915_gem_gtt.h | 8 +-
drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +-
drivers/gpu/drm/i915/i915_gem_request.c | 13 ++-
drivers/gpu/drm/i915/i915_gpu_error.c | 6 +-
drivers/gpu/drm/i915/i915_memcpy.c | 101 +++++++++++++++++++
drivers/gpu/drm/i915/i915_memory.c | 128 ++++++++++++++++++++++++
drivers/gpu/drm/i915/intel_overlay.c | 4 +-
drivers/gpu/drm/i915/intel_ringbuffer.c | 2 +-
include/drm/drm_mm.h | 12 +++
include/linux/io-mapping.h | 87 ++++++++++-------
20 files changed, 554 insertions(+), 185 deletions(-)
create mode 100644 drivers/gpu/drm/i915/i915_memcpy.c
create mode 100644 drivers/gpu/drm/i915/i915_memory.c
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index e3dba6f44a79..a79e25e4d547 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -46,7 +46,7 @@ obj-$(CONFIG_DRM_RADEON)+= radeon/
obj-$(CONFIG_DRM_AMDGPU)+= amd/amdgpu/
obj-$(CONFIG_DRM_MGA) += mga/
obj-$(CONFIG_DRM_I810) += i810/
-obj-$(CONFIG_DRM_I915) += i915/
+obj-y += i915/
obj-$(CONFIG_DRM_MGAG200) += mgag200/
obj-$(CONFIG_DRM_VC4) += vc4/
obj-$(CONFIG_DRM_CIRRUS_QEMU) += cirrus/
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index cb39f45d6a16..57cb299c8594 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -46,6 +46,7 @@
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/export.h>
+#include <linux/interval_tree_generic.h>
/**
* DOC: Overview
@@ -103,6 +104,72 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
u64 end,
enum drm_mm_search_flags flags);
+#define START(node) ((node)->start)
+#define LAST(node) ((node)->start + (node)->size - 1)
+
+INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
+ u64, __subtree_last,
+ START, LAST, static inline, drm_mm_interval_tree)
+
+struct drm_mm_node *
+drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last)
+{
+ return drm_mm_interval_tree_iter_first(&mm->interval_tree,
+ start, last);
+}
+EXPORT_SYMBOL(drm_mm_interval_first);
+
+struct drm_mm_node *
+drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last)
+{
+ return drm_mm_interval_tree_iter_next(node, start, last);
+}
+EXPORT_SYMBOL(drm_mm_interval_next);
+
+static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
+ struct drm_mm_node *node)
+{
+ struct drm_mm *mm = hole_node->mm;
+ struct rb_node **link, *rb;
+ struct drm_mm_node *parent;
+
+ node->__subtree_last = LAST(node);
+
+ if (hole_node->allocated) {
+ rb = &hole_node->rb;
+ while (rb) {
+ parent = rb_entry(rb, struct drm_mm_node, rb);
+ if (parent->__subtree_last >= node->__subtree_last)
+ break;
+
+ parent->__subtree_last = node->__subtree_last;
+ rb = rb_parent(rb);
+ }
+
+ rb = &hole_node->rb;
+ link = &hole_node->rb.rb_right;
+ } else {
+ rb = NULL;
+ link = &mm->interval_tree.rb_node;
+ }
+
+ while (*link) {
+ rb = *link;
+ parent = rb_entry(rb, struct drm_mm_node, rb);
+ if (parent->__subtree_last < node->__subtree_last)
+ parent->__subtree_last = node->__subtree_last;
+ if (node->start < parent->start)
+ link = &parent->rb.rb_left;
+ else
+ link = &parent->rb.rb_right;
+ }
+
+ rb_link_node(&node->rb, rb, link);
+ rb_insert_augmented(&node->rb,
+ &mm->interval_tree,
+ &drm_mm_interval_tree_augment);
+}
+
static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
struct drm_mm_node *node,
u64 size, unsigned alignment,
@@ -153,6 +220,8 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
INIT_LIST_HEAD(&node->hole_stack);
list_add(&node->node_list, &hole_node->node_list);
+ drm_mm_interval_tree_add_node(hole_node, node);
+
BUG_ON(node->start + node->size > adj_end);
node->hole_follows = 0;
@@ -178,41 +247,55 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
*/
int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
{
+ u64 end = node->start + node->size;
struct drm_mm_node *hole;
- u64 end;
- u64 hole_start;
- u64 hole_end;
+ u64 hole_start, hole_end;
- BUG_ON(node == NULL);
+ if (WARN_ON(node->size == 0))
+ return -EINVAL;
end = node->start + node->size;
/* Find the relevant hole to add our node to */
- drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
- if (hole_start > node->start || hole_end < end)
- continue;
+ hole = drm_mm_interval_tree_iter_first(&mm->interval_tree,
+ node->start, ~(u64)0);
+ if (hole) {
+ if (hole->start < end)
+ return -ENOSPC;
+ } else {
+ hole = list_entry(&mm->head_node.node_list,
+ typeof(*hole), node_list);
+ }
- node->mm = mm;
- node->allocated = 1;
+ hole = list_last_entry(&hole->node_list, typeof(*hole), node_list);
+ if (!hole->hole_follows)
+ return -ENOSPC;
- INIT_LIST_HEAD(&node->hole_stack);
- list_add(&node->node_list, &hole->node_list);
+ hole_start = __drm_mm_hole_node_start(hole);
+ hole_end = __drm_mm_hole_node_end(hole);
+ if (hole_start > node->start || hole_end < end)
+ return -ENOSPC;
- if (node->start == hole_start) {
- hole->hole_follows = 0;
- list_del_init(&hole->hole_stack);
- }
+ node->mm = mm;
+ node->allocated = 1;
- node->hole_follows = 0;
- if (end != hole_end) {
- list_add(&node->hole_stack, &mm->hole_stack);
- node->hole_follows = 1;
- }
+ INIT_LIST_HEAD(&node->hole_stack);
+ list_add(&node->node_list, &hole->node_list);
- return 0;
+ drm_mm_interval_tree_add_node(hole, node);
+
+ if (node->start == hole_start) {
+ hole->hole_follows = 0;
+ list_del_init(&hole->hole_stack);
+ }
+
+ node->hole_follows = 0;
+ if (end != hole_end) {
+ list_add(&node->hole_stack, &mm->hole_stack);
+ node->hole_follows = 1;
}
- return -ENOSPC;
+ return 0;
}
EXPORT_SYMBOL(drm_mm_reserve_node);
@@ -239,6 +322,9 @@ int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
{
struct drm_mm_node *hole_node;
+ if (WARN_ON(size == 0))
+ return -EINVAL;
+
hole_node = drm_mm_search_free_generic(mm, size, alignment,
color, sflags);
if (!hole_node)
@@ -302,6 +388,8 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
INIT_LIST_HEAD(&node->hole_stack);
list_add(&node->node_list, &hole_node->node_list);
+ drm_mm_interval_tree_add_node(hole_node, node);
+
BUG_ON(node->start < start);
BUG_ON(node->start < adj_start);
BUG_ON(node->start + node->size > adj_end);
@@ -340,6 +428,9 @@ int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, struct drm_mm_node *n
{
struct drm_mm_node *hole_node;
+ if (WARN_ON(size == 0))
+ return -EINVAL;
+
hole_node = drm_mm_search_free_in_range_generic(mm,
size, alignment, color,
start, end, sflags);
@@ -390,6 +481,7 @@ void drm_mm_remove_node(struct drm_mm_node *node)
} else
list_move(&prev_node->hole_stack, &mm->hole_stack);
+ drm_mm_interval_tree_remove(node, &mm->interval_tree);
list_del(&node->node_list);
node->allocated = 0;
}
@@ -516,11 +608,13 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
{
list_replace(&old->node_list, &new->node_list);
list_replace(&old->hole_stack, &new->hole_stack);
+ rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree);
new->hole_follows = old->hole_follows;
new->mm = old->mm;
new->start = old->start;
new->size = old->size;
new->color = old->color;
+ new->__subtree_last = old->__subtree_last;
old->allocated = 0;
new->allocated = 1;
@@ -758,6 +852,8 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size)
mm->head_node.size = start - mm->head_node.start;
list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
+ mm->interval_tree = RB_ROOT;
+
mm->color_adjust = NULL;
}
EXPORT_SYMBOL(drm_mm_init);
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index dda724f04445..bf4508132887 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -3,12 +3,15 @@
# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror
+subdir-ccflags-y += \
+ $(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA)
# Please keep these build lists sorted!
# core driver code
i915-y := i915_drv.o \
i915_irq.o \
+ i915_memcpy.o \
i915_params.o \
i915_pci.o \
i915_suspend.o \
@@ -110,6 +113,9 @@ i915-y += intel_gvt.o
include $(src)/gvt/Makefile
endif
-obj-$(CONFIG_DRM_I915) += i915.o
+obj-$(CONFIG_DRM_I915) += i915.o
+ifdef CONFIG_DRM_I915
+obj-y += i915_memory.o
+endif
CFLAGS_i915_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 8cfc264ec9f6..c48d564b110f 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -827,6 +827,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
mutex_init(&dev_priv->wm.wm_mutex);
mutex_init(&dev_priv->pps_mutex);
+ i915_memcpy_init_early(dev_priv);
+
ret = i915_workqueues_init(dev_priv);
if (ret < 0)
return ret;
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index d0734fc8c560..ebefeea93bc9 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -570,6 +570,7 @@ struct drm_i915_error_state {
long jiffies;
pid_t pid;
u32 seqno;
+ u32 head;
u32 tail;
} *requests;
@@ -2177,6 +2178,13 @@ struct drm_i915_gem_object {
((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
/**
+ * Have we taken a reference for the object for incomplete GPU
+ * activity?
+ */
+#define I915_BO_ACTIVE_REF_SHIFT (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)
+#define I915_BO_ACTIVE_REF_BIT BIT(I915_BO_ACTIVE_REF_SHIFT)
+
+ /**
* This is set if the object has been written to since last bound
* to the GTT
*/
@@ -2346,6 +2354,26 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
}
+static inline bool
+i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
+{
+ return obj->flags & I915_BO_ACTIVE_REF_BIT;
+}
+
+static inline void
+i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
+{
+ obj->flags |= I915_BO_ACTIVE_REF_BIT;
+}
+
+static inline void
+i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
+{
+ obj->flags &= ~I915_BO_ACTIVE_REF_BIT;
+}
+
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
+
static inline unsigned int
i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
{
@@ -3904,4 +3932,11 @@ static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
return false;
}
+void i915_memcpy_init_early(struct drm_i915_private *dev_priv);
+bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len);
+
+int remap_io_mapping(struct vm_area_struct *vma,
+ unsigned long addr, unsigned long pfn, unsigned long size,
+ struct io_mapping *iomap);
+
#endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ce8e67009c35..740ba6971a32 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -879,7 +879,7 @@ i915_gem_gtt_pread(struct drm_device *dev,
* and write to user memory which may result into page
* faults, and so we cannot perform this under struct_mutex.
*/
- if (slow_user_access(ggtt->mappable, page_base,
+ if (slow_user_access(&ggtt->mappable, page_base,
page_offset, user_data,
page_length, false)) {
ret = -EFAULT;
@@ -1168,11 +1168,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
* If the object is non-shmem backed, we retry again with the
* path that handles page fault.
*/
- if (fast_user_write(ggtt->mappable, page_base,
+ if (fast_user_write(&ggtt->mappable, page_base,
page_offset, user_data, page_length)) {
hit_slow_path = true;
mutex_unlock(&dev->struct_mutex);
- if (slow_user_access(ggtt->mappable,
+ if (slow_user_access(&ggtt->mappable,
page_base,
page_offset, user_data,
page_length, true)) {
@@ -1684,7 +1684,6 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
struct i915_vma *ggtt;
pgoff_t page_offset;
- unsigned long pfn;
unsigned int flags;
int ret;
@@ -1754,44 +1753,13 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
goto err_unpin;
/* Finally, remap it using the new GTT offset */
- pfn = dev_priv->ggtt.mappable_base + ggtt->node.start;
- pfn >>= PAGE_SHIFT;
-
- if (ggtt->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
- if (!obj->fault_mappable) {
- unsigned long size = min_t(unsigned long,
- vma->vm_end - vma->vm_start,
- obj->base.size);
- int i;
-
- for (i = 0; i < size >> PAGE_SHIFT; i++) {
- ret = vm_insert_pfn(vma,
- (unsigned long)vma->vm_start + i * PAGE_SIZE,
- pfn + i);
- if (ret)
- break;
- }
- } else
- ret = vm_insert_pfn(vma,
- (unsigned long)vmf->virtual_address,
- pfn + page_offset);
- } else {
- /* Overriding existing pages in partial view does not cause
- * us any trouble as TLBs are still valid because the fault
- * is due to userspace losing part of the mapping or never
- * having accessed it before (at this partials' range).
- */
- const struct i915_ggtt_view *view = &ggtt->ggtt_view;
- unsigned long base = vma->vm_start +
- (view->params.partial.offset << PAGE_SHIFT);
- unsigned int i;
-
- for (i = 0; i < view->params.partial.size; i++) {
- ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
- if (ret)
- break;
- }
- }
+ ret = remap_io_mapping(vma,
+ vma->vm_start + (ggtt->ggtt_view.params.partial.offset << PAGE_SHIFT),
+ (dev_priv->ggtt.mappable_base + ggtt->node.start) >> PAGE_SHIFT,
+ min_t(u64, ggtt->size, vma->vm_end - vma->vm_start),
+ &dev_priv->ggtt.mappable);
+ if (ret)
+ goto err_unpin;
obj->fault_mappable = true;
err_unpin:
@@ -2399,7 +2367,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,
list_move_tail(&obj->global_list,
&request->i915->mm.bound_list);
- i915_gem_object_put(obj);
+ if (i915_gem_object_has_active_reference(obj)) {
+ i915_gem_object_clear_active_reference(obj);
+ i915_gem_object_put(obj);
+ }
}
static bool i915_context_is_banned(const struct i915_gem_context *ctx)
@@ -2632,6 +2603,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
if (vma->vm->file == fpriv)
i915_vma_close(vma);
+
+ if (i915_gem_object_is_active(obj) &&
+ !i915_gem_object_has_active_reference(obj)) {
+ i915_gem_object_set_active_reference(obj);
+ i915_gem_object_get(obj);
+ }
mutex_unlock(&obj->base.dev->struct_mutex);
}
@@ -4141,6 +4118,18 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
intel_runtime_pm_put(dev_priv);
}
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
+{
+ if (!obj)
+ return;
+
+ GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
+ if (i915_gem_object_is_active(obj))
+ i915_gem_object_set_active_reference(obj);
+ else
+ i915_gem_object_put(obj);
+}
+
int i915_gem_suspend(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index ed989596d9a3..cb25cad3318c 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
list_for_each_entry_safe(obj, next,
&pool->cache_list[n],
batch_pool_link)
- i915_gem_object_put(obj);
+ __i915_gem_object_release_unless_active(obj);
INIT_LIST_HEAD(&pool->cache_list[n]);
}
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 408a818c61f2..25904e524fdc 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
if (ce->ring)
intel_ring_free(ce->ring);
- i915_gem_object_put(ce->state);
+ __i915_gem_object_release_unless_active(ce->state);
}
put_pid(ctx->pid);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 896d88c8a9fa..ddd4e316ccc8 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -164,7 +164,6 @@ eb_lookup_vmas(struct eb_vmas *eb,
goto err;
}
- i915_gem_object_get(obj);
list_add_tail(&obj->obj_exec_link, &objects);
}
spin_unlock(&file->table_lock);
@@ -275,7 +274,6 @@ static void eb_destroy(struct eb_vmas *eb)
exec_list);
list_del_init(&vma->exec_list);
i915_gem_execbuffer_unreserve_vma(vma);
- i915_gem_object_put(vma->obj);
}
kfree(eb);
}
@@ -470,7 +468,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
offset += page << PAGE_SHIFT;
}
- vaddr = io_mapping_map_atomic_wc(cache->i915->ggtt.mappable, offset);
+ vaddr = io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
cache->page = page;
cache->vaddr = (unsigned long)vaddr;
@@ -1013,7 +1011,6 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
list_del_init(&vma->exec_list);
i915_gem_execbuffer_unreserve_vma(vma);
- i915_gem_object_put(vma->obj);
}
mutex_unlock(&dev->struct_mutex);
@@ -1286,15 +1283,12 @@ void i915_vma_move_to_active(struct i915_vma *vma,
obj->dirty = 1; /* be paranoid */
- /* Add a reference if we're newly entering the active list.
- * The order in which we add operations to the retirement queue is
+ /* The order in which we add operations to the retirement queue is
* vital here: mark_active adds to the start of the callback list,
* such that subsequent callbacks are called first. Therefore we
* add the active reference first and queue for it to be dropped
* *last*.
*/
- if (!i915_gem_object_is_active(obj))
- i915_gem_object_get(obj);
i915_gem_object_set_active(obj, idx);
i915_gem_active_set(&obj->last_read[idx], req);
@@ -1429,7 +1423,6 @@ i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
vma->exec_entry = shadow_exec_entry;
vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
- i915_gem_object_get(shadow_batch_obj);
list_add_tail(&vma->exec_list, &eb->vmas);
err:
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index bcd21af1471e..655962e74614 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -32,6 +32,8 @@
#include "i915_trace.h"
#include "intel_drv.h"
+#define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
+
/**
* DOC: Global GTT views
*
@@ -343,7 +345,7 @@ static int __setup_page_dma(struct drm_device *dev,
static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
{
- return __setup_page_dma(dev, p, GFP_KERNEL);
+ return __setup_page_dma(dev, p, I915_GFP_DMA);
}
static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
@@ -405,33 +407,18 @@ static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
fill_page_dma(dev, p, v);
}
-static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev)
+static int
+setup_scratch_page(struct drm_device *dev,
+ struct i915_page_dma *scratch,
+ gfp_t gfp)
{
- struct i915_page_scratch *sp;
- int ret;
-
- sp = kzalloc(sizeof(*sp), GFP_KERNEL);
- if (sp == NULL)
- return ERR_PTR(-ENOMEM);
-
- ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO);
- if (ret) {
- kfree(sp);
- return ERR_PTR(ret);
- }
-
- set_pages_uc(px_page(sp), 1);
-
- return sp;
+ return __setup_page_dma(dev, scratch, gfp | __GFP_ZERO);
}
-static void free_scratch_page(struct drm_device *dev,
- struct i915_page_scratch *sp)
+static void cleanup_scratch_page(struct drm_device *dev,
+ struct i915_page_dma *scratch)
{
- set_pages_wb(px_page(sp), 1);
-
- cleanup_px(dev, sp);
- kfree(sp);
+ cleanup_page_dma(dev, scratch);
}
static struct i915_page_table *alloc_pt(struct drm_device *dev)
@@ -477,7 +464,7 @@ static void gen8_initialize_pt(struct i915_address_space *vm,
{
gen8_pte_t scratch_pte;
- scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+ scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, true);
fill_px(vm->dev, pt, scratch_pte);
@@ -488,9 +475,9 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
{
gen6_pte_t scratch_pte;
- WARN_ON(px_dma(vm->scratch_page) == 0);
+ WARN_ON(vm->scratch_page.daddr == 0);
- scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+ scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, true, 0);
fill32_px(vm->dev, pt, scratch_pte);
@@ -774,7 +761,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
bool use_scratch)
{
struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
- gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+ gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, use_scratch);
if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
@@ -880,9 +867,9 @@ static int gen8_init_scratch(struct i915_address_space *vm)
struct drm_device *dev = vm->dev;
int ret;
- vm->scratch_page = alloc_scratch_page(dev);
- if (IS_ERR(vm->scratch_page))
- return PTR_ERR(vm->scratch_page);
+ ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA);
+ if (ret)
+ return ret;
vm->scratch_pt = alloc_pt(dev);
if (IS_ERR(vm->scratch_pt)) {
@@ -916,7 +903,7 @@ free_pd:
free_pt:
free_pt(dev, vm->scratch_pt);
free_scratch_page:
- free_scratch_page(dev, vm->scratch_page);
+ cleanup_scratch_page(dev, &vm->scratch_page);
return ret;
}
@@ -960,7 +947,7 @@ static void gen8_free_scratch(struct i915_address_space *vm)
free_pdp(dev, vm->scratch_pdp);
free_pd(dev, vm->scratch_pd);
free_pt(dev, vm->scratch_pt);
- free_scratch_page(dev, vm->scratch_page);
+ cleanup_scratch_page(dev, &vm->scratch_page);
}
static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev,
@@ -1457,7 +1444,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
struct i915_address_space *vm = &ppgtt->base;
uint64_t start = ppgtt->base.start;
uint64_t length = ppgtt->base.total;
- gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+ gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, true);
if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
@@ -1574,7 +1561,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
uint32_t pte, pde;
uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
- scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+ scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, true, 0);
gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
@@ -1799,7 +1786,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
unsigned first_pte = first_entry % GEN6_PTES;
unsigned last_pte, i;
- scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+ scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, true, 0);
while (num_entries) {
@@ -1945,14 +1932,15 @@ unwind_out:
static int gen6_init_scratch(struct i915_address_space *vm)
{
struct drm_device *dev = vm->dev;
+ int ret;
- vm->scratch_page = alloc_scratch_page(dev);
- if (IS_ERR(vm->scratch_page))
- return PTR_ERR(vm->scratch_page);
+ ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA);
+ if (ret)
+ return ret;
vm->scratch_pt = alloc_pt(dev);
if (IS_ERR(vm->scratch_pt)) {
- free_scratch_page(dev, vm->scratch_page);
+ cleanup_scratch_page(dev, &vm->scratch_page);
return PTR_ERR(vm->scratch_pt);
}
@@ -1966,7 +1954,7 @@ static void gen6_free_scratch(struct i915_address_space *vm)
struct drm_device *dev = vm->dev;
free_pt(dev, vm->scratch_pt);
- free_scratch_page(dev, vm->scratch_page);
+ cleanup_scratch_page(dev, &vm->scratch_page);
}
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
@@ -2532,7 +2520,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
first_entry, num_entries, max_entries))
num_entries = max_entries;
- scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+ scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC,
use_scratch);
for (i = 0; i < num_entries; i++)
@@ -2564,7 +2552,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
first_entry, num_entries, max_entries))
num_entries = max_entries;
- scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+ scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC, use_scratch, 0);
for (i = 0; i < num_entries; i++)
@@ -2829,7 +2817,6 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
if (dev_priv->mm.aliasing_ppgtt) {
struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-
ppgtt->base.cleanup(&ppgtt->base);
kfree(ppgtt);
}
@@ -2847,7 +2834,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
ggtt->base.cleanup(&ggtt->base);
arch_phys_wc_del(ggtt->mtrr);
- io_mapping_free(ggtt->mappable);
+ io_mapping_fini(&ggtt->mappable);
}
static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
@@ -2931,8 +2918,8 @@ static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
{
struct pci_dev *pdev = ggtt->base.dev->pdev;
- struct i915_page_scratch *scratch_page;
phys_addr_t phys_addr;
+ int ret;
/* For Modern GENs the PTEs and register space are split in the BAR */
phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
@@ -2953,16 +2940,16 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
return -ENOMEM;
}
- scratch_page = alloc_scratch_page(ggtt->base.dev);
- if (IS_ERR(scratch_page)) {
+ ret = setup_scratch_page(ggtt->base.dev,
+ &ggtt->base.scratch_page,
+ GFP_DMA32);
+ if (ret) {
DRM_ERROR("Scratch setup failed\n");
/* iounmap will also get called at remove, but meh */
iounmap(ggtt->gsm);
- return PTR_ERR(scratch_page);
+ return ret;
}
- ggtt->base.scratch_page = scratch_page;
-
return 0;
}
@@ -3044,7 +3031,7 @@ static void gen6_gmch_remove(struct i915_address_space *vm)
struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
iounmap(ggtt->gsm);
- free_scratch_page(vm->dev, vm->scratch_page);
+ cleanup_scratch_page(vm->dev, &vm->scratch_page);
}
static int gen8_gmch_probe(struct i915_ggtt *ggtt)
@@ -3245,9 +3232,9 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
if (!HAS_LLC(dev_priv))
ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
- ggtt->mappable =
- io_mapping_create_wc(ggtt->mappable_base, ggtt->mappable_end);
- if (!ggtt->mappable) {
+ if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
+ dev_priv->ggtt.mappable_base,
+ dev_priv->ggtt.mappable_end)) {
ret = -EIO;
goto out_gtt_cleanup;
}
@@ -3729,7 +3716,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
ptr = vma->iomap;
if (ptr == NULL) {
- ptr = io_mapping_map_wc(i915_vm_to_ggtt(vma->vm)->mappable,
+ ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
vma->node.start,
vma->node.size);
if (ptr == NULL)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index cae8b67b99c4..31132aca16bd 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -300,10 +300,6 @@ struct i915_page_dma {
#define px_page(px) (px_base(px)->page)
#define px_dma(px) (px_base(px)->daddr)
-struct i915_page_scratch {
- struct i915_page_dma base;
-};
-
struct i915_page_table {
struct i915_page_dma base;
@@ -349,7 +345,7 @@ struct i915_address_space {
bool closed;
- struct i915_page_scratch *scratch_page;
+ struct i915_page_dma scratch_page;
struct i915_page_table *scratch_pt;
struct i915_page_directory *scratch_pd;
struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
@@ -427,13 +423,13 @@ struct i915_address_space {
*/
struct i915_ggtt {
struct i915_address_space base;
+ struct io_mapping mappable; /* Mapping to our CPU mappable region */
size_t stolen_size; /* Total size of stolen memory */
size_t stolen_usable_size; /* Total size minus BIOS reserved */
size_t stolen_reserved_base;
size_t stolen_reserved_size;
u64 mappable_end; /* End offset that we can CPU map */
- struct io_mapping *mappable; /* Mapping to our CPU mappable region */
phys_addr_t mappable_base; /* PA of our GMADR */
/** "Graphics Stolen Memory" holds the global PTEs */
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index f0eb218541f8..adcd02abeabb 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -221,6 +221,6 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
err_unpin:
i915_vma_unpin(so.vma);
err_obj:
- i915_gem_object_put(so.obj);
+ __i915_gem_object_release_unless_active(so.obj);
return ret;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 8fdd313248f9..afe28112e8f6 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -421,6 +421,13 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
if (ret)
goto err_ctx;
+ /* Record the position of the start of the request so that
+ * should we detect the updated seqno part-way through the
+ * GPU processing the request, we never over-estimate the
+ * position of the head.
+ */
+ req->head = req->ring->tail;
+
return req;
err_ctx:
@@ -489,8 +496,6 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
trace_i915_gem_request_add(request);
- request->head = request_start;
-
/* Seal the request and mark it as pending execution. Note that
* we may inspect this state, without holding any locks, during
* hangcheck. Hence we apply the barrier to ensure that we do not
@@ -503,10 +508,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
list_add_tail(&request->link, &engine->request_list);
list_add_tail(&request->ring_link, &ring->request_list);
- /* Record the position of the start of the request so that
+ /* Record the position of the start of the breadcrumb so that
* should we detect the updated seqno part-way through the
* GPU processing the request, we never over-estimate the
- * position of the head.
+ * position of the ring's HEAD.
*/
request->postfix = ring->tail;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index e9631a582878..9135d64f985c 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -495,10 +495,11 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
dev_priv->engine[i].name,
ee->num_requests);
for (j = 0; j < ee->num_requests; j++) {
- err_printf(m, " pid %d, seqno 0x%08x, emitted %ld, tail 0x%08x\n",
+ err_printf(m, " pid %d, seqno 0x%08x, emitted %ld, head 0x%08x tail 0x%08x\n",
ee->requests[j].pid,
ee->requests[j].seqno,
ee->requests[j].jiffies,
+ ee->requests[j].head,
ee->requests[j].tail);
}
}
@@ -702,7 +703,7 @@ i915_error_object_create(struct drm_i915_private *i915,
I915_CACHE_NONE, 0);
s = (void *__force)
- io_mapping_map_atomic_wc(ggtt->mappable, slot);
+ io_mapping_map_atomic_wc(&ggtt->mappable, slot);
ret = compress_page(&zstream, s, dst);
io_mapping_unmap_atomic(s);
@@ -1073,6 +1074,7 @@ static void engine_record_requests(struct intel_engine_cs *engine,
erq = &ee->requests[count++];
erq->seqno = request->fence.seqno;
erq->jiffies = request->emitted_jiffies;
+ erq->head = request->head;
erq->tail = request->tail;
rcu_read_lock();
diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c
new file mode 100644
index 000000000000..50fc5799255f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_memcpy.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <asm/fpu/api.h>
+
+#include "i915_drv.h"
+
+DEFINE_STATIC_KEY_FALSE(has_movntdqa);
+
+#ifdef CONFIG_AS_MOVNTDQA
+static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
+{
+ kernel_fpu_begin();
+
+ len >>= 4;
+ while (len >= 4) {
+ asm("movntdqa (%0), %%xmm0\n"
+ "movntdqa 16(%0), %%xmm1\n"
+ "movntdqa 32(%0), %%xmm2\n"
+ "movntdqa 48(%0), %%xmm3\n"
+ "movaps %%xmm0, (%1)\n"
+ "movaps %%xmm1, 16(%1)\n"
+ "movaps %%xmm2, 32(%1)\n"
+ "movaps %%xmm3, 48(%1)\n"
+ :: "r" (src), "r" (dst) : "memory");
+ src += 64;
+ dst += 64;
+ len -= 4;
+ }
+ while (len--) {
+ asm("movntdqa (%0), %%xmm0\n"
+ "movaps %%xmm0, (%1)\n"
+ :: "r" (src), "r" (dst) : "memory");
+ src += 16;
+ dst += 16;
+ }
+
+ kernel_fpu_end();
+}
+#endif
+
+/**
+ * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
+ * @dst: destination pointer
+ * @src: source pointer
+ * @len: how many bytes to copy
+ *
+ * i915_memcpy_from_wc copies @len bytes from @src to @dst using
+ * non-temporal instructions where available. Note that all arguments
+ * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
+ * of 16.
+ *
+ * To test whether accelerated reads from WC are supported, use
+ * i915_memcpy_from_wc(NULL, NULL, 0);
+ *
+ * Returns true if the copy was successful, false if the preconditions
+ * are not met.
+ */
+bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
+{
+ if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
+ return false;
+
+#ifdef CONFIG_AS_MOVNTDQA
+ if (static_branch_likely(&has_movntdqa)) {
+ if (len)
+ __memcpy_ntdqa(dst, src, len);
+ return true;
+ }
+#endif
+
+ return false;
+}
+
+void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
+{
+ if (static_cpu_has(X86_FEATURE_XMM4_1))
+ static_branch_enable(&has_movntdqa);
+}
diff --git a/drivers/gpu/drm/i915/i915_memory.c b/drivers/gpu/drm/i915/i915_memory.c
new file mode 100644
index 000000000000..fe7465f70e28
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_memory.c
@@ -0,0 +1,128 @@
+#include <linux/mm.h>
+#include <linux/io-mapping.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+
+#include <asm/pgalloc.h>
+#include <asm/tlb.h>
+#include <asm/tlbflush.h>
+#include <asm/pgtable.h>
+
+#include "i915_drv.h"
+
+struct remap_pfn {
+ struct mm_struct *mm;
+ unsigned long addr;
+ unsigned long pfn;
+ pgprot_t prot;
+};
+
+static inline void remap_pfn(struct remap_pfn *r, pte_t *pte)
+{
+ set_pte_at(r->mm, r->addr, pte,
+ pte_mkspecial(pfn_pte(r->pfn, r->prot)));
+ r->pfn++;
+ r->addr += PAGE_SIZE;
+}
+
+static inline void remap_pte_range(struct remap_pfn *r,
+ pmd_t *pmd,
+ unsigned long end)
+{
+ pte_t *pte;
+ spinlock_t *ptl;
+
+ pte = pte_alloc_map_lock(r->mm, pmd, r->addr, &ptl);
+ if (unlikely(!pte)) {
+ r->addr = ~0UL;
+ return;
+ }
+
+ arch_enter_lazy_mmu_mode();
+ do
+ remap_pfn(r, pte++);
+ while (r->addr < end);
+ arch_leave_lazy_mmu_mode();
+
+ pte_unmap_unlock(pte - 1, ptl);
+}
+
+static inline void remap_pmd_range(struct remap_pfn *r,
+ pud_t *pud,
+ unsigned long end)
+{
+ pmd_t *pmd;
+
+ pmd = pmd_alloc(r->mm, pud, r->addr);
+ if (unlikely(!pmd)) {
+ r->addr = ~0UL;
+ return;
+ }
+
+ do
+ remap_pte_range(r, pmd++, pmd_addr_end(r->addr, end));
+ while (r->addr < end);
+}
+
+static inline void remap_pud_range(struct remap_pfn *r,
+ pgd_t *pgd,
+ unsigned long end)
+{
+ pud_t *pud;
+
+ pud = pud_alloc(r->mm, pgd, r->addr);
+ if (unlikely(!pud)) {
+ r->addr = ~0UL;
+ return;
+ }
+
+ do
+ remap_pmd_range(r, pud++, pud_addr_end(r->addr, end));
+ while (r->addr < end);
+}
+
+/**
+ * remap_io_mapping - remap an IO mapping to userspace
+ * @vma: user vma to map to
+ * @addr: target user address to start at
+ * @pfn: physical address of kernel memory
+ * @size: size of map area
+ * @iomap: the source io_mapping
+ *
+ * Note: this is only safe if the mm semaphore is held when called.
+ */
+int remap_io_mapping(struct vm_area_struct *vma,
+ unsigned long addr, unsigned long pfn, unsigned long size,
+ struct io_mapping *iomap)
+{
+ unsigned long end = addr + PAGE_ALIGN(size);
+ struct remap_pfn r;
+ pgd_t *pgd;
+
+ if (WARN_ON(addr >= end))
+ return -EINVAL;
+
+#define MUST_SET (VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP)
+ if (WARN_ON((vma->vm_flags & MUST_SET) != MUST_SET))
+ return -EINVAL;
+#undef MUST_SET
+
+ r.mm = vma->vm_mm;
+ r.addr = addr;
+ r.pfn = pfn;
+ r.prot = __pgprot((pgprot_val(iomap->prot) & _PAGE_CACHE_MASK) |
+ (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK));
+
+ pgd = pgd_offset(r.mm, addr);
+ do
+ remap_pud_range(&r, pgd++, pgd_addr_end(r.addr, end));
+ while (r.addr < end);
+
+ if (unlikely(r.addr > end)) {
+ zap_vma_ptes(vma, addr, end - addr);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(remap_io_mapping);
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index cd6cc5c52eaf..736abca1d945 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -195,7 +195,7 @@ intel_overlay_map_regs(struct intel_overlay *overlay)
if (OVERLAY_NEEDS_PHYSICAL(dev_priv))
regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_handle->vaddr;
else
- regs = io_mapping_map_wc(dev_priv->ggtt.mappable,
+ regs = io_mapping_map_wc(&dev_priv->ggtt.mappable,
overlay->flip_addr,
PAGE_SIZE);
@@ -1482,7 +1482,7 @@ intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
regs = (struct overlay_registers __iomem *)
overlay->reg_bo->phys_handle->vaddr;
else
- regs = io_mapping_map_atomic_wc(dev_priv->ggtt.mappable,
+ regs = io_mapping_map_atomic_wc(&dev_priv->ggtt.mappable,
overlay->flip_addr);
return regs;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 3cefce588c82..17dae2ceb692 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -2009,7 +2009,7 @@ void intel_ring_unpin(struct intel_ring *ring)
static void intel_destroy_ringbuffer_obj(struct intel_ring *ring)
{
- i915_gem_object_put(ring->obj);
+ __i915_gem_object_release_unless_active(ring->obj);
ring->obj = NULL;
}
diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h
index fc65118e5077..205ddcf6d55d 100644
--- a/include/drm/drm_mm.h
+++ b/include/drm/drm_mm.h
@@ -37,6 +37,7 @@
* Generic range manager structs
*/
#include <linux/bug.h>
+#include <linux/rbtree.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/spinlock.h>
@@ -61,6 +62,7 @@ enum drm_mm_allocator_flags {
struct drm_mm_node {
struct list_head node_list;
struct list_head hole_stack;
+ struct rb_node rb;
unsigned hole_follows : 1;
unsigned scanned_block : 1;
unsigned scanned_prev_free : 1;
@@ -70,6 +72,7 @@ struct drm_mm_node {
unsigned long color;
u64 start;
u64 size;
+ u64 __subtree_last;
struct drm_mm *mm;
};
@@ -79,6 +82,9 @@ struct drm_mm {
/* head_node.node_list is the list of all memory nodes, ordered
* according to the (increasing) start address of the memory node. */
struct drm_mm_node head_node;
+ /* Keep an interval_tree for fast lookup of drm_mm_nodes by address. */
+ struct rb_root interval_tree;
+
unsigned int scan_check_range : 1;
unsigned scan_alignment;
unsigned long scan_color;
@@ -295,6 +301,12 @@ void drm_mm_init(struct drm_mm *mm,
void drm_mm_takedown(struct drm_mm *mm);
bool drm_mm_clean(struct drm_mm *mm);
+struct drm_mm_node *
+drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last);
+
+struct drm_mm_node *
+drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last);
+
void drm_mm_init_scan(struct drm_mm *mm,
u64 size,
unsigned alignment,
diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h
index 645ad06b5d52..cdb8870ecf90 100644
--- a/include/linux/io-mapping.h
+++ b/include/linux/io-mapping.h
@@ -31,16 +31,16 @@
* See Documentation/io-mapping.txt
*/
-#ifdef CONFIG_HAVE_ATOMIC_IOMAP
-
-#include <asm/iomap.h>
-
struct io_mapping {
resource_size_t base;
unsigned long size;
pgprot_t prot;
+ void __iomem *iomem;
};
+#ifdef CONFIG_HAVE_ATOMIC_IOMAP
+
+#include <asm/iomap.h>
/*
* For small address space machines, mapping large objects
* into the kernel virtual space isn't practical. Where
@@ -49,34 +49,25 @@ struct io_mapping {
*/
static inline struct io_mapping *
-io_mapping_create_wc(resource_size_t base, unsigned long size)
+io_mapping_init_wc(struct io_mapping *iomap,
+ resource_size_t base,
+ unsigned long size)
{
- struct io_mapping *iomap;
pgprot_t prot;
- iomap = kmalloc(sizeof(*iomap), GFP_KERNEL);
- if (!iomap)
- goto out_err;
-
if (iomap_create_wc(base, size, &prot))
- goto out_free;
+ return NULL;
iomap->base = base;
iomap->size = size;
iomap->prot = prot;
return iomap;
-
-out_free:
- kfree(iomap);
-out_err:
- return NULL;
}
static inline void
-io_mapping_free(struct io_mapping *mapping)
+io_mapping_fini(struct io_mapping *mapping)
{
iomap_free(mapping->base, mapping->size);
- kfree(mapping);
}
/* Atomic map/unmap */
@@ -121,21 +112,40 @@ io_mapping_unmap(void __iomem *vaddr)
#else
#include <linux/uaccess.h>
-
-/* this struct isn't actually defined anywhere */
-struct io_mapping;
+#include <asm/pgtable_types.h>
/* Create the io_mapping object*/
static inline struct io_mapping *
-io_mapping_create_wc(resource_size_t base, unsigned long size)
+io_mapping_init_wc(struct io_mapping *iomap,
+ resource_size_t base,
+ unsigned long size)
+{
+ iomap->base = base;
+ iomap->size = size;
+ iomap->iomem = ioremap_wc(base, size);
+ iomap->prot = pgprot_writecombine(PAGE_KERNEL_IO);
+
+ return iomap;
+}
+
+static inline void
+io_mapping_fini(struct io_mapping *mapping)
+{
+ iounmap(mapping->iomem);
+}
+
+/* Non-atomic map/unmap */
+static inline void __iomem *
+io_mapping_map_wc(struct io_mapping *mapping,
+ unsigned long offset,
+ unsigned long size)
{
- return (struct io_mapping __force *) ioremap_wc(base, size);
+ return mapping->iomem + offset;
}
static inline void
-io_mapping_free(struct io_mapping *mapping)
+io_mapping_unmap(void __iomem *vaddr)
{
- iounmap((void __force __iomem *) mapping);
}
/* Atomic map/unmap */
@@ -145,30 +155,37 @@ io_mapping_map_atomic_wc(struct io_mapping *mapping,
{
preempt_disable();
pagefault_disable();
- return ((char __force __iomem *) mapping) + offset;
+ return io_mapping_map_wc(mapping, offset, PAGE_SIZE);
}
static inline void
io_mapping_unmap_atomic(void __iomem *vaddr)
{
+ io_mapping_unmap(vaddr);
pagefault_enable();
preempt_enable();
}
-/* Non-atomic map/unmap */
-static inline void __iomem *
-io_mapping_map_wc(struct io_mapping *mapping,
- unsigned long offset,
- unsigned long size)
+#endif /* HAVE_ATOMIC_IOMAP */
+
+static inline struct io_mapping *
+io_mapping_create_wc(resource_size_t base,
+ unsigned long size)
{
- return ((char __force __iomem *) mapping) + offset;
+ struct io_mapping *iomap;
+
+ iomap = kmalloc(sizeof(*iomap), GFP_KERNEL);
+ if (!iomap)
+ return NULL;
+
+ return io_mapping_init_wc(iomap, base, size);
}
static inline void
-io_mapping_unmap(void __iomem *vaddr)
+io_mapping_free(struct io_mapping *iomap)
{
+ io_mapping_fini(iomap);
+ kfree(iomap);
}
-#endif /* HAVE_ATOMIC_IOMAP */
-
#endif /* _LINUX_IO_MAPPING_H */
--
2.8.1
More information about the Intel-gfx-trybot
mailing list