[PATCH 08/14] drm/i915: Remove redundant TLB invalidate on switching contexts
Chris Wilson
chris at chris-wilson.co.uk
Sat Aug 6 13:34:02 UTC 2016
We are required to reload the TLBs around context switches
(MI_SET_CONTEXT specifically) and the recommendation is do that before
the MI_SET_CONTEXT so that it is serialised with the switch and not
forgotten:
[DevSNB] If Flush TLB invalidation Mode is enabled it’s the driver’s
responsibility to invalidate the TLBs at least once after the previous
context switch after any GTT mappings changed (including new GTT entries).
This can be done by a pipeline PIPE_CONTROL with TLB inv bit set
immediately before MI_SET_CONTEXT.
However, we already do an unconditional TLB invalidate before every
batch so this condition is satifisfied.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/Makefile | 1 +
drivers/gpu/drm/i915/i915_cmd_parser.c | 170 ++++++++++++---------------
drivers/gpu/drm/i915/i915_drv.h | 17 ++-
drivers/gpu/drm/i915/i915_gem.c | 12 +-
drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++---
drivers/gpu/drm/i915/i915_gem_context.c | 46 ++------
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 6 +
drivers/gpu/drm/i915/i915_gem_internal.c | 156 ++++++++++++++++++++++++
drivers/gpu/drm/i915/i915_gem_render_state.c | 155 ++++++++++++++++--------
drivers/gpu/drm/i915/i915_gem_render_state.h | 4 +-
drivers/gpu/drm/i915/intel_engine_cs.c | 11 +-
drivers/gpu/drm/i915/intel_lrc.c | 2 +-
drivers/gpu/drm/i915/intel_ringbuffer.c | 10 +-
drivers/gpu/drm/i915/intel_ringbuffer.h | 3 +
14 files changed, 393 insertions(+), 228 deletions(-)
create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index bf4508132887..0afe7b3cad07 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -33,6 +33,7 @@ i915-y += i915_cmd_parser.o \
i915_gem_execbuffer.o \
i915_gem_fence.o \
i915_gem_gtt.o \
+ i915_gem_internal.o \
i915_gem.o \
i915_gem_render_state.o \
i915_gem_request.o \
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index a1f4683f5c35..b35f31483887 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -458,6 +458,7 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
REG32(GEN7_GPGPU_DISPATCHDIMX),
REG32(GEN7_GPGPU_DISPATCHDIMY),
REG32(GEN7_GPGPU_DISPATCHDIMZ),
+ REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0),
REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1),
REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2),
@@ -473,6 +474,7 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
REG32(GEN7_L3SQCREG1),
REG32(GEN7_L3CNTLREG2),
REG32(GEN7_L3CNTLREG3),
+ REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
};
static const struct drm_i915_reg_descriptor hsw_render_regs[] = {
@@ -502,7 +504,10 @@ static const struct drm_i915_reg_descriptor hsw_render_regs[] = {
};
static const struct drm_i915_reg_descriptor gen7_blt_regs[] = {
+ REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),
+ REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
REG32(BCS_SWCTRL),
+ REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
};
static const struct drm_i915_reg_descriptor ivb_master_regs[] = {
@@ -746,17 +751,15 @@ static void fini_hash_table(struct intel_engine_cs *engine)
* Optionally initializes fields related to batch buffer command parsing in the
* struct intel_engine_cs based on whether the platform requires software
* command parsing.
- *
- * Return: non-zero if initialization fails
*/
-int intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
+void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
{
const struct drm_i915_cmd_table *cmd_tables;
int cmd_table_count;
int ret;
if (!IS_GEN7(engine->i915))
- return 0;
+ return;
switch (engine->id) {
case RCS:
@@ -811,24 +814,32 @@ int intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
break;
default:
MISSING_CASE(engine->id);
- BUG();
+ return;
}
- BUG_ON(!validate_cmds_sorted(engine, cmd_tables, cmd_table_count));
- BUG_ON(!validate_regs_sorted(engine));
+ if (!hash_empty(engine->cmd_hash)) {
+ DRM_DEBUG_DRIVER("%s: no commands?\n", engine->name);
+ return;
+ }
- WARN_ON(!hash_empty(engine->cmd_hash));
+ if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) {
+ DRM_ERROR("%s: command descriptions are not sorted\n",
+ engine->name);
+ return;
+ }
+ if (!validate_regs_sorted(engine)) {
+ DRM_ERROR("%s: registers are not sorted\n", engine->name);
+ return;
+ }
ret = init_hash_table(engine, cmd_tables, cmd_table_count);
if (ret) {
- DRM_ERROR("CMD: cmd_parser_init failed!\n");
+ DRM_ERROR("%s: initialised failed!\n", engine->name);
fini_hash_table(engine);
- return ret;
+ return;
}
engine->needs_cmd_parser = true;
-
- return 0;
}
/**
@@ -931,98 +942,61 @@ find_reg_in_tables(const struct drm_i915_reg_table *tables,
return NULL;
}
-static u32 *vmap_batch(struct drm_i915_gem_object *obj,
- unsigned start, unsigned len)
-{
- int i;
- void *addr = NULL;
- struct sg_page_iter sg_iter;
- int first_page = start >> PAGE_SHIFT;
- int last_page = (len + start + 4095) >> PAGE_SHIFT;
- int npages = last_page - first_page;
- struct page **pages;
-
- pages = drm_malloc_ab(npages, sizeof(*pages));
- if (pages == NULL) {
- DRM_DEBUG_DRIVER("Failed to get space for pages\n");
- goto finish;
- }
-
- i = 0;
- for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, first_page) {
- pages[i++] = sg_page_iter_page(&sg_iter);
- if (i == npages)
- break;
- }
-
- addr = vmap(pages, i, 0, PAGE_KERNEL);
- if (addr == NULL) {
- DRM_DEBUG_DRIVER("Failed to vmap pages\n");
- goto finish;
- }
-
-finish:
- if (pages)
- drm_free_large(pages);
- return (u32*)addr;
-}
-
-/* Returns a vmap'd pointer to dest_obj, which the caller must unmap */
-static u32 *copy_batch(struct drm_i915_gem_object *dest_obj,
+/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
+static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
struct drm_i915_gem_object *src_obj,
u32 batch_start_offset,
- u32 batch_len)
+ u32 batch_len,
+ bool *needs_clflush_after)
{
- unsigned int needs_clflush;
- void *src_base, *src;
- void *dst = NULL;
+ unsigned int src_needs_clflush;
+ unsigned int dst_needs_clflush;
+ void *dst, *ptr;
+ int offset, n;
int ret;
- if (batch_len > dest_obj->base.size ||
- batch_len + batch_start_offset > src_obj->base.size)
- return ERR_PTR(-E2BIG);
-
- if (WARN_ON(dest_obj->pages_pin_count == 0))
- return ERR_PTR(-ENODEV);
-
- ret = i915_gem_obj_prepare_shmem_read(src_obj, &needs_clflush);
- if (ret) {
- DRM_DEBUG_DRIVER("CMD: failed to prepare shadow batch\n");
+ ret = i915_gem_obj_prepare_shmem_read(src_obj, &src_needs_clflush);
+ if (ret)
return ERR_PTR(ret);
- }
- src_base = vmap_batch(src_obj, batch_start_offset, batch_len);
- if (!src_base) {
- DRM_DEBUG_DRIVER("CMD: Failed to vmap batch\n");
- ret = -ENOMEM;
+ ret = i915_gem_obj_prepare_shmem_write(dst_obj, &dst_needs_clflush);
+ if (ret) {
+ dst = ERR_PTR(ret);
goto unpin_src;
}
- ret = i915_gem_object_set_to_cpu_domain(dest_obj, true);
- if (ret) {
- DRM_DEBUG_DRIVER("CMD: Failed to set shadow batch to CPU\n");
- goto unmap_src;
- }
+ dst = i915_gem_object_pin_map(dst_obj);
+ if (IS_ERR(dst))
+ goto unpin_dst;
- dst = vmap_batch(dest_obj, 0, batch_len);
- if (!dst) {
- DRM_DEBUG_DRIVER("CMD: Failed to vmap shadow batch\n");
- ret = -ENOMEM;
- goto unmap_src;
- }
+ ptr = dst;
+ offset = offset_in_page(batch_start_offset);
+ if (dst_needs_clflush & CLFLUSH_BEFORE)
+ batch_len = roundup(batch_len, boot_cpu_data.x86_clflush_size);
- src = src_base + offset_in_page(batch_start_offset);
- if (needs_clflush)
- drm_clflush_virt_range(src, batch_len);
+ for (n = batch_start_offset >> PAGE_SHIFT; batch_len; n++) {
+ int len = min_t(int, batch_len, PAGE_SIZE - offset);
+ void *vaddr;
- memcpy(dst, src, batch_len);
+ vaddr = kmap_atomic(i915_gem_object_get_page(src_obj, n));
+ if (src_needs_clflush)
+ drm_clflush_virt_range(vaddr + offset, len);
+ memcpy(ptr, vaddr + offset, len);
+ kunmap_atomic(vaddr);
-unmap_src:
- vunmap(src_base);
+ ptr += len;
+ batch_len -= len;
+ offset = 0;
+ }
+
+ /* dst_obj is returned with vmap pinned */
+ *needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER;
+
+unpin_dst:
+ i915_gem_object_unpin_pages(dst_obj);
unpin_src:
i915_gem_object_unpin_pages(src_obj);
-
- return ret ? ERR_PTR(ret) : dst;
+ return dst;
}
static bool check_cmd(const struct intel_engine_cs *engine,
@@ -1179,16 +1153,18 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
u32 batch_len,
bool is_master)
{
- u32 *cmd, *batch_base, *batch_end;
+ u32 *cmd, *batch_end;
struct drm_i915_cmd_descriptor default_desc = { 0 };
bool oacontrol_set = false; /* OACONTROL tracking. See check_cmd() */
+ bool needs_clflush_after = false;
int ret = 0;
- batch_base = copy_batch(shadow_batch_obj, batch_obj,
- batch_start_offset, batch_len);
- if (IS_ERR(batch_base)) {
+ cmd = copy_batch(shadow_batch_obj, batch_obj,
+ batch_start_offset, batch_len,
+ &needs_clflush_after);
+ if (IS_ERR(cmd)) {
DRM_DEBUG_DRIVER("CMD: Failed to copy batch\n");
- return PTR_ERR(batch_base);
+ return PTR_ERR(cmd);
}
/*
@@ -1196,9 +1172,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
* large or larger and copy_batch() will write MI_NOPs to the extra
* space. Parsing should be faster in some cases this way.
*/
- batch_end = batch_base + (batch_len / sizeof(*batch_end));
-
- cmd = batch_base;
+ batch_end = cmd + (batch_len / sizeof(*batch_end));
while (cmd < batch_end) {
const struct drm_i915_cmd_descriptor *desc;
u32 length;
@@ -1257,7 +1231,9 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
ret = -EINVAL;
}
- vunmap(batch_base);
+ if (ret == 0 && needs_clflush_after)
+ drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len);
+ i915_gem_object_unpin_map(shadow_batch_obj);
return ret;
}
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 6abd7c34e003..e738ccf22551 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -904,8 +904,6 @@ struct i915_gem_context {
unsigned hw_id;
u32 user_handle;
- u32 ggtt_alignment;
-
struct intel_context {
struct i915_vma *state;
struct intel_ring *ring;
@@ -3148,13 +3146,14 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
-static inline int __sg_page_count(struct scatterlist *sg)
+static inline int __sg_page_count(const struct scatterlist *sg)
{
return sg->length >> PAGE_SHIFT;
}
struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+ unsigned int n);
static inline dma_addr_t
i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
@@ -3174,7 +3173,8 @@ i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
}
static inline struct page *
-i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
+i915_gem_object_get_page(struct drm_i915_gem_object *obj,
+ unsigned int n)
{
if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT))
return NULL;
@@ -3522,6 +3522,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
u32 gtt_offset,
u32 size);
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+ unsigned int size);
+
/* i915_gem_shrinker.c */
unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
unsigned long target,
@@ -3585,7 +3590,7 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
/* i915_cmd_parser.c */
int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
-int intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
+void intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
int intel_engine_cmd_parser(struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj,
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 2bf39bbc2803..89092994d5bf 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -4611,16 +4611,14 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
/* Like i915_gem_object_get_page(), but mark the returned page dirty */
struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+ unsigned int n)
{
- struct page *page;
+ struct page *page = i915_gem_object_get_page(obj, n);
- /* Only default objects have per-page dirty tracking */
- if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
- return NULL;
+ if (!i915_gem_object_is_dirty(obj))
+ set_page_dirty(page);
- page = i915_gem_object_get_page(obj, n);
- set_page_dirty(page);
return page;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index cb25cad3318c..3934c9103cf2 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
size_t size)
{
struct drm_i915_gem_object *obj = NULL;
- struct drm_i915_gem_object *tmp, *next;
+ struct drm_i915_gem_object *tmp;
struct list_head *list;
- int n;
+ int n, ret;
lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
n = ARRAY_SIZE(pool->cache_list) - 1;
list = &pool->cache_list[n];
- list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+ list_for_each_entry(tmp, list, batch_pool_link) {
/* The batches are strictly LRU ordered */
if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
&tmp->base.dev->struct_mutex))
break;
- /* While we're looping, do some clean up */
- if (tmp->madv == __I915_MADV_PURGED) {
- list_del(&tmp->batch_pool_link);
- i915_gem_object_put(tmp);
- continue;
- }
-
if (tmp->base.size >= size) {
obj = tmp;
break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
}
if (obj == NULL) {
- int ret;
-
- obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+ obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+ size);
if (IS_ERR(obj))
return obj;
-
- ret = i915_gem_object_get_pages(obj);
- if (ret)
- return ERR_PTR(ret);
-
- obj->madv = I915_MADV_DONTNEED;
}
+ ret = i915_gem_object_get_pages(obj);
+ if (ret)
+ return ERR_PTR(ret);
+
list_move_tail(&obj->batch_pool_link, list);
i915_gem_object_pin_pages(obj);
return obj;
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index e530176edacf..ecef433fe396 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -93,24 +93,9 @@
#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
-/* This is a HW constraint. The value below is the largest known requirement
- * I've seen in a spec to date, and that was a workaround for a non-shipping
- * part. It should be safe to decrease this, but it's more future proof as is.
- */
-#define GEN6_CONTEXT_ALIGN (64<<10)
-#define GEN7_CONTEXT_ALIGN 4096
-
/* Initial size (as log2) to preallocate the handle->object hashtable */
#define VMA_HT_BITS 2u /* 4 x 2 pointers, 64 bytes minimum */
-static size_t get_context_alignment(struct drm_i915_private *dev_priv)
-{
- if (IS_GEN6(dev_priv))
- return GEN6_CONTEXT_ALIGN;
-
- return GEN7_CONTEXT_ALIGN;
-}
-
static int get_context_size(struct drm_i915_private *dev_priv)
{
int ret;
@@ -343,8 +328,6 @@ __create_hw_context(struct drm_device *dev,
list_add_tail(&ctx->link, &dev_priv->context_list);
ctx->i915 = dev_priv;
- ctx->ggtt_alignment = get_context_alignment(dev_priv);
-
ctx->vma.ht_bits = VMA_HT_BITS;
ctx->vma.ht_size = 1 << ctx->vma.ht_bits;
ctx->vma.ht = kzalloc(sizeof(*ctx->vma.ht)*ctx->vma.ht_size,
@@ -662,17 +645,6 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
0;
int len, ret;
- /* w/a: If Flush TLB Invalidation Mode is enabled, driver must do a TLB
- * invalidation prior to MI_SET_CONTEXT. On GEN6 we don't set the value
- * explicitly, so we rely on the value at ring init, stored in
- * itlb_before_ctx_switch.
- */
- if (IS_GEN6(dev_priv)) {
- ret = engine->emit_flush(req, EMIT_INVALIDATE);
- if (ret)
- return ret;
- }
-
/* These flags are for resource streamer on HSW+ */
if (IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8)
flags |= (HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN);
@@ -796,10 +768,10 @@ static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt,
}
static bool
-needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt,
- struct intel_engine_cs *engine,
- struct i915_gem_context *to)
+needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt, struct intel_engine_cs *engine)
{
+ struct i915_hw_ppgtt *last_ppgtt;
+
if (!ppgtt)
return false;
@@ -808,7 +780,9 @@ needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt,
return true;
/* Same context without new entries, skip */
- if (engine->last_context == to &&
+ last_ppgtt =
+ engine->last_context->ppgtt ?: engine->i915->mm.aliasing_ppgtt;
+ if (last_ppgtt == ppgtt &&
!(intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
return false;
@@ -865,9 +839,7 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
return 0;
/* Trying to pin first makes error handling easier. */
- ret = i915_vma_pin(to->engine[RCS].state,
- 0, to->ggtt_alignment,
- PIN_GLOBAL);
+ ret = i915_vma_pin(to->engine[RCS].state, 0, 0, PIN_GLOBAL);
if (ret)
return ret;
@@ -885,7 +857,7 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
*/
flush_cpu_writes(to->engine[RCS].state->obj);
- if (needs_pd_load_pre(ppgtt, engine, to)) {
+ if (needs_pd_load_pre(ppgtt, engine)) {
/* Older GENs and non render rings still want the load first,
* "PP_DCLV followed by PP_DIR_BASE register through Load
* Register Immediate commands in Ring Buffer before submitting
@@ -1006,7 +978,7 @@ int i915_switch_context(struct drm_i915_gem_request *req)
struct i915_hw_ppgtt *ppgtt =
to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
- if (needs_pd_load_pre(ppgtt, engine, to)) {
+ if (needs_pd_load_pre(ppgtt, engine)) {
int ret;
trace_switch_mm(engine, to);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index e53c66cb0b2e..3d03a2853a29 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -1803,6 +1803,12 @@ i915_gem_do_execbuffer(struct drm_device *dev,
ret = -EINVAL;
goto err;
}
+ if (args->batch_start_offset > eb.batch->size ||
+ args->batch_len > eb.batch->size - args->batch_start_offset) {
+ DRM_DEBUG("Attempting to use out-of-bounds batch\n");
+ ret = -EINVAL;
+ goto err;
+ }
if (intel_engine_needs_cmd_parser(eb.engine) && args->batch_len) {
struct i915_vma *vma;
diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
new file mode 100644
index 000000000000..5347ddd92a09
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright © 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+static void __i915_gem_object_free_pages(struct sg_table *st)
+{
+ struct sg_page_iter sg_iter;
+
+ for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
+ put_page(sg_page_iter_page(&sg_iter));
+
+ sg_free_table(st);
+ kfree(st);
+}
+
+static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
+{
+ const unsigned int npages = obj->base.size / PAGE_SIZE;
+ struct sg_table *st;
+ struct scatterlist *sg;
+ unsigned long last_pfn = 0; /* suppress gcc warning */
+ gfp_t gfp;
+ int i;
+
+ st = kmalloc(sizeof(*st), GFP_KERNEL);
+ if (!st)
+ return -ENOMEM;
+
+ if (sg_alloc_table(st, npages, GFP_KERNEL)) {
+ kfree(st);
+ return -ENOMEM;
+ }
+
+ sg = st->sgl;
+ st->nents = 0;
+
+ gfp = GFP_KERNEL | __GFP_HIGHMEM;
+ gfp |= __GFP_NORETRY | __GFP_NOWARN;
+ gfp &= ~(__GFP_IO | __GFP_RECLAIM);
+ for (i = 0; i < npages; i++) {
+ struct page *page;
+
+ page = alloc_page(gfp);
+ if (!page) {
+ i915_gem_shrink_all(to_i915(obj->base.dev));
+ page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
+ if (!page)
+ goto err;
+ }
+
+#ifdef CONFIG_SWIOTLB
+ if (swiotlb_nr_tbl()) {
+ st->nents++;
+ sg_set_page(sg, page, PAGE_SIZE, 0);
+ sg = sg_next(sg);
+ continue;
+ }
+#endif
+ if (!i || page_to_pfn(page) != last_pfn + 1) {
+ if (i)
+ sg = sg_next(sg);
+ st->nents++;
+ sg_set_page(sg, page, PAGE_SIZE, 0);
+ } else {
+ sg->length += PAGE_SIZE;
+ }
+ last_pfn = page_to_pfn(page);
+ }
+#ifdef CONFIG_SWIOTLB
+ if (!swiotlb_nr_tbl())
+#endif
+ sg_mark_end(sg);
+ obj->pages = st;
+
+ if (i915_gem_gtt_prepare_object(obj)) {
+ obj->pages = NULL;
+ goto err;
+ }
+
+ obj->madv = I915_MADV_DONTNEED;
+ return 0;
+
+err:
+ sg_mark_end(sg);
+ __i915_gem_object_free_pages(st);
+ return -ENOMEM;
+}
+
+static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
+{
+ __i915_gem_object_free_pages(obj->pages);
+
+ i915_gem_object_clear_dirty(obj);
+ obj->madv = I915_MADV_WILLNEED;
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
+ .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
+ .get_pages = i915_gem_object_get_pages_internal,
+ .put_pages = i915_gem_object_put_pages_internal,
+};
+
+/**
+ * Creates a new object that wraps some internal memory for private use.
+ * This object is not backed by swappable storage, and as such its contents
+ * are volatile and only valid whilst pinned. If the object is reaped by the
+ * shrinker, its pages and data will be discarded. Equally, it is not a full
+ * GEM object and so not valid for access from userspace. This makes it useful
+ * for hardware interfaces like ringbuffers (which are pinned from the time
+ * the request is written to the time the hardware stops accessing it), but
+ * not for contexts (which need to be preserved when not active for later
+ * reuse).
+ */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+ unsigned int size)
+{
+ struct drm_i915_gem_object *obj;
+
+ obj = i915_gem_object_alloc(dev);
+ if (!obj)
+ return ERR_PTR(-ENOMEM);
+
+ drm_gem_private_object_init(dev, &obj->base, size);
+ i915_gem_object_init(obj, &i915_gem_object_internal_ops);
+
+ obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+ obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+ obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE;
+
+ return obj;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index adcd02abeabb..14c5e80af073 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -28,18 +28,19 @@
#include "i915_drv.h"
#include "intel_renderstate.h"
-struct render_state {
+struct intel_render_state {
const struct intel_renderstate_rodata *rodata;
- struct drm_i915_gem_object *obj;
struct i915_vma *vma;
- u32 aux_batch_size;
- u32 aux_batch_offset;
+ u32 batch_offset;
+ u32 batch_size;
+ u32 aux_offset;
+ u32 aux_size;
};
static const struct intel_renderstate_rodata *
-render_state_get_rodata(const struct drm_i915_gem_request *req)
+render_state_get_rodata(const struct intel_engine_cs *engine)
{
- switch (INTEL_GEN(req->i915)) {
+ switch (INTEL_GEN(engine->i915)) {
case 6:
return &gen6_null_state;
case 7:
@@ -71,22 +72,22 @@ render_state_get_rodata(const struct drm_i915_gem_request *req)
(batch)[(i)++] = (val); \
} while(0)
-static int render_state_setup(struct render_state *so)
+static int render_state_setup(struct intel_render_state *so,
+ struct drm_i915_private *i915)
{
- struct drm_device *dev = so->obj->base.dev;
const struct intel_renderstate_rodata *rodata = so->rodata;
- const bool has_64bit_reloc = INTEL_GEN(dev) >= 8;
+ const bool has_64bit_reloc = INTEL_GEN(i915) >= 8;
+ struct drm_i915_gem_object *obj = so->vma->obj;
unsigned int i = 0, reloc_index = 0;
- struct page *page;
+ unsigned needs_clflush;
u32 *d;
int ret;
- ret = i915_gem_object_set_to_cpu_domain(so->obj, true);
+ ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
if (ret)
return ret;
- page = i915_gem_object_get_dirty_page(so->obj, 0);
- d = kmap(page);
+ d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0));
while (i < rodata->batch_items) {
u32 s = rodata->batch[i];
@@ -111,12 +112,15 @@ static int render_state_setup(struct render_state *so)
d[i++] = s;
}
+ so->batch_offset = so->vma->node.start;
+ so->batch_size = rodata->batch_items * sizeof(u32);
+
while (i % CACHELINE_DWORDS)
OUT_BATCH(d, i, MI_NOOP);
- so->aux_batch_offset = i * sizeof(u32);
+ so->aux_offset = i * sizeof(u32);
- if (HAS_POOLED_EU(dev)) {
+ if (HAS_POOLED_EU(i915)) {
/*
* We always program 3x6 pool config but depending upon which
* subslice is disabled HW drops down to appropriate config
@@ -144,83 +148,130 @@ static int render_state_setup(struct render_state *so)
}
OUT_BATCH(d, i, MI_BATCH_BUFFER_END);
- so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset;
-
+ so->aux_size = i * sizeof(u32) - so->aux_offset;
+ so->aux_offset += so->batch_offset;
/*
* Since we are sending length, we need to strictly conform to
* all requirements. For Gen2 this must be a multiple of 8.
*/
- so->aux_batch_size = ALIGN(so->aux_batch_size, 8);
+ so->aux_size = ALIGN(so->aux_size, 8);
- kunmap(page);
-
- ret = i915_gem_object_set_to_gtt_domain(so->obj, false);
- if (ret)
- return ret;
+ if (needs_clflush)
+ drm_clflush_virt_range(d, i*sizeof(u32));
+ ret = i915_gem_object_set_to_gtt_domain(obj, false);
if (rodata->reloc[reloc_index] != -1) {
DRM_ERROR("only %d relocs resolved\n", reloc_index);
- return -EINVAL;
+ ret = -EINVAL;
}
- return 0;
-
err_out:
- kunmap(page);
+ kunmap_atomic(d);
+ i915_gem_object_unpin_pages(obj);
return ret;
}
#undef OUT_BATCH
-int i915_gem_render_state_init(struct drm_i915_gem_request *req)
+int i915_gem_render_state_init(struct intel_engine_cs *engine)
{
- struct render_state so;
+ struct intel_render_state *so;
+ const struct intel_renderstate_rodata *rodata;
+ struct drm_i915_gem_object *obj;
int ret;
- if (WARN_ON(req->engine->id != RCS))
- return -ENOENT;
+ if (engine->id != RCS)
+ return 0;
- so.rodata = render_state_get_rodata(req);
- if (!so.rodata)
+ rodata = render_state_get_rodata(engine);
+ if (!rodata)
return 0;
- if (so.rodata->batch_items * 4 > 4096)
+ if (rodata->batch_items * 4 > 4096)
return -EINVAL;
- so.obj = i915_gem_object_create(&req->i915->drm, 4096);
- if (IS_ERR(so.obj))
- return PTR_ERR(so.obj);
+ so = kmalloc(sizeof(*so), GFP_KERNEL);
+ if (!so)
+ return -ENOMEM;
- so.vma = i915_gem_object_ggtt_pin(so.obj, NULL, 0, 0, 0);
- if (IS_ERR(so.vma)) {
- ret = PTR_ERR(so.vma);
+ obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
+ if (IS_ERR(obj)) {
+ ret = PTR_ERR(obj);
+ goto err_free;
+ }
+
+ so->vma = i915_gem_obj_lookup_or_create_vma(obj,
+ &engine->i915->ggtt.base,
+ NULL);
+ if (IS_ERR(so->vma)) {
+ ret = PTR_ERR(so->vma);
goto err_obj;
}
- ret = render_state_setup(&so);
+ so->rodata = rodata;
+ engine->render_state = so;
+ return 0;
+
+err_obj:
+ i915_gem_object_put(obj);
+err_free:
+ kfree(so);
+ return ret;
+}
+
+int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
+{
+ struct intel_render_state *so;
+ int ret;
+
+ so = req->engine->render_state;
+ if (!so)
+ return 0;
+
+ /* Recreate the page after shrinking */
+ if (!so->vma->obj->pages)
+ so->batch_offset = -1;
+
+ ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL);
if (ret)
- goto err_unpin;
+ return ret;
- ret = req->engine->emit_bb_start(req, so.vma->node.start,
- so.rodata->batch_items * 4,
+ if (so->vma->node.start != so->batch_offset) {
+ ret = render_state_setup(so, req->i915);
+ if (ret)
+ goto err_unpin;
+ }
+
+ ret = req->engine->emit_bb_start(req,
+ so->batch_offset, so->batch_size,
I915_DISPATCH_SECURE);
if (ret)
goto err_unpin;
- if (so.aux_batch_size > 8) {
+ if (so->aux_size > 8) {
ret = req->engine->emit_bb_start(req,
- (so.vma->node.start +
- so.aux_batch_offset),
- so.aux_batch_size,
+ so->aux_offset, so->aux_size,
I915_DISPATCH_SECURE);
if (ret)
goto err_unpin;
}
- i915_vma_move_to_active(so.vma, req, 0);
+ i915_vma_move_to_active(so->vma, req, 0);
err_unpin:
- i915_vma_unpin(so.vma);
-err_obj:
- __i915_gem_object_release_unless_active(so.obj);
+ i915_vma_unpin(so->vma);
return ret;
}
+
+void i915_gem_render_state_fini(struct intel_engine_cs *engine)
+{
+ struct intel_render_state *so;
+
+ so = engine->render_state;
+ if (so == NULL)
+ return;
+
+ __i915_gem_object_release_unless_active(so->vma->obj);
+ kfree(so);
+
+ engine->render_state = NULL;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
index 18cce3f06e9c..87481845799d 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.h
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -26,6 +26,8 @@
struct drm_i915_gem_request;
-int i915_gem_render_state_init(struct drm_i915_gem_request *req);
+int i915_gem_render_state_init(struct intel_engine_cs *engine);
+int i915_gem_render_state_emit(struct drm_i915_gem_request *req);
+void i915_gem_render_state_fini(struct intel_engine_cs *engine);
#endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 0dd3d1de18aa..394ef2ca2bd2 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -193,6 +193,8 @@ void intel_engine_setup_common(struct intel_engine_cs *engine)
intel_engine_init_requests(engine);
intel_engine_init_hangcheck(engine);
i915_gem_batch_pool_init(engine, &engine->batch_pool);
+
+ intel_engine_init_cmd_parser(engine);
}
/**
@@ -214,7 +216,11 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
if (ret)
return ret;
- return intel_engine_init_cmd_parser(engine);
+ ret = i915_gem_render_state_init(engine);
+ if (ret)
+ return ret;
+
+ return 0;
}
/**
@@ -226,7 +232,8 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
*/
void intel_engine_cleanup_common(struct intel_engine_cs *engine)
{
- intel_engine_cleanup_cmd_parser(engine);
+ i915_gem_render_state_fini(engine);
intel_engine_fini_breadcrumbs(engine);
+ intel_engine_cleanup_cmd_parser(engine);
i915_gem_batch_pool_fini(&engine->batch_pool);
}
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index aebf6c7599ed..97c457dbb383 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -1659,7 +1659,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
if (ret)
DRM_ERROR("MOCS failed to program: expect performance issues.\n");
- return i915_gem_render_state_init(req);
+ return i915_gem_render_state_emit(req);
}
/**
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 130ba99be9d1..bbbf128701ff 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -633,7 +633,7 @@ int intel_init_pipe_control(struct intel_engine_cs *engine, int size)
obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
if (!obj)
- obj = i915_gem_object_create(&engine->i915->drm, size);
+ obj = i915_gem_object_create_internal(&engine->i915->drm, size);
if (IS_ERR(obj)) {
DRM_ERROR("Failed to allocate scratch page\n");
ret = PTR_ERR(obj);
@@ -700,7 +700,7 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
if (ret != 0)
return ret;
- ret = i915_gem_render_state_init(req);
+ ret = i915_gem_render_state_emit(req);
if (ret)
return ret;
@@ -1879,7 +1879,7 @@ static int init_status_page(struct intel_engine_cs *engine)
if (engine->status_page.vma)
return 0;
- obj = i915_gem_object_create(&engine->i915->drm, 4096);
+ obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
if (IS_ERR(obj)) {
DRM_ERROR("Failed to allocate status page\n");
return PTR_ERR(obj);
@@ -2098,8 +2098,7 @@ static int intel_ring_context_pin(struct i915_gem_context *ctx,
return 0;
if (ce->state) {
- ret = i915_vma_pin(ce->state, 0, ctx->ggtt_alignment,
- PIN_GLOBAL | PIN_HIGH);
+ ret = i915_vma_pin(ce->state, 0, 0, PIN_GLOBAL | PIN_HIGH);
if (ret)
goto error;
}
@@ -2165,7 +2164,6 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
ret = intel_ring_context_pin(dev_priv->kernel_context, engine);
if (ret)
goto error;
-
ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
if (IS_ERR(ring)) {
ret = PTR_ERR(ring);
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 680e3fbeef37..7e7cdc302b22 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -132,6 +132,7 @@ struct i915_ctx_workarounds {
};
struct drm_i915_gem_request;
+struct intel_render_state;
struct intel_engine_cs {
struct drm_i915_private *i915;
@@ -154,6 +155,8 @@ struct intel_engine_cs {
struct intel_ring *buffer;
struct list_head buffers;
+ struct intel_render_state *render_state;
+
/* Rather than have every client wait upon all user interrupts,
* with the herd waking after every interrupt and each doing the
* heavyweight seqno dance, we delegate the task (of being the
--
2.8.1
More information about the Intel-gfx-trybot
mailing list