[PATCH 4/4] drm/i915/gem: Introduce migration fence handling for pipelined migration
Thomas Hellström
thomas.hellstrom at linux.intel.com
Sat Jun 19 18:00:32 UTC 2021
We want to be able to pipeline migration blits and clears.
Introduce a migration fence that can be get using a drm_i915_gem_object ops
addition, that can be waited upon or that can be used to queue other work
on to indicate migration complete.
It's important that we don't expose uninitialized memory
(migration not complete or in error state) to user-space so make sure we wait
or check for errors before exposing memory to user-space
by binding vmas or faulting in CPU ptes.
Also make sure we wait before setting up kernel mappings so there
are no surprises for in-kernel users getting their data overwritten.
Signed-off-by: Thomas Hellström <thomas.hellstrom at linux.intel.com>
---
drivers/gpu/drm/i915/gem/i915_gem_mman.c | 5 +++
drivers/gpu/drm/i915/gem/i915_gem_object.h | 30 ++++++++++++++++
.../gpu/drm/i915/gem/i915_gem_object_types.h | 5 +++
drivers/gpu/drm/i915/gem/i915_gem_pages.c | 5 ++-
drivers/gpu/drm/i915/i915_vma.c | 36 +++++++++++++++----
5 files changed, 73 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 2fd155742bd2..6e0e5bc31392 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -261,6 +261,11 @@ static vm_fault_t vm_fault_cpu(struct vm_fault *vmf)
if (err)
goto out;
+ /* Don't expose uninitialized memory to user-space! */
+ err = i915_gem_object_wait_migration(obj, true);
+ if (err)
+ goto out;
+
iomap = -1;
if (!i915_gem_object_has_struct_page(obj)) {
iomap = obj->mm.region->iomap.base;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 7bf4dd46d8d2..418ca05b1ac3 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -605,6 +605,36 @@ bool i915_gem_object_migratable(struct drm_i915_gem_object *obj);
bool i915_gem_object_validates_to_lmem(struct drm_i915_gem_object *obj);
+static inline struct dma_fence *
+i915_gem_object_migration_fence_get(struct drm_i915_gem_object *obj)
+{
+ return !obj->ops->migration_fence_get ? NULL :
+ obj->ops->migration_fence_get(obj);
+}
+
+static inline int
+i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, bool intr)
+{
+ struct dma_fence *fence;
+ int ret;
+
+ fence = i915_gem_object_migration_fence_get(obj);
+ if (IS_ERR_OR_NULL(fence))
+ return PTR_ERR_OR_ZERO(fence);
+
+ if (dma_fence_is_signaled(fence)) {
+ ret = fence->error;
+ goto out_put;
+ }
+
+ ret = dma_fence_wait(fence, intr);
+ if (!ret)
+ ret = fence->error;
+out_put:
+ dma_fence_put(fence);
+ return ret;
+}
+
#ifdef CONFIG_MMU_NOTIFIER
static inline bool
i915_gem_object_is_userptr(struct drm_i915_gem_object *obj)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index 3a2d9ecf8e03..4bc23c652d89 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -80,6 +80,11 @@ struct drm_i915_gem_object_ops {
void (*delayed_free)(struct drm_i915_gem_object *obj);
void (*release)(struct drm_i915_gem_object *obj);
+ /**
+ * migration_fence - Obtain the object's migration fence
+ */
+ struct dma_fence *(*migration_fence_get)(struct drm_i915_gem_object *obj);
+
const struct vm_operations_struct *mmap_ops;
const char *name; /* friendly name for debug, e.g. lockdep classes */
};
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
index 086005c1c7ea..ee232d36dda1 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
@@ -387,7 +387,10 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
}
if (!ptr) {
- if (GEM_WARN_ON(type == I915_MAP_WC &&
+ err = i915_gem_object_wait_migration(obj, true);
+ if (err)
+ ptr = ERR_PTR(err);
+ else if (GEM_WARN_ON(type == I915_MAP_WC &&
!static_cpu_has(X86_FEATURE_PAT)))
ptr = ERR_PTR(-ENODEV);
else if (i915_gem_object_has_struct_page(obj))
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 63565057d830..d13c433bc8bc 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -406,8 +406,9 @@ int i915_vma_bind(struct i915_vma *vma,
trace_i915_vma_bind(vma, bind_flags);
if (work && bind_flags & vma->vm->bind_async_flags) {
+ struct drm_i915_gem_object *obj = vma->obj;
struct dma_fence *prev;
- int ret = 0;
+ int ret;
work->vma = vma;
work->cache_level = cache_level;
@@ -415,8 +416,7 @@ int i915_vma_bind(struct i915_vma *vma,
/*
* Note we only want to chain up to the migration fence on
- * the pages (not the object itself). As we don't track that,
- * yet, we have to use the exclusive fence instead.
+ * the pages (not the object itself).
*
* Also note that we do not want to track the async vma as
* part of the obj->resv->excl_fence as it only affects
@@ -424,17 +424,35 @@ int i915_vma_bind(struct i915_vma *vma,
*/
prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
if (!IS_ERR_OR_NULL(prev)) {
+ /*
+ * We're chaining up on a previous bind, not completed
+ * yet. This means we're also chaining on the
+ * migration fence if any.
+ */
__i915_sw_fence_await_dma_fence(&work->base.chain,
prev,
&work->cb);
dma_fence_put(prev);
- } else if (IS_ERR(prev)) {
- ret = PTR_ERR(prev);
- if (ret == -EAGAIN)
- ret = 0;
+ } else if (PTR_ERR_OR_ZERO(prev) == -EAGAIN) {
+ prev = NULL;
+ }
+
+ if (!prev && obj) {
+ /*
+ * No bind in progress, and no previous fatal bind
+ * error. Make sure we chain on the migration fence.
+ */
+ prev = i915_gem_object_migration_fence_get(obj);
+ if (!IS_ERR_OR_NULL(prev)) {
+ __i915_sw_fence_await_dma_fence(&work->base.chain,
+ prev,
+ &work->cb);
+ dma_fence_put(prev);
+ }
}
/* Propagate previous error or enable the queue_work() */
+ ret = PTR_ERR_OR_ZERO(prev);
work->base.dma.error = ret;
if (ret)
return ret;
@@ -934,6 +952,10 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
if (err)
goto err_fence;
}
+ } else if (vma->obj) {
+ err = i915_gem_object_wait_migration(vma->obj, true);
+ if (err)
+ goto err_fence;
}
/*
--
2.31.1
More information about the Intel-gfx-trybot
mailing list