[PATCH] drm/i915: Update dma_fence_work
Thomas Hellström
thomas.hellstrom at linux.intel.com
Tue Oct 12 15:59:58 UTC 2021
Ready for upstreaming by removing the dependency on i915_sw_fence
Signed-off-by: Thomas Hellström <thomas.hellstrom at linux.intel.com>
---
drivers/gpu/drm/i915/gem/i915_gem_clflush.c | 37 +++--
drivers/gpu/drm/i915/i915_sw_fence_work.c | 155 +++++++++++++++-----
drivers/gpu/drm/i915/i915_sw_fence_work.h | 61 ++++----
drivers/gpu/drm/i915/i915_vma.c | 19 ++-
4 files changed, 185 insertions(+), 87 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
index f0435c6feb68..1f6ecc7cee97 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
@@ -28,6 +28,11 @@ static void clflush_work(struct dma_fence_work *base)
{
struct clflush *clflush = container_of(base, typeof(*clflush), base);
+ if (base->error) {
+ dma_fence_set_error(&base->dma, base->error);
+ return;
+ }
+
__do_clflush(clflush->obj);
}
@@ -103,16 +108,28 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj,
if (!(flags & I915_CLFLUSH_SYNC))
clflush = clflush_work_create(obj);
if (clflush) {
- i915_sw_fence_await_reservation(&clflush->base.chain,
- obj->base.resv, NULL, true,
- i915_fence_timeout(to_i915(obj->base.dev)),
- I915_FENCE_GFP);
- dma_resv_add_excl_fence(obj->base.resv, &clflush->base.dma);
- dma_fence_work_commit(&clflush->base);
- } else if (obj->mm.pages) {
- __do_clflush(obj);
- } else {
- GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
+ int ret;
+
+ ret = dma_fence_work_add_resv_dependencies
+ (&clflush->base, obj->base.resv,
+ i915_fence_timeout(to_i915(obj->base.dev)),
+ I915_FENCE_GFP, true);
+
+ if (ret) {
+ dma_fence_work_set_error_once(&clflush->base, ret);
+ dma_fence_work_commit(&clflush->base);
+ clflush = NULL;
+ } else {
+ dma_resv_add_excl_fence(obj->base.resv, &clflush->base.dma);
+ dma_fence_work_commit(&clflush->base);
+ }
+ }
+
+ if (!clflush) {
+ if (obj->mm.pages)
+ __do_clflush(obj);
+ else
+ GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
}
obj->cache_dirty = false;
diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.c b/drivers/gpu/drm/i915/i915_sw_fence_work.c
index 5b33ef23d54c..e583ccd74761 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence_work.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence_work.c
@@ -1,55 +1,72 @@
// SPDX-License-Identifier: MIT
/*
- * Copyright © 2019 Intel Corporation
+ * Copyright © 2019-2021 Intel Corporation
*/
+#include <linux/slab.h>
+#include <linux/dma-resv.h>
#include "i915_sw_fence_work.h"
-static void fence_complete(struct dma_fence_work *f)
+static void dma_fence_work_complete(struct dma_fence_work *f)
{
if (f->ops->release)
f->ops->release(f);
+
+ dma_fence_put(&f->dma);
+}
+
+static void dma_fence_work_irq_work(struct irq_work *irq_work)
+{
+ struct dma_fence_work *f = container_of(irq_work, typeof(*f), irq_work);
+
dma_fence_signal(&f->dma);
+ if (f->ops->release)
+ /* Note we take the signaled path in dma_fence_work_work() */
+ queue_work(system_unbound_wq, &f->work);
+ else
+ dma_fence_work_complete(f);
}
-static void fence_work(struct work_struct *work)
+static void dma_fence_work_cb_func(struct dma_fence *fence,
+ struct dma_fence_cb *cb)
{
- struct dma_fence_work *f = container_of(work, typeof(*f), work);
+ struct dma_fence_work_cb *wcb = container_of(cb, typeof(*wcb), cb);
+ struct dma_fence_work *f = wcb->waiter;
- f->ops->work(f);
+ dma_fence_work_set_error_once(f, fence->error);
- fence_complete(f);
- dma_fence_put(&f->dma);
+ if (!atomic_dec_and_test(&f->pending))
+ return;
+
+ if (f->ops->work)
+ queue_work(system_unbound_wq, &f->work);
+ else
+ irq_work_queue(&f->irq_work);
+
+ if (wcb->kfree_on_callback)
+ kfree(wcb);
}
-static int __i915_sw_fence_call
-fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+static void dma_fence_work_work(struct work_struct *work)
{
- struct dma_fence_work *f = container_of(fence, typeof(*f), chain);
-
- switch (state) {
- case FENCE_COMPLETE:
- if (fence->error)
- dma_fence_set_error(&f->dma, fence->error);
-
- if (!f->dma.error) {
- dma_fence_get(&f->dma);
- if (test_bit(DMA_FENCE_WORK_IMM, &f->dma.flags))
- fence_work(&f->work);
- else
- queue_work(system_unbound_wq, &f->work);
- } else {
- fence_complete(f);
- }
- break;
-
- case FENCE_FREE:
- dma_fence_put(&f->dma);
- break;
+ struct dma_fence_work *f = container_of(work, typeof(*f), work);
+
+ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->dma.flags)) {
+ dma_fence_work_complete(f);
+ return;
}
- return NOTIFY_DONE;
+ if (f->ops->work) {
+ bool cookie = dma_fence_begin_signalling();
+
+ f->ops->work(f);
+ dma_fence_end_signalling(cookie);
+ }
+
+ dma_fence_signal(&f->dma);
+
+ dma_fence_work_complete(f);
}
static const char *get_driver_name(struct dma_fence *fence)
@@ -68,8 +85,6 @@ static void fence_release(struct dma_fence *fence)
{
struct dma_fence_work *f = container_of(fence, typeof(*f), dma);
- i915_sw_fence_fini(&f->chain);
-
BUILD_BUG_ON(offsetof(typeof(*f), dma));
dma_fence_free(&f->dma);
}
@@ -84,16 +99,78 @@ void dma_fence_work_init(struct dma_fence_work *f,
const struct dma_fence_work_ops *ops)
{
f->ops = ops;
+ f->error = 0;
spin_lock_init(&f->lock);
dma_fence_init(&f->dma, &fence_ops, &f->lock, 0, 0);
- i915_sw_fence_init(&f->chain, fence_notify);
- INIT_WORK(&f->work, fence_work);
+ INIT_WORK(&f->work, dma_fence_work_work);
+ init_irq_work(&f->irq_work, dma_fence_work_irq_work);
+ atomic_set(&f->pending, 1);
+}
+
+void dma_fence_work_dependency(struct dma_fence_work *f,
+ struct dma_fence *dependency,
+ struct dma_fence_work_cb *wcb,
+ bool kfree_on_callback)
+{
+ wcb->waiter = f;
+ wcb->kfree_on_callback = kfree_on_callback;
+ atomic_inc(&f->pending);
+ if (dma_fence_add_callback(dependency, &wcb->cb, dma_fence_work_cb_func))
+ atomic_dec(&f->pending);
}
-int dma_fence_work_chain(struct dma_fence_work *f, struct dma_fence *signal)
+void dma_fence_work_commit(struct dma_fence_work *f)
{
- if (!signal)
- return 0;
+ if (atomic_dec_and_test(&f->pending))
+ queue_work(system_unbound_wq, &f->work);
+}
+
+void dma_fence_work_commit_imm(struct dma_fence_work *f)
+{
+ if (atomic_dec_and_test(&f->pending))
+ dma_fence_work_work(&f->work);
+}
+
+static int dma_fence_work_dependency_alloc(struct dma_fence_work *f,
+ struct dma_fence *dependency,
+ unsigned long timeout,
+ gfp_t gfp)
+{
+ struct dma_fence_work_cb *wcb;
+ signed long ret = 0;
+
+ if (dma_fence_is_signaled(dependency))
+ return dependency->error;
+
+ wcb = kmalloc(sizeof(*wcb), gfp);
+ if (!wcb) {
+ ret = dma_fence_wait_timeout(dependency, false, timeout);
+ if (ret < 0)
+ return ret;
+ else if (ret == 0)
+ return -ETIME;
+ } else {
+ dma_fence_work_dependency(f, dependency, wcb, true);
+ }
+
+ return ret;
+}
+
+int dma_fence_work_add_resv_dependencies(struct dma_fence_work *f,
+ struct dma_resv *resv,
+ unsigned long timeout,
+ gfp_t gfp,
+ bool write)
+{
+ struct dma_resv_iter iter;
+ struct dma_fence *fence;
+ int ret;
+
+ dma_resv_for_each_fence(&iter, resv, write, fence) {
+ ret = dma_fence_work_dependency_alloc(f, fence, timeout, gfp);
+ if (ret)
+ return ret;
+ }
- return __i915_sw_fence_await_dma_fence(&f->chain, signal, &f->cb);
+ return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.h b/drivers/gpu/drm/i915/i915_sw_fence_work.h
index d56806918d13..5549ed83636a 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence_work.h
+++ b/drivers/gpu/drm/i915/i915_sw_fence_work.h
@@ -1,19 +1,20 @@
/* SPDX-License-Identifier: MIT */
/*
- * Copyright © 2019 Intel Corporation
+ * Copyright © 2019-2021 Intel Corporation
*/
#ifndef I915_SW_FENCE_WORK_H
#define I915_SW_FENCE_WORK_H
#include <linux/dma-fence.h>
+#include <linux/irq_work.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
-#include "i915_sw_fence.h"
struct dma_fence_work;
+struct dma_resv;
struct dma_fence_work_ops {
const char *name;
@@ -23,45 +24,43 @@ struct dma_fence_work_ops {
struct dma_fence_work {
struct dma_fence dma;
- spinlock_t lock;
-
- struct i915_sw_fence chain;
- struct i915_sw_dma_fence_cb cb;
-
struct work_struct work;
+ struct irq_work irq_work;
+ /** Used as the dma_fence lock */
+ spinlock_t lock;
+ atomic_t pending;
+ int error;
const struct dma_fence_work_ops *ops;
};
-enum {
- DMA_FENCE_WORK_IMM = DMA_FENCE_FLAG_USER_BITS,
+struct dma_fence_work_cb {
+ struct dma_fence_cb cb;
+ struct dma_fence_work *waiter;
+ bool kfree_on_callback;
};
void dma_fence_work_init(struct dma_fence_work *f,
const struct dma_fence_work_ops *ops);
-int dma_fence_work_chain(struct dma_fence_work *f, struct dma_fence *signal);
-static inline void dma_fence_work_commit(struct dma_fence_work *f)
-{
- i915_sw_fence_commit(&f->chain);
-}
+void dma_fence_work_dependency(struct dma_fence_work *f,
+ struct dma_fence *dependency,
+ struct dma_fence_work_cb *wcb,
+ bool kfree_on_callback);
-/**
- * dma_fence_work_commit_imm: Commit the fence, and if possible execute locally.
- * @f: the fenced worker
- *
- * Instead of always scheduling a worker to execute the callback (see
- * dma_fence_work_commit()), we try to execute the callback immediately in
- * the local context. It is required that the fence be committed before it
- * is published, and that no other threads try to tamper with the number
- * of asynchronous waits on the fence (or else the callback will be
- * executed in the wrong context, i.e. not the callers).
- */
-static inline void dma_fence_work_commit_imm(struct dma_fence_work *f)
-{
- if (atomic_read(&f->chain.pending) <= 1)
- __set_bit(DMA_FENCE_WORK_IMM, &f->dma.flags);
+void dma_fence_work_commit(struct dma_fence_work *f);
- dma_fence_work_commit(f);
-}
+void dma_fence_work_commit_imm(struct dma_fence_work *f);
+int dma_fence_work_add_resv_dependencies(struct dma_fence_work *f,
+ struct dma_resv *resv,
+ unsigned long timeout,
+ gfp_t gfp,
+ bool write);
+
+static inline void
+dma_fence_work_set_error_once(struct dma_fence_work *f, int error)
+{
+ if (error)
+ cmpxchg(&f->error, 0, error);
+}
#endif /* I915_SW_FENCE_WORK_H */
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 4b7fc4647e46..0a0874980574 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -291,7 +291,7 @@ struct i915_vma_work {
struct i915_vm_pt_stash stash;
struct i915_vma *vma;
struct drm_i915_gem_object *pinned;
- struct i915_sw_dma_fence_cb cb;
+ struct dma_fence_work_cb cb;
enum i915_cache_level cache_level;
unsigned int flags;
};
@@ -301,6 +301,11 @@ static void __vma_bind(struct dma_fence_work *work)
struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
struct i915_vma *vma = vw->vma;
+ if (work->error) {
+ dma_fence_set_error(&work->dma, work->error);
+ return;
+ }
+
vma->ops->bind_vma(vw->vm, &vw->stash,
vma, vw->cache_level, vw->flags);
}
@@ -333,7 +338,7 @@ struct i915_vma_work *i915_vma_work(void)
return NULL;
dma_fence_work_init(&vw->base, &bind_ops);
- vw->base.dma.error = -EAGAIN; /* disable the worker by default */
+ vw->base.error = -EAGAIN; /* disable the worker by default */
return vw;
}
@@ -416,16 +421,16 @@ int i915_vma_bind(struct i915_vma *vma,
* part of the obj->resv->excl_fence as it only affects
* execution and not content or object's backing store lifetime.
*/
+
+ work->base.error = 0; /* enable the queue_work() */
+
prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
if (prev) {
- __i915_sw_fence_await_dma_fence(&work->base.chain,
- prev,
- &work->cb);
+ dma_fence_work_dependency(&work->base, prev, &work->cb,
+ false);
dma_fence_put(prev);
}
- work->base.dma.error = 0; /* enable the queue_work() */
-
if (vma->obj) {
__i915_gem_object_pin_pages(vma->obj);
work->pinned = i915_gem_object_get(vma->obj);
--
2.31.1
More information about the Intel-gfx-trybot
mailing list