[PATCH 2/2] drm/i915: Update dma_fence_work
Thomas Hellström
thomas.hellstrom at linux.intel.com
Wed Oct 13 07:35:48 UTC 2021
Ready for upstreaming by removing the dependency on i915_sw_fence
Signed-off-by: Thomas Hellström <thomas.hellstrom at linux.intel.com>
---
drivers/gpu/drm/i915/Makefile | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_clflush.c | 42 +++--
drivers/gpu/drm/i915/i915_dma_fence_work.c | 184 ++++++++++++++++++++
drivers/gpu/drm/i915/i915_dma_fence_work.h | 67 +++++++
drivers/gpu/drm/i915/i915_sw_fence_work.c | 99 -----------
drivers/gpu/drm/i915/i915_sw_fence_work.h | 67 -------
drivers/gpu/drm/i915/i915_vma.c | 21 ++-
7 files changed, 295 insertions(+), 187 deletions(-)
create mode 100644 drivers/gpu/drm/i915/i915_dma_fence_work.c
create mode 100644 drivers/gpu/drm/i915/i915_dma_fence_work.h
delete mode 100644 drivers/gpu/drm/i915/i915_sw_fence_work.c
delete mode 100644 drivers/gpu/drm/i915/i915_sw_fence_work.h
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index cdc244bbbfc1..57ff28231b76 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -62,7 +62,7 @@ i915-y += \
i915_memcpy.o \
i915_mm.o \
i915_sw_fence.o \
- i915_sw_fence_work.o \
+ i915_dma_fence_work.o \
i915_syncmap.o \
i915_user_extensions.o
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
index 81c536a163cf..d0cf5b012ed7 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c
@@ -8,7 +8,7 @@
#include "i915_drv.h"
#include "i915_gem_clflush.h"
-#include "i915_sw_fence_work.h"
+#include "i915_dma_fence_work.h"
#include "i915_trace.h"
struct clflush {
@@ -26,6 +26,11 @@ static void clflush_work(struct dma_fence_work *base)
{
struct clflush *clflush = container_of(base, typeof(*clflush), base);
+ if (base->error) {
+ dma_fence_set_error(&base->dma, base->error);
+ return;
+ }
+
__do_clflush(clflush->obj);
}
@@ -101,17 +106,30 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj,
if (!(flags & I915_CLFLUSH_SYNC) && !i915_gem_object_is_framebuffer(obj))
clflush = clflush_work_create(obj);
if (clflush) {
- i915_sw_fence_await_reservation(&clflush->base.chain,
- obj->base.resv, NULL, true,
- i915_fence_timeout(to_i915(obj->base.dev)),
- I915_FENCE_GFP);
- dma_resv_add_excl_fence(obj->base.resv, &clflush->base.dma);
- dma_fence_work_commit(&clflush->base);
- } else if (obj->mm.pages) {
- __do_clflush(obj);
- i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
- } else {
- GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
+ int ret;
+
+ ret = dma_fence_work_add_resv_dependencies
+ (&clflush->base, obj->base.resv,
+ i915_fence_timeout(to_i915(obj->base.dev)),
+ I915_FENCE_GFP, true);
+
+ if (ret) {
+ dma_fence_work_set_error_once(&clflush->base, ret);
+ dma_fence_work_commit(&clflush->base);
+ clflush = NULL;
+ } else {
+ dma_resv_add_excl_fence(obj->base.resv, &clflush->base.dma);
+ dma_fence_work_commit(&clflush->base);
+ }
+ }
+
+ if (!clflush) {
+ if (obj->mm.pages) {
+ __do_clflush(obj);
+ i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
+ } else {
+ GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
+ }
}
obj->cache_dirty = false;
diff --git a/drivers/gpu/drm/i915/i915_dma_fence_work.c b/drivers/gpu/drm/i915/i915_dma_fence_work.c
new file mode 100644
index 000000000000..2b29628acace
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_dma_fence_work.c
@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: MIT
+
+/*
+ * Copyright © 2019-2021 Intel Corporation
+ */
+
+#include <linux/slab.h>
+#include <linux/dma-resv.h>
+
+#include "i915_dma_fence_work.h"
+
+static void dma_fence_work_complete(struct dma_fence_work *f)
+{
+ if (f->ops->release)
+ f->ops->release(f);
+
+ dma_fence_put(&f->dma);
+}
+
+static void dma_fence_work_irq_work(struct irq_work *irq_work)
+{
+ struct dma_fence_work *f = container_of(irq_work, typeof(*f), irq_work);
+
+ dma_fence_signal(&f->dma);
+
+ /* Never run release from irq context */
+ if (f->ops->release)
+ queue_work(system_unbound_wq, &f->work);
+ else
+ dma_fence_work_complete(f);
+}
+
+static void dma_fence_work_cb_func(struct dma_fence *fence,
+ struct dma_fence_cb *cb)
+{
+ struct dma_fence_work_cb *wcb = container_of(cb, typeof(*wcb), cb);
+ struct dma_fence_work *f = wcb->waiter;
+
+ dma_fence_work_set_error_once(f, fence->error);
+
+ if (refcount_dec_and_test(&f->pending)) {
+ /*
+ * To avoid lockdep issues with recursive fence locks,
+ * we can't signal from the callback. Queue work to signal,
+ * and use low-latency irq work if there is no work
+ * callback to run.
+ */
+ if (f->ops->work)
+ queue_work(system_unbound_wq, &f->work);
+ else
+ irq_work_queue(&f->irq_work);
+ }
+
+ if (wcb->kfree_on_callback)
+ kfree(wcb);
+}
+
+static void dma_fence_work_work(struct work_struct *work)
+{
+ struct dma_fence_work *f = container_of(work, typeof(*f), work);
+
+ if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->dma.flags)) {
+ if (f->ops->work) {
+ bool cookie = dma_fence_begin_signalling();
+
+ f->ops->work(f);
+ dma_fence_end_signalling(cookie);
+ }
+
+ dma_fence_signal(&f->dma);
+ }
+
+ dma_fence_work_complete(f);
+}
+
+static const char *get_driver_name(struct dma_fence *fence)
+{
+ return "dma-fence";
+}
+
+static const char *get_timeline_name(struct dma_fence *fence)
+{
+ struct dma_fence_work *f = container_of(fence, typeof(*f), dma);
+
+ return f->ops->name ?: "work";
+}
+
+static void fence_release(struct dma_fence *fence)
+{
+ struct dma_fence_work *f = container_of(fence, typeof(*f), dma);
+
+ BUILD_BUG_ON(offsetof(typeof(*f), dma));
+ dma_fence_free(&f->dma);
+}
+
+static const struct dma_fence_ops fence_ops = {
+ .get_driver_name = get_driver_name,
+ .get_timeline_name = get_timeline_name,
+ .release = fence_release,
+};
+
+
+void dma_fence_work_init(struct dma_fence_work *f,
+ const struct dma_fence_work_ops *ops)
+{
+ f->ops = ops;
+ f->error = 0;
+ spin_lock_init(&f->lock);
+ dma_fence_init(&f->dma, &fence_ops, &f->lock, 0, 0);
+ INIT_WORK(&f->work, dma_fence_work_work);
+ init_irq_work(&f->irq_work, dma_fence_work_irq_work);
+ refcount_set(&f->pending, 1);
+}
+
+void dma_fence_work_dependency(struct dma_fence_work *f,
+ struct dma_fence *dependency,
+ struct dma_fence_work_cb *wcb,
+ bool kfree_on_callback)
+{
+ wcb->waiter = f;
+ wcb->kfree_on_callback = kfree_on_callback;
+ refcount_inc(&f->pending);
+ if (dma_fence_add_callback(dependency, &wcb->cb, dma_fence_work_cb_func))
+ refcount_dec(&f->pending);
+}
+
+void dma_fence_work_commit(struct dma_fence_work *f)
+{
+ if (refcount_dec_and_test(&f->pending))
+ queue_work(system_unbound_wq, &f->work);
+}
+
+void dma_fence_work_commit_imm(struct dma_fence_work *f)
+{
+ if (refcount_dec_and_test(&f->pending))
+ dma_fence_work_work(&f->work);
+}
+
+static int dma_fence_work_dependency_alloc(struct dma_fence_work *f,
+ struct dma_fence *dependency,
+ unsigned long timeout,
+ gfp_t gfp)
+{
+ struct dma_fence_work_cb *wcb;
+ signed long ret = 0;
+
+ might_sleep();
+ if (dma_fence_is_signaled(dependency))
+ return dependency->error;
+
+ wcb = kmalloc(sizeof(*wcb), gfp);
+ if (!wcb) {
+ ret = dma_fence_wait_timeout(dependency, false, timeout);
+ if (ret < 0)
+ return ret;
+ else if (ret == 0)
+ return -ETIME;
+ } else {
+ dma_fence_work_dependency(f, dependency, wcb, true);
+ }
+
+ return ret;
+}
+
+int dma_fence_work_add_resv_dependencies(struct dma_fence_work *f,
+ struct dma_resv *resv,
+ unsigned long timeout,
+ gfp_t gfp,
+ bool write)
+{
+ struct dma_resv_iter iter;
+ struct dma_fence *fence;
+ int ret;
+
+ dma_resv_assert_held(resv);
+
+ dma_resv_for_each_fence(&iter, resv, write, fence) {
+ ret = dma_fence_work_dependency_alloc(f, fence, timeout, gfp);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_dma_fence_work.h b/drivers/gpu/drm/i915/i915_dma_fence_work.h
new file mode 100644
index 000000000000..6a8d51c9a261
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_dma_fence_work.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: MIT */
+
+/*
+ * Copyright © 2019-2021 Intel Corporation
+ */
+
+#ifndef I915_SW_FENCE_WORK_H
+#define I915_SW_FENCE_WORK_H
+
+#include <linux/dma-fence.h>
+#include <linux/irq_work.h>
+#include <linux/refcount.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+
+struct dma_fence_work;
+struct dma_resv;
+
+struct dma_fence_work_ops {
+ const char *name;
+ void (*work)(struct dma_fence_work *f);
+ void (*release)(struct dma_fence_work *f);
+};
+
+struct dma_fence_work {
+ struct dma_fence dma;
+ struct work_struct work;
+ struct irq_work irq_work;
+ /** Used as the dma_fence lock */
+ spinlock_t lock;
+ refcount_t pending;
+ int error;
+ const struct dma_fence_work_ops *ops;
+};
+
+struct dma_fence_work_cb {
+ struct dma_fence_cb cb;
+ struct dma_fence_work *waiter;
+ bool kfree_on_callback;
+};
+
+void dma_fence_work_init(struct dma_fence_work *f,
+ const struct dma_fence_work_ops *ops);
+
+void dma_fence_work_dependency(struct dma_fence_work *f,
+ struct dma_fence *dependency,
+ struct dma_fence_work_cb *wcb,
+ bool kfree_on_callback);
+
+void dma_fence_work_commit(struct dma_fence_work *f);
+
+void dma_fence_work_commit_imm(struct dma_fence_work *f);
+
+int dma_fence_work_add_resv_dependencies(struct dma_fence_work *f,
+ struct dma_resv *resv,
+ unsigned long timeout,
+ gfp_t gfp,
+ bool write);
+
+static inline void
+dma_fence_work_set_error_once(struct dma_fence_work *f, int error)
+{
+ if (error)
+ cmpxchg(&f->error, 0, error);
+}
+#endif /* I915_SW_FENCE_WORK_H */
diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.c b/drivers/gpu/drm/i915/i915_sw_fence_work.c
deleted file mode 100644
index 5b33ef23d54c..000000000000
--- a/drivers/gpu/drm/i915/i915_sw_fence_work.c
+++ /dev/null
@@ -1,99 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-/*
- * Copyright © 2019 Intel Corporation
- */
-
-#include "i915_sw_fence_work.h"
-
-static void fence_complete(struct dma_fence_work *f)
-{
- if (f->ops->release)
- f->ops->release(f);
- dma_fence_signal(&f->dma);
-}
-
-static void fence_work(struct work_struct *work)
-{
- struct dma_fence_work *f = container_of(work, typeof(*f), work);
-
- f->ops->work(f);
-
- fence_complete(f);
- dma_fence_put(&f->dma);
-}
-
-static int __i915_sw_fence_call
-fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
-{
- struct dma_fence_work *f = container_of(fence, typeof(*f), chain);
-
- switch (state) {
- case FENCE_COMPLETE:
- if (fence->error)
- dma_fence_set_error(&f->dma, fence->error);
-
- if (!f->dma.error) {
- dma_fence_get(&f->dma);
- if (test_bit(DMA_FENCE_WORK_IMM, &f->dma.flags))
- fence_work(&f->work);
- else
- queue_work(system_unbound_wq, &f->work);
- } else {
- fence_complete(f);
- }
- break;
-
- case FENCE_FREE:
- dma_fence_put(&f->dma);
- break;
- }
-
- return NOTIFY_DONE;
-}
-
-static const char *get_driver_name(struct dma_fence *fence)
-{
- return "dma-fence";
-}
-
-static const char *get_timeline_name(struct dma_fence *fence)
-{
- struct dma_fence_work *f = container_of(fence, typeof(*f), dma);
-
- return f->ops->name ?: "work";
-}
-
-static void fence_release(struct dma_fence *fence)
-{
- struct dma_fence_work *f = container_of(fence, typeof(*f), dma);
-
- i915_sw_fence_fini(&f->chain);
-
- BUILD_BUG_ON(offsetof(typeof(*f), dma));
- dma_fence_free(&f->dma);
-}
-
-static const struct dma_fence_ops fence_ops = {
- .get_driver_name = get_driver_name,
- .get_timeline_name = get_timeline_name,
- .release = fence_release,
-};
-
-void dma_fence_work_init(struct dma_fence_work *f,
- const struct dma_fence_work_ops *ops)
-{
- f->ops = ops;
- spin_lock_init(&f->lock);
- dma_fence_init(&f->dma, &fence_ops, &f->lock, 0, 0);
- i915_sw_fence_init(&f->chain, fence_notify);
- INIT_WORK(&f->work, fence_work);
-}
-
-int dma_fence_work_chain(struct dma_fence_work *f, struct dma_fence *signal)
-{
- if (!signal)
- return 0;
-
- return __i915_sw_fence_await_dma_fence(&f->chain, signal, &f->cb);
-}
diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.h b/drivers/gpu/drm/i915/i915_sw_fence_work.h
deleted file mode 100644
index d56806918d13..000000000000
--- a/drivers/gpu/drm/i915/i915_sw_fence_work.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-
-/*
- * Copyright © 2019 Intel Corporation
- */
-
-#ifndef I915_SW_FENCE_WORK_H
-#define I915_SW_FENCE_WORK_H
-
-#include <linux/dma-fence.h>
-#include <linux/spinlock.h>
-#include <linux/workqueue.h>
-
-#include "i915_sw_fence.h"
-
-struct dma_fence_work;
-
-struct dma_fence_work_ops {
- const char *name;
- void (*work)(struct dma_fence_work *f);
- void (*release)(struct dma_fence_work *f);
-};
-
-struct dma_fence_work {
- struct dma_fence dma;
- spinlock_t lock;
-
- struct i915_sw_fence chain;
- struct i915_sw_dma_fence_cb cb;
-
- struct work_struct work;
- const struct dma_fence_work_ops *ops;
-};
-
-enum {
- DMA_FENCE_WORK_IMM = DMA_FENCE_FLAG_USER_BITS,
-};
-
-void dma_fence_work_init(struct dma_fence_work *f,
- const struct dma_fence_work_ops *ops);
-int dma_fence_work_chain(struct dma_fence_work *f, struct dma_fence *signal);
-
-static inline void dma_fence_work_commit(struct dma_fence_work *f)
-{
- i915_sw_fence_commit(&f->chain);
-}
-
-/**
- * dma_fence_work_commit_imm: Commit the fence, and if possible execute locally.
- * @f: the fenced worker
- *
- * Instead of always scheduling a worker to execute the callback (see
- * dma_fence_work_commit()), we try to execute the callback immediately in
- * the local context. It is required that the fence be committed before it
- * is published, and that no other threads try to tamper with the number
- * of asynchronous waits on the fence (or else the callback will be
- * executed in the wrong context, i.e. not the callers).
- */
-static inline void dma_fence_work_commit_imm(struct dma_fence_work *f)
-{
- if (atomic_read(&f->chain.pending) <= 1)
- __set_bit(DMA_FENCE_WORK_IMM, &f->dma.flags);
-
- dma_fence_work_commit(f);
-}
-
-#endif /* I915_SW_FENCE_WORK_H */
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 4b7fc4647e46..4200047af178 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -34,7 +34,7 @@
#include "gt/intel_gt_requests.h"
#include "i915_drv.h"
-#include "i915_sw_fence_work.h"
+#include "i915_dma_fence_work.h"
#include "i915_trace.h"
#include "i915_vma.h"
@@ -291,7 +291,7 @@ struct i915_vma_work {
struct i915_vm_pt_stash stash;
struct i915_vma *vma;
struct drm_i915_gem_object *pinned;
- struct i915_sw_dma_fence_cb cb;
+ struct dma_fence_work_cb cb;
enum i915_cache_level cache_level;
unsigned int flags;
};
@@ -301,6 +301,11 @@ static void __vma_bind(struct dma_fence_work *work)
struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
struct i915_vma *vma = vw->vma;
+ if (work->error) {
+ dma_fence_set_error(&work->dma, work->error);
+ return;
+ }
+
vma->ops->bind_vma(vw->vm, &vw->stash,
vma, vw->cache_level, vw->flags);
}
@@ -333,7 +338,7 @@ struct i915_vma_work *i915_vma_work(void)
return NULL;
dma_fence_work_init(&vw->base, &bind_ops);
- vw->base.dma.error = -EAGAIN; /* disable the worker by default */
+ vw->base.error = -EAGAIN; /* disable the worker by default */
return vw;
}
@@ -416,16 +421,16 @@ int i915_vma_bind(struct i915_vma *vma,
* part of the obj->resv->excl_fence as it only affects
* execution and not content or object's backing store lifetime.
*/
+
+ work->base.error = 0; /* enable the queue_work() */
+
prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
if (prev) {
- __i915_sw_fence_await_dma_fence(&work->base.chain,
- prev,
- &work->cb);
+ dma_fence_work_dependency(&work->base, prev, &work->cb,
+ false);
dma_fence_put(prev);
}
- work->base.dma.error = 0; /* enable the queue_work() */
-
if (vma->obj) {
__i915_gem_object_pin_pages(vma->obj);
work->pinned = i915_gem_object_get(vma->obj);
--
2.31.1
More information about the Intel-gfx-trybot
mailing list