[PATCH 2/2] async: Add support for explicit fine-grained barriers
Chris Wilson
chris at chris-wilson.co.uk
Tue May 24 10:30:31 UTC 2016
The current async-domain model supports running a multitude of
independent tasks with a coarse synchronisation point. This is
sufficient for its original purpose of allowing independent drivers to
run concurrently during various phases (booting, early resume, late
resume etc). However, for greater exploitation, drivers themselves want
to schedule multiple tasks within a phase (or between phases) and
control the order of execution within those tasks relative to each
other. To enable this, we extend the synchronisation scheme to support
explicit barriers called a fence, which act as a semaphore. A fence can
be placed into the async-domain with many dependencies and will only be
passed when all of those dependencies are met. This allows us to build
N:M barriers to precisely control execution between many interoperating
tasks. Each task is itself a fence, useful for coordinating sequential
execution and supporting the current coarse-grained barrier.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Ignore-Cc: Daniel Vetter <daniel.vetter at ffwll.ch>
Ignore-Cc: Lukas Wunner <lukas at wunner.de>
Ignore-Cc: Tejun Heo <tj at kernel.org>
Ignore-CC: Dan Williams <dan.j.williams at intel.com>
Ignore-Cc: Andrew Morton <akpm at linux-foundation.org>
Ignore-Cc: Ross Zwisler <ross.zwisler at linux.intel.com>
Ignore-Cc: Andrey Ryabinin <aryabinin at virtuozzo.com>
Ignore-Cc: Oleg Nesterov <oleg at redhat.com>
Ignore-CC: Johannes Berg <johannes.berg at intel.com>
Ignore-Cc: Robert Jarzmik <robert.jarzmik at free.fr>
Ignore-Cc: Alexander Potapenko <glider at google.com>
Ignore-Cc: Geert Uytterhoeven <geert at linux-m68k.org>
Ignore-Cc: "David S. Miller" <davem at davemloft.net>
Ignore-CC: Rasmus Villemoes <linux at rasmusvillemoes.dk>
Ignore-Cc: Peter Zijlstra <peterz at infradead.org>
Ignore-Cc: Kees Cook <keescook at chromium.org>
Ignore-Cc: Valentin Rothberg <valentinrothberg at gmail.com>
Ignore-Cc: Jason Baron <jbaron at akamai.com>
Ignore-Cc: Dmitry Vyukov <dvyukov at google.com>
Ignore-Cc: Ingo Molnar <mingo at kernel.org>
Ignore-Cc: Hannes Frederic Sowa <hannes at stressinduktion.org>
Ignore-Cc: Chris Metcalf <cmetcalf at ezchip.com>
Ignore-Cc: Andy Shevchenko <andriy.shevchenko at linux.intel.com>
Ignore-Cc: David Decotigny <decot at googlers.com>
Ignore-Cc: linux-kernel at vger.kernel.org
Ignore-Cc: linux-kselftest at vger.kernel.org
---
include/linux/async.h | 82 +++++++++-
kernel/async.c | 406 +++++++++++++++++++++++++++++++++++++-----------
lib/test-async-domain.c | 297 +++++++++++++++++++++++++++++++++++
3 files changed, 694 insertions(+), 91 deletions(-)
diff --git a/include/linux/async.h b/include/linux/async.h
index 6b0226bdaadc..24df3274f318 100644
--- a/include/linux/async.h
+++ b/include/linux/async.h
@@ -13,29 +13,48 @@
#define __ASYNC_H__
#include <linux/types.h>
+#include <linux/gfp.h>
+#include <linux/kref.h>
#include <linux/list.h>
typedef u64 async_cookie_t;
typedef void (*async_func_t) (void *data, async_cookie_t cookie);
+
+struct async_fence {
+ wait_queue_head_t wait;
+ unsigned long flags;
+ struct kref kref;
+ atomic_t pending;
+};
+
+struct async_work {
+ struct async_fence fence;
+ /* private */
+};
+
struct async_domain {
struct list_head pending;
+ struct async_fence *barrier;
unsigned registered:1;
};
+#define ASYNC_DOMAIN_INIT(_name, _r) { \
+ .pending = LIST_HEAD_INIT(_name.pending), \
+ .registered = _r \
+}
+
/*
* domain participates in global async_synchronize_full
*/
#define ASYNC_DOMAIN(_name) \
- struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \
- .registered = 1 }
+ struct async_domain _name = ASYNC_DOMAIN_INIT(_name, 1)
/*
* domain is free to go out of scope as soon as all pending work is
* complete, this domain does not participate in async_synchronize_full
*/
#define ASYNC_DOMAIN_EXCLUSIVE(_name) \
- struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \
- .registered = 0 }
+ struct async_domain _name = ASYNC_DOMAIN_INIT(_name, 0)
extern async_cookie_t async_schedule(async_func_t func, void *data);
extern async_cookie_t async_schedule_domain(async_func_t func, void *data,
@@ -46,5 +65,60 @@ extern void async_synchronize_full_domain(struct async_domain *domain);
extern void async_synchronize_cookie(async_cookie_t cookie);
extern void async_synchronize_cookie_domain(async_cookie_t cookie,
struct async_domain *domain);
+
+extern void async_barrier(void);
+extern void async_barrier_domain(struct async_domain *domain);
+
extern bool current_is_async(void);
+
+
+struct async_fence *async_fence_create(gfp_t gfp);
+struct async_fence *async_fence_get(struct async_fence *fence);
+int async_fence_add(struct async_fence *fence,
+ struct async_fence *after,
+ gfp_t gfp);
+void async_fence_pending(struct async_fence *fence);
+void async_fence_signal(struct async_fence *fence);
+void async_fence_wait(struct async_fence *fence);
+static inline bool async_fence_complete(struct async_fence *fence)
+{
+ return atomic_read(&fence->pending) < 0;
+}
+void async_fence_put(struct async_fence *fence);
+
+struct async_work *async_work_create(async_func_t func, void *data, gfp_t gfp);
+
+static inline struct async_work *async_work_get(struct async_work *work)
+{
+ async_fence_get(&work->fence);
+ return work;
+}
+
+static inline async_cookie_t
+async_work_after(struct async_work *work, struct async_fence *fence)
+{
+ return async_fence_add(&work->fence, fence, GFP_KERNEL);
+}
+
+static inline async_cookie_t
+async_work_before(struct async_work *work, struct async_fence *fence)
+{
+ return async_fence_add(fence, &work->fence, GFP_KERNEL);
+}
+
+static inline void async_work_wait(struct async_work *work)
+{
+ async_fence_wait(&work->fence);
+}
+
+static inline void async_work_put(struct async_work *work)
+{
+ async_fence_put(&work->fence);
+}
+
+async_cookie_t queue_async_work(struct async_domain *domain,
+ struct async_work *work,
+ gfp_t gfp);
+async_cookie_t schedule_async_work(struct async_work *work);
+
#endif
diff --git a/kernel/async.c b/kernel/async.c
index d2edd6efec56..ba9d78a912b9 100644
--- a/kernel/async.c
+++ b/kernel/async.c
@@ -2,6 +2,7 @@
* async.c: Asynchronous function calls for boot performance
*
* (C) Copyright 2009 Intel Corporation
+ * (C) Copyright 2016 Intel Corporation
* Author: Arjan van de Ven <arjan at linux.intel.com>
*
* This program is free software; you can redistribute it and/or
@@ -59,59 +60,192 @@ asynchronous and synchronous parts of the kernel.
#include "workqueue_internal.h"
-static async_cookie_t next_cookie = 1;
+#define MAX_WORK 32768
-#define MAX_WORK 32768
-#define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
+struct async_entry {
+ struct async_work base;
+ struct work_struct work;
+
+ struct list_head pending_link[2];
+
+ async_cookie_t cookie;
+ async_func_t func;
+ void *data;
+};
+
+#define ASYNC_WORK_BIT 1
+#define ASYNC_QUEUED_BIT 2
static LIST_HEAD(async_global_pending); /* pending from all registered doms */
static ASYNC_DOMAIN(async_dfl_domain);
static DEFINE_SPINLOCK(async_lock);
+static unsigned async_pending_count;
-struct async_entry {
- struct list_head domain_list;
- struct list_head global_list;
- struct work_struct work;
- async_cookie_t cookie;
- async_func_t func;
- void *data;
- struct async_domain *domain;
-};
+static async_cookie_t assign_cookie(void)
+{
+ static async_cookie_t next_cookie;
+ if (++next_cookie == 0)
+ next_cookie = 1;
+ return next_cookie;
+}
+
+static void async_fence_free(struct kref *kref)
+{
+ struct async_fence *fence = container_of(kref, typeof(*fence), kref);
+
+ WARN_ON(atomic_read(&fence->pending) > 0);
-static DECLARE_WAIT_QUEUE_HEAD(async_done);
+ kfree(fence);
+}
-static atomic_t entry_count;
+void async_fence_put(struct async_fence *fence)
+{
+ if (fence)
+ kref_put(&fence->kref, async_fence_free);
+}
+EXPORT_SYMBOL_GPL(async_fence_put);
-static async_cookie_t lowest_in_progress(struct async_domain *domain)
+struct async_fence *async_fence_get(struct async_fence *fence)
{
- struct list_head *pending;
- async_cookie_t ret = ASYNC_COOKIE_MAX;
+ if (fence)
+ kref_get(&fence->kref);
+ return fence;
+}
+EXPORT_SYMBOL_GPL(async_fence_get);
+
+static void async_fence_execute(struct async_fence *fence)
+{
+ atomic_dec(&fence->pending);
+
+ if (test_bit(ASYNC_WORK_BIT, &fence->flags)) {
+ struct async_entry *entry =
+ container_of(fence, typeof(*entry), base.fence);
+ queue_work(system_unbound_wq, &entry->work);
+ return;
+ }
+
+ wake_up_all(&fence->wait);
+}
+
+void async_fence_pending(struct async_fence *fence)
+{
+ WARN_ON(atomic_inc_return(&fence->pending) <= 1);
+}
+EXPORT_SYMBOL_GPL(async_fence_pending);
+
+void async_fence_signal(struct async_fence *fence)
+{
+ if (atomic_dec_and_test(&fence->pending))
+ async_fence_execute(fence);
+}
+EXPORT_SYMBOL_GPL(async_fence_signal);
+
+void async_fence_wait(struct async_fence *fence)
+{
+ wait_event(fence->wait, async_fence_complete(fence));
+}
+EXPORT_SYMBOL_GPL(async_fence_wait);
+
+static void async_fence_init(struct async_fence *fence)
+{
+ kref_init(&fence->kref);
+ atomic_set(&fence->pending, 1);
+ fence->flags = 0;
+}
+
+struct async_fence *async_fence_create(gfp_t gfp)
+{
+ struct async_fence *fence;
+
+ fence = kmalloc(sizeof(*fence), gfp);
+ if (!fence)
+ return NULL;
+
+ async_fence_init(fence);
+ return fence;
+}
+EXPORT_SYMBOL_GPL(async_fence_create);
+
+static int async_fence_wake(wait_queue_t *wq,
+ unsigned mode, int flags, void *key)
+{
+ list_del(&wq->task_list);
+ async_fence_signal(wq->private);
+ async_fence_put(wq->private);
+ kfree(wq);
+ return 0;
+}
+
+static bool async_check_not_after(struct async_fence *fence,
+ struct async_fence *after)
+{
+ wait_queue_t *wq;
+
+ if (fence == NULL)
+ return false;
+
+ if (fence == after)
+ return true;
+
+ list_for_each_entry(wq, &after->wait.task_list, task_list) {
+ if (wq->func != async_fence_wake)
+ continue;
+
+ if (async_check_not_after(wq->private, after))
+ return true;
+ }
+
+ return false;
+}
+
+int async_fence_add(struct async_fence *fence,
+ struct async_fence *after,
+ gfp_t gfp)
+{
+ wait_queue_t *wq;
unsigned long flags;
+ int pending;
- spin_lock_irqsave(&async_lock, flags);
+ /* The dependency graph must be acyclic */
+ if (WARN_ON(async_check_not_after(after, fence)))
+ return -EINVAL;
- if (domain)
- pending = &domain->pending;
- else
- pending = &async_global_pending;
+ if (!after || async_fence_complete(after))
+ return 0;
- if (!list_empty(pending))
- ret = list_first_entry(pending, struct async_entry,
- domain_list)->cookie;
+ wq = kmalloc(sizeof(*wq), gfp);
+ if (!wq) {
+ if (WARN_ON(!gfpflags_allow_blocking(gfp)))
+ return -ENOMEM;
- spin_unlock_irqrestore(&async_lock, flags);
- return ret;
+ async_fence_wait(after);
+ return 0;
+ }
+
+ wq->flags = 0;
+ wq->func = async_fence_wake;
+ wq->private = async_fence_get(fence);
+
+ async_fence_pending(fence);
+
+ spin_lock_irqsave(&after->wait.lock, flags);
+ if (likely(!async_fence_complete(after))) {
+ __add_wait_queue_tail(&after->wait, wq);
+ pending = 1;
+ } else {
+ async_fence_wake(wq, 0, 0, NULL);
+ pending = 0;
+ }
+ spin_unlock_irqrestore(&after->wait.lock, flags);
+
+ return pending;
}
+EXPORT_SYMBOL_GPL(async_fence_add);
-/*
- * pick the first pending entry and run it
- */
static void async_run_entry_fn(struct work_struct *work)
{
- struct async_entry *entry =
- container_of(work, struct async_entry, work);
- unsigned long flags;
- ktime_t uninitialized_var(calltime), delta, rettime;
+ struct async_entry *entry = container_of(work, typeof(*entry), work);
+ ktime_t uninitialized_var(calltime);
/* 1) run (and print duration) */
if (initcall_debug && system_state == SYSTEM_BOOTING) {
@@ -122,8 +256,7 @@ static void async_run_entry_fn(struct work_struct *work)
}
entry->func(entry->data, entry->cookie);
if (initcall_debug && system_state == SYSTEM_BOOTING) {
- rettime = ktime_get();
- delta = ktime_sub(rettime, calltime);
+ ktime_t delta = ktime_sub(ktime_get(), calltime);
pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n",
(long long)entry->cookie,
entry->func,
@@ -131,69 +264,78 @@ static void async_run_entry_fn(struct work_struct *work)
}
/* 2) remove self from the pending queues */
- spin_lock_irqsave(&async_lock, flags);
- list_del_init(&entry->domain_list);
- list_del_init(&entry->global_list);
-
- /* 3) free the entry */
- kfree(entry);
- atomic_dec(&entry_count);
-
- spin_unlock_irqrestore(&async_lock, flags);
+ spin_lock_irq(&async_lock);
+ list_del(&entry->pending_link[0]);
+ list_del(&entry->pending_link[1]);
+ async_pending_count--;
+ spin_unlock_irq(&async_lock);
- /* 4) wake up any waiters */
- wake_up(&async_done);
+ /* 3) wake up any waiters */
+ wake_up_all(&entry->base.fence.wait);
+ async_fence_put(&entry->base.fence);
}
-static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain)
+struct async_work *async_work_create(async_func_t func, void *data, gfp_t gfp)
{
struct async_entry *entry;
- unsigned long flags;
- async_cookie_t newcookie;
- /* allow irq-off callers */
- entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
+ entry = kmalloc(sizeof(*entry), gfp);
+ if (!entry)
+ return NULL;
- /*
- * If we're out of memory or if there's too much work
- * pending already, we execute synchronously.
- */
- if (!entry || atomic_read(&entry_count) > MAX_WORK) {
- kfree(entry);
- spin_lock_irqsave(&async_lock, flags);
- newcookie = next_cookie++;
- spin_unlock_irqrestore(&async_lock, flags);
+ async_fence_init(&entry->base.fence);
- /* low on memory.. run synchronously */
- func(data, newcookie);
- return newcookie;
- }
- INIT_LIST_HEAD(&entry->domain_list);
- INIT_LIST_HEAD(&entry->global_list);
INIT_WORK(&entry->work, async_run_entry_fn);
entry->func = func;
entry->data = data;
- entry->domain = domain;
- spin_lock_irqsave(&async_lock, flags);
+ set_bit(ASYNC_WORK_BIT, &entry->base.fence.flags);
- /* allocate cookie and queue */
- newcookie = entry->cookie = next_cookie++;
+ return &entry->base;
+}
+EXPORT_SYMBOL_GPL(async_work_create);
- list_add_tail(&entry->domain_list, &domain->pending);
- if (domain->registered)
- list_add_tail(&entry->global_list, &async_global_pending);
+static void async_barrier_delete(struct async_domain *domain)
+{
+ async_fence_put(domain->barrier);
+ domain->barrier = NULL;
+}
+
+async_cookie_t queue_async_work(struct async_domain *domain,
+ struct async_work *work,
+ gfp_t gfp)
+{
+ struct async_entry *entry = container_of(work, typeof(*entry), base);
+ unsigned long flags;
- atomic_inc(&entry_count);
+ if (WARN_ON(test_and_set_bit(ASYNC_QUEUED_BIT,
+ &entry->base.fence.flags)))
+ return 0;
+
+ spin_lock_irqsave(&async_lock, flags);
+ entry->cookie = assign_cookie();
+ list_add_tail(&entry->pending_link[0], &domain->pending);
+ INIT_LIST_HEAD(&entry->pending_link[1]);
+ if (domain->registered)
+ list_add_tail(&entry->pending_link[1], &async_global_pending);
+ async_pending_count++;
spin_unlock_irqrestore(&async_lock, flags);
+ if (!async_fence_add(&entry->base.fence, domain->barrier, gfp))
+ async_barrier_delete(domain);
+
/* mark that this task has queued an async job, used by module init */
current->flags |= PF_USED_ASYNC;
- /* schedule for execution */
- queue_work(system_unbound_wq, &entry->work);
+ async_fence_signal(async_fence_get(&entry->base.fence));
- return newcookie;
+ return entry->cookie;
+}
+EXPORT_SYMBOL_GPL(queue_async_work);
+
+async_cookie_t schedule_async_work(struct async_work *work)
+{
+ return queue_async_work(&async_dfl_domain, work, GFP_KERNEL);
}
/**
@@ -206,7 +348,7 @@ static async_cookie_t __async_schedule(async_func_t func, void *data, struct asy
*/
async_cookie_t async_schedule(async_func_t func, void *data)
{
- return __async_schedule(func, data, &async_dfl_domain);
+ return async_schedule_domain(func, data, &async_dfl_domain);
}
EXPORT_SYMBOL_GPL(async_schedule);
@@ -225,10 +367,68 @@ EXPORT_SYMBOL_GPL(async_schedule);
async_cookie_t async_schedule_domain(async_func_t func, void *data,
struct async_domain *domain)
{
- return __async_schedule(func, data, domain);
+ struct async_work *work;
+ async_cookie_t cookie = 0;
+
+ work = NULL;
+ if (READ_ONCE(async_pending_count) < MAX_WORK)
+ work = async_work_create(func, data, GFP_ATOMIC);
+ if (work) {
+ cookie = queue_async_work(domain, work, GFP_ATOMIC);
+ async_work_put(work);
+ }
+ if (!cookie) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&async_lock, flags);
+ cookie = assign_cookie();
+ spin_unlock_irqrestore(&async_lock, flags);
+
+ func(data, cookie);
+ }
+
+ return cookie;
}
EXPORT_SYMBOL_GPL(async_schedule_domain);
+static struct async_fence *
+__async_barrier_create(struct async_domain *domain, gfp_t gfp)
+{
+ struct async_fence *fence;
+ struct async_entry *entry;
+ unsigned long flags;
+
+ fence = async_fence_create(gfp);
+ if (!fence)
+ return NULL;
+
+ async_fence_add(fence, domain->barrier, gfp);
+
+ spin_lock_irqsave(&async_lock, flags);
+ list_for_each_entry(entry, &domain->pending, pending_link[0])
+ async_fence_add(fence, &entry->base.fence, gfp);
+ spin_unlock_irqrestore(&async_lock, flags);
+
+ async_fence_signal(fence);
+
+ return fence;
+}
+
+void async_barrier(void)
+{
+ async_barrier_domain(&async_dfl_domain);
+}
+EXPORT_SYMBOL_GPL(async_barrier);
+
+void async_barrier_domain(struct async_domain *domain)
+{
+ async_fence_put(domain->barrier);
+ domain->barrier = __async_barrier_create(domain, GFP_ATOMIC);
+ if (!domain->barrier)
+ async_synchronize_full_domain(domain);
+}
+EXPORT_SYMBOL_GPL(async_barrier_domain);
+
/**
* async_synchronize_full - synchronize all asynchronous function calls
*
@@ -253,8 +453,10 @@ void async_unregister_domain(struct async_domain *domain)
{
spin_lock_irq(&async_lock);
WARN_ON(!domain->registered || !list_empty(&domain->pending));
- domain->registered = 0;
spin_unlock_irq(&async_lock);
+
+ async_barrier_delete(domain);
+ domain->registered = 0;
}
EXPORT_SYMBOL_GPL(async_unregister_domain);
@@ -267,7 +469,7 @@ EXPORT_SYMBOL_GPL(async_unregister_domain);
*/
void async_synchronize_full_domain(struct async_domain *domain)
{
- async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
+ async_synchronize_cookie_domain(0, domain);
}
EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
@@ -282,19 +484,49 @@ EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
*/
void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
{
- ktime_t uninitialized_var(starttime), delta, endtime;
+ ktime_t uninitialized_var(starttime);
+ struct list_head *pending;
+
+ pending = domain ? &domain->pending : &async_global_pending;
if (initcall_debug && system_state == SYSTEM_BOOTING) {
pr_debug("async_waiting @ %i\n", task_pid_nr(current));
starttime = ktime_get();
}
- wait_event(async_done, lowest_in_progress(domain) >= cookie);
+ do {
+ struct async_fence *fence = NULL;
+ unsigned long flags;
- if (initcall_debug && system_state == SYSTEM_BOOTING) {
- endtime = ktime_get();
- delta = ktime_sub(endtime, starttime);
+ spin_lock_irqsave(&async_lock, flags);
+ if (!list_empty(pending)) {
+ struct async_entry *entry;
+
+ if (cookie) {
+ entry = list_first_entry(pending,
+ struct async_entry,
+ pending_link[!domain]);
+ if ((s64)(cookie - entry->cookie) > 0)
+ fence = async_fence_get(&entry->base.fence);
+ } else {
+ entry = list_last_entry(pending,
+ struct async_entry,
+ pending_link[!domain]);
+ cookie = entry->cookie;
+ fence = async_fence_get(&entry->base.fence);
+ }
+ }
+ spin_unlock_irqrestore(&async_lock, flags);
+ if (!fence)
+ break;
+
+ async_fence_wait(fence);
+ async_fence_put(fence);
+ } while (1);
+
+ if (initcall_debug && system_state == SYSTEM_BOOTING) {
+ ktime_t delta = ktime_sub(ktime_get(), starttime);
pr_debug("async_continuing @ %i after %lli usec\n",
task_pid_nr(current),
(long long)ktime_to_ns(delta) >> 10);
diff --git a/lib/test-async-domain.c b/lib/test-async-domain.c
index 558a71414fb6..c810abc1b0af 100644
--- a/lib/test-async-domain.c
+++ b/lib/test-async-domain.c
@@ -21,6 +21,267 @@ static void task_B(void *data, async_cookie_t cookie)
smp_store_mb(*result, 'B');
}
+static int __init test_x(const char *name,
+ struct async_domain *domain,
+ async_func_t func,
+ const long expected)
+{
+ struct async_work *A;
+ long result = 0;
+
+ A = async_work_create(func, &result, GFP_KERNEL);
+ if (!A)
+ return -ENOMEM;
+
+ queue_async_work(domain, A, GFP_KERNEL);
+ async_work_wait(A);
+ async_work_put(A);
+
+ if (READ_ONCE(result) != expected) {
+ pr_warn("%s expected %c [%ld], got %ld\n",
+ name, (char)expected, expected, result);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __init test_A(struct async_domain *domain)
+{
+ return test_x(__func__, domain, task_A, 'A');
+}
+
+static int __init test_B(struct async_domain *domain)
+{
+ return test_x(__func__, domain, task_B, 'B');
+}
+
+static int __init test_x_fence(const char *name,
+ struct async_domain *domain,
+ async_func_t func,
+ const long expected)
+{
+ struct async_work *A;
+ struct async_fence *fence;
+ long result = 0;
+
+ A = async_work_create(func, &result, GFP_KERNEL);
+ if (!A)
+ return -ENOMEM;
+
+ fence = async_fence_create(GFP_KERNEL);
+ if (!fence)
+ return -ENOMEM;
+
+ queue_async_work(domain, A, GFP_KERNEL);
+
+ async_fence_add(fence, &A->fence, GFP_KERNEL);
+ async_fence_signal(fence);
+
+ async_fence_wait(fence);
+
+ async_work_put(A);
+ async_fence_put(fence);
+
+ if (READ_ONCE(result) != expected) {
+ pr_warn("%s expected %c [%ld], got %ld\n",
+ name, (char)expected, expected, result);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __init test_A_fence(struct async_domain *domain)
+{
+ return test_x_fence(__func__, domain, task_A, 'A');
+}
+
+static int __init test_B_fence(struct async_domain *domain)
+{
+ return test_x_fence(__func__, domain, task_B, 'B');
+}
+
+static int __init test_x_fence_y(const char *name,
+ struct async_domain *domain,
+ async_func_t x,
+ async_func_t y,
+ const long expected)
+{
+ struct async_work *A, *B;
+ struct async_fence *fence;
+ long result = 0;
+
+ A = async_work_create(x, &result, GFP_KERNEL);
+ if (!A)
+ return -ENOMEM;
+
+ B = async_work_create(y, &result, GFP_KERNEL);
+ if (!B)
+ return -ENOMEM;
+
+ fence = async_fence_create(GFP_KERNEL);
+ if (!fence)
+ return -ENOMEM;
+
+ async_fence_add(fence, &A->fence, GFP_KERNEL);
+ async_fence_signal(fence);
+
+ queue_async_work(domain, A, GFP_KERNEL);
+ async_work_put(A);
+
+ async_work_after(B, fence);
+ queue_async_work(domain, B, GFP_KERNEL);
+ async_fence_put(fence);
+
+ async_work_wait(B);
+ async_work_put(B);
+
+ if (READ_ONCE(result) != expected) {
+ pr_warn("%s expected %c [%ld], got %ld\n",
+ name, (char)expected, expected, result);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __init test_A_fence_B(struct async_domain *domain)
+{
+ return test_x_fence_y(__func__, domain, task_A, task_B, 'B');
+}
+
+static int __init test_B_fence_A(struct async_domain *domain)
+{
+ return test_x_fence_y(__func__, domain, task_B, task_A, 'A');
+}
+
+struct long_context {
+ struct async_fence *barrier;
+ long *src;
+ long result;
+};
+
+static void task_wait(void *data, async_cookie_t cookie)
+{
+ struct long_context *ctx = data;
+
+ async_fence_wait(ctx->barrier);
+ smp_store_mb(ctx->result, READ_ONCE(*ctx->src));
+}
+
+static int __init test_pause(struct async_domain *domain)
+{
+ struct long_context ctx;
+ struct async_work *A, *B;
+ const long expected = 'B';
+ long out_B = 'A';
+
+ ctx.result = 0;
+ ctx.src = &out_B;
+
+ A = async_work_create(task_wait, &ctx, GFP_KERNEL);
+ if (!A)
+ return -ENOMEM;
+
+ B = async_work_create(task_B, &out_B, GFP_KERNEL);
+ if (!B)
+ return -ENOMEM;
+
+ ctx.barrier = &B->fence;
+
+ queue_async_work(domain, A, GFP_KERNEL);
+ queue_async_work(domain, B, GFP_KERNEL);
+ async_work_put(B);
+
+ async_work_wait(A);
+ async_work_put(A);
+
+ if (READ_ONCE(ctx.result) != expected) {
+ pr_warn("%s expected %c [%ld], got %ld\n",
+ __func__, (char)expected, expected, ctx.result);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void task_signal(void *data, async_cookie_t cookie)
+{
+ struct long_context *ctx = data;
+
+ async_fence_signal(ctx->barrier);
+}
+
+static int __init test_manual(struct async_domain *domain)
+{
+ struct long_context ctx;
+ struct async_work *A, *B, *C;
+ const long expected = 'B';
+ long out_B = 'A';
+
+ ctx.result = 0;
+ ctx.src = &out_B;
+ ctx.barrier = async_fence_create(GFP_KERNEL);
+
+ A = async_work_create(task_wait, &ctx, GFP_KERNEL);
+ if (!A)
+ return -ENOMEM;
+
+ B = async_work_create(task_B, &out_B, GFP_KERNEL);
+ if (!B)
+ return -ENOMEM;
+
+ C = async_work_create(task_signal, &ctx, GFP_KERNEL);
+ if (!B)
+ return -ENOMEM;
+
+ async_work_after(C, &B->fence);
+
+ queue_async_work(domain, A, GFP_KERNEL);
+ queue_async_work(domain, B, GFP_KERNEL);
+ queue_async_work(domain, C, GFP_KERNEL);
+
+ async_work_wait(A);
+
+ async_work_put(C);
+ async_work_put(B);
+ async_work_put(A);
+ async_fence_put(ctx.barrier);
+
+ if (READ_ONCE(ctx.result) != expected) {
+ pr_warn("%s expected %c [%ld], got %ld\n",
+ __func__, (char)expected, expected, ctx.result);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __init test_sync(struct async_domain *domain)
+{
+ struct async_work *B;
+ const long expected = 'B';
+ long result = 0;
+
+ B = async_work_create(task_B, &result, GFP_KERNEL);
+ if (!B)
+ return -ENOMEM;
+
+ queue_async_work(domain, B, GFP_KERNEL);
+ async_work_put(B);
+
+ async_synchronize_full_domain(domain);
+
+ if (READ_ONCE(result) != expected) {
+ pr_warn("%s expected %c [%ld], got %ld\n",
+ __func__, (char)expected, expected, result);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int __init test_implicit(struct async_domain *domain)
{
const long expected = 'B';
@@ -99,6 +360,42 @@ static int __init test_async_domain_init(void)
pr_info("Testing async-domains\n");
+ ret = test_A(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_A_fence(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_A_fence_B(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_B(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_B_fence(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_B_fence_A(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_pause(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_manual(&domain);
+ if (ret)
+ return ret;
+
+ ret = test_sync(&domain);
+ if (ret)
+ return ret;
+
ret = test_implicit(&domain);
if (ret)
return ret;
--
2.8.1
More information about the Intel-gfx-trybot
mailing list