[Mesa-dev] [PATCH] mesa: Add new fast mtx_t mutex type for basic use cases
Kristian Høgsberg
krh at bitplanet.net
Tue Jan 27 21:08:41 PST 2015
While modern pthread mutexes are very fast, they still incur a call to an
external DSO and overhead of the generality and features of pthread mutexes.
Most mutexes in mesa only needs lock/unlock, and the idea here is that we can
inline the atomic operation and make the fast case just two intructions.
Mutexes are subtle and finicky to implement, so we carefully copy the
implementation from Ulrich Dreppers well-written and well-reviewed paper:
"Futexes Are Tricky"
http://www.akkadia.org/drepper/futex.pdf
We implement "mutex3", which gives us a mutex that has no syscalls on
uncontended lock or unlock. Further, the uncontended case boils down to a
cmpxchg and an untaken branch and the uncontended unlock is just a locked decr
and an untaken branch. We use __builtin_expect() to indicate that contention
is unlikely so that gcc will put the contention code out of the main code
flow.
A fast mutex only supports lock/unlock, can't be recursive or used with
condition variables. We keep the pthread mutex implementation around as
full_mtx_t for the few places where we use condition variables or recursive
locking. For platforms or compilers where futex and atomics aren't available,
mtx_t falls back to the pthread mutex.
The pthread mutex lock/unlock overhead shows up on benchmarks for CPU bound
applications. Most CPU bound cases are helped and some of our internal
bind_buffer_object heavy benchmarks gain up to 10%.
Signed-off-by: Kristian Høgsberg <krh at bitplanet.net>
---
include/c11/threads_posix.h | 145 ++++++++++++++++++++++++++++++++---
include/c11/threads_win32.h | 25 ++++++
src/gallium/auxiliary/os/os_thread.h | 10 +--
src/mesa/main/mtypes.h | 4 +-
src/mesa/main/shared.c | 4 +-
src/mesa/main/texobj.c | 4 +-
src/mesa/main/texobj.h | 4 +-
7 files changed, 171 insertions(+), 25 deletions(-)
diff --git a/include/c11/threads_posix.h b/include/c11/threads_posix.h
index f9c165d..3da4742 100644
--- a/include/c11/threads_posix.h
+++ b/include/c11/threads_posix.h
@@ -59,13 +59,15 @@ Configuration macro:
#endif
// FIXME: temporary non-standard hack to ease transition
-#define _MTX_INITIALIZER_NP PTHREAD_MUTEX_INITIALIZER
+#define _FULL_MTX_INITIALIZER_NP PTHREAD_MUTEX_INITIALIZER
+
+#define _MTX_INITIALIZER_NP { 0 }
/*---------------------------- types ----------------------------*/
typedef pthread_cond_t cnd_t;
typedef pthread_t thrd_t;
typedef pthread_key_t tss_t;
-typedef pthread_mutex_t mtx_t;
+typedef pthread_mutex_t full_mtx_t;
typedef pthread_once_t once_flag;
@@ -135,7 +137,7 @@ cnd_signal(cnd_t *cond)
// 7.25.3.5
static inline int
-cnd_timedwait(cnd_t *cond, mtx_t *mtx, const xtime *xt)
+cnd_timedwait(cnd_t *cond, full_mtx_t *mtx, const xtime *xt)
{
struct timespec abs_time;
int rt;
@@ -148,7 +150,7 @@ cnd_timedwait(cnd_t *cond, mtx_t *mtx, const xtime *xt)
// 7.25.3.6
static inline int
-cnd_wait(cnd_t *cond, mtx_t *mtx)
+cnd_wait(cnd_t *cond, full_mtx_t *mtx)
{
if (!cond || !mtx) return thrd_error;
pthread_cond_wait(cond, mtx);
@@ -159,7 +161,7 @@ cnd_wait(cnd_t *cond, mtx_t *mtx)
/*-------------------- 7.25.4 Mutex functions --------------------*/
// 7.25.4.1
static inline void
-mtx_destroy(mtx_t *mtx)
+full_mtx_destroy(full_mtx_t *mtx)
{
assert(mtx);
pthread_mutex_destroy(mtx);
@@ -167,7 +169,7 @@ mtx_destroy(mtx_t *mtx)
// 7.25.4.2
static inline int
-mtx_init(mtx_t *mtx, int type)
+full_mtx_init(full_mtx_t *mtx, int type)
{
pthread_mutexattr_t attr;
if (!mtx) return thrd_error;
@@ -191,7 +193,7 @@ mtx_init(mtx_t *mtx, int type)
// 7.25.4.3
static inline int
-mtx_lock(mtx_t *mtx)
+full_mtx_lock(full_mtx_t *mtx)
{
if (!mtx) return thrd_error;
pthread_mutex_lock(mtx);
@@ -199,14 +201,14 @@ mtx_lock(mtx_t *mtx)
}
static inline int
-mtx_trylock(mtx_t *mtx);
+full_mtx_trylock(full_mtx_t *mtx);
static inline void
thrd_yield(void);
// 7.25.4.4
static inline int
-mtx_timedlock(mtx_t *mtx, const xtime *xt)
+full_mtx_timedlock(full_mtx_t *mtx, const xtime *xt)
{
if (!mtx || !xt) return thrd_error;
{
@@ -222,7 +224,7 @@ mtx_timedlock(mtx_t *mtx, const xtime *xt)
#else
time_t expire = time(NULL);
expire += xt->sec;
- while (mtx_trylock(mtx) != thrd_success) {
+ while (full_mtx_trylock(mtx) != thrd_success) {
time_t now = time(NULL);
if (expire < now)
return thrd_busy;
@@ -236,7 +238,7 @@ mtx_timedlock(mtx_t *mtx, const xtime *xt)
// 7.25.4.5
static inline int
-mtx_trylock(mtx_t *mtx)
+full_mtx_trylock(full_mtx_t *mtx)
{
if (!mtx) return thrd_error;
return (pthread_mutex_trylock(mtx) == 0) ? thrd_success : thrd_busy;
@@ -244,13 +246,132 @@ mtx_trylock(mtx_t *mtx)
// 7.25.4.6
static inline int
-mtx_unlock(mtx_t *mtx)
+full_mtx_unlock(full_mtx_t *mtx)
{
if (!mtx) return thrd_error;
pthread_mutex_unlock(mtx);
return thrd_success;
}
+#if defined(__GNUC__)
+
+/* mtx_t - Fast, simple mutex
+ *
+ * While modern pthread mutexes are very fast (implemented using futex), they
+ * still incur a call to an external DSO and overhead of the generality and
+ * features of pthread mutexes. Most mutexes in mesa only needs lock/unlock,
+ * and the idea here is that we can inline the atomic operation and make the
+ * fast case just two intructions. Mutexes are subtle and finicky to
+ * implement, so we carefully copy the implementation from Ulrich Dreppers
+ * well-written and well-reviewed paper:
+ *
+ * "Futexes Are Tricky"
+ * http://www.akkadia.org/drepper/futex.pdf
+ *
+ * We implement "mutex3", which gives us a mutex that has no syscalls on
+ * uncontended lock or unlock. Further, the uncontended case boils down to a
+ * locked cmpxchg and an untaken branch, the uncontended unlock is just a
+ * locked decr and an untaken branch. We use __builtin_expect() to indicate
+ * that contention is unlikely so that gcc will put the contention code out of
+ * the main code flow.
+ *
+ * A fast mutex only supports lock/unlock, can't be recursive or used with
+ * condition variables.
+ */
+
+typedef struct {
+ uint32_t val;
+} mtx_t;
+
+#include <stdint.h>
+#include <values.h>
+#include <linux/futex.h>
+#include <sys/time.h>
+#include <sys/syscall.h>
+
+static inline long sys_futex(void *addr1, int op, int val1, struct timespec *timeout, void *addr2, int val3)
+{
+ return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3);
+}
+
+static inline int futex_wake(uint32_t *addr) {
+ return sys_futex(addr, FUTEX_WAKE, 1, NULL, NULL, 0);
+}
+
+static inline int futex_wait(uint32_t *addr, int32_t value) {
+ return sys_futex(addr, FUTEX_WAIT, value, NULL, NULL, 0);
+}
+
+static inline void
+mtx_init(mtx_t *mtx, int type)
+{
+ assert(type == mtx_plain);
+
+ mtx->val = 0;
+}
+
+static inline void
+mtx_destroy(mtx_t *mtx)
+{
+}
+
+static inline void
+mtx_lock(mtx_t *mtx)
+{
+ uint32_t c;
+
+ c = __sync_val_compare_and_swap(&mtx->val, 0, 1);
+ if (__builtin_expect(c != 0, 0)) {
+ if (c != 2)
+ c = __sync_lock_test_and_set(&mtx->val, 2);
+ while (c != 0) {
+ futex_wait(&mtx->val, 2);
+ c = __sync_lock_test_and_set(&mtx->val, 2);
+ }
+ }
+}
+
+static inline void
+mtx_unlock(mtx_t *mtx)
+{
+ uint32_t c;
+
+ c = __sync_fetch_and_sub(&mtx->val, 1);
+ if (__builtin_expect(c != 1, 0)) {
+ mtx->val = 0;
+ futex_wake(&mtx->val);
+ }
+}
+
+#else
+
+typedef full_mtx_t mtx_t;
+
+static inline void
+mtx_init(mtx_t *mtx)
+{
+ full_mtx_init(mtx, mtx_plain)
+}
+
+static inline void
+mtx_destroy(mtx_t *mtx)
+{
+ full_mtx_destroy(mtx)
+}
+
+static inline void
+mtx_lock(mtx_t *mtx)
+{
+ full_mtx_lock(mtx);
+}
+
+static inline void
+mtx_unlock(mtx_t *mtx)
+{
+ full_mtx_unlock(mtx);
+}
+
+#endif
/*------------------- 7.25.5 Thread functions -------------------*/
// 7.25.5.1
diff --git a/include/c11/threads_win32.h b/include/c11/threads_win32.h
index d017c31..ff06cfe 100644
--- a/include/c11/threads_win32.h
+++ b/include/c11/threads_win32.h
@@ -471,6 +471,31 @@ mtx_unlock(mtx_t *mtx)
return thrd_success;
}
+typedef mtx_t fast_mtx_t;
+
+static inline void
+fast_mtx_init(fast_mtx_t *mtx)
+{
+ mtx_init(mtx, mtx_plain)
+}
+
+static inline void
+fast_mtx_destroy(fast_mtx_t *mtx)
+{
+ mtx_destroy(mtx)
+}
+
+static inline void
+fast_mtx_lock(fast_mtx_t *mtx)
+{
+ mtx_lock(mtx);
+}
+
+static inline void
+fast_mtx_unlock(fast_mtx_t *mtx)
+{
+ mtx_unlock(mtx);
+}
/*------------------- 7.25.5 Thread functions -------------------*/
// 7.25.5.1
diff --git a/src/gallium/auxiliary/os/os_thread.h b/src/gallium/auxiliary/os/os_thread.h
index ff46a89..4acda4b 100644
--- a/src/gallium/auxiliary/os/os_thread.h
+++ b/src/gallium/auxiliary/os/os_thread.h
@@ -88,22 +88,22 @@ static INLINE int pipe_thread_destroy( pipe_thread thread )
/* pipe_mutex
*/
-typedef mtx_t pipe_mutex;
+typedef full_mtx_t pipe_mutex;
#define pipe_static_mutex(mutex) \
static pipe_mutex mutex = _MTX_INITIALIZER_NP
#define pipe_mutex_init(mutex) \
- (void) mtx_init(&(mutex), mtx_plain)
+ (void) full_mtx_init(&(mutex), mtx_plain)
#define pipe_mutex_destroy(mutex) \
- mtx_destroy(&(mutex))
+ full_mtx_destroy(&(mutex))
#define pipe_mutex_lock(mutex) \
- (void) mtx_lock(&(mutex))
+ (void) full_mtx_lock(&(mutex))
#define pipe_mutex_unlock(mutex) \
- (void) mtx_unlock(&(mutex))
+ (void) full_mtx_unlock(&(mutex))
/* pipe_condvar
diff --git a/src/mesa/main/mtypes.h b/src/mesa/main/mtypes.h
index 4c83379..4406527 100644
--- a/src/mesa/main/mtypes.h
+++ b/src/mesa/main/mtypes.h
@@ -3094,7 +3094,7 @@ struct gl_sync_object
*/
struct gl_shared_state
{
- mtx_t Mutex; /**< for thread safety */
+ mtx_t Mutex; /**< for thread safety */
GLint RefCount; /**< Reference count */
struct _mesa_HashTable *DisplayList; /**< Display lists hash table */
struct _mesa_HashTable *TexObjects; /**< Texture objects hash table */
@@ -3112,7 +3112,7 @@ struct gl_shared_state
* \todo Improve the granularity of locking.
*/
/*@{*/
- mtx_t TexMutex; /**< texobj thread safety */
+ full_mtx_t TexMutex; /**< texobj thread safety */
GLuint TextureStateStamp; /**< state notification for shared tex */
/*@}*/
diff --git a/src/mesa/main/shared.c b/src/mesa/main/shared.c
index ccf5355..bac8401 100644
--- a/src/mesa/main/shared.c
+++ b/src/mesa/main/shared.c
@@ -113,7 +113,7 @@ _mesa_alloc_shared_state(struct gl_context *ctx)
assert(shared->DefaultTex[TEXTURE_1D_INDEX]->RefCount == 1);
/* Mutex and timestamp for texobj state validation */
- mtx_init(&shared->TexMutex, mtx_recursive);
+ full_mtx_init(&shared->TexMutex, mtx_recursive);
shared->TextureStateStamp = 0;
shared->FrameBuffers = _mesa_NewHashTable();
@@ -357,7 +357,7 @@ free_shared_state(struct gl_context *ctx, struct gl_shared_state *shared)
_mesa_DeleteHashTable(shared->TexObjects);
mtx_destroy(&shared->Mutex);
- mtx_destroy(&shared->TexMutex);
+ full_mtx_destroy(&shared->TexMutex);
free(shared);
}
diff --git a/src/mesa/main/texobj.c b/src/mesa/main/texobj.c
index a99dd7a..10da9bc 100644
--- a/src/mesa/main/texobj.c
+++ b/src/mesa/main/texobj.c
@@ -2026,7 +2026,7 @@ _mesa_IsTexture( GLuint texture )
void
_mesa_lock_context_textures( struct gl_context *ctx )
{
- mtx_lock(&ctx->Shared->TexMutex);
+ full_mtx_lock(&ctx->Shared->TexMutex);
if (ctx->Shared->TextureStateStamp != ctx->TextureStateTimestamp) {
ctx->NewState |= _NEW_TEXTURE;
@@ -2039,7 +2039,7 @@ void
_mesa_unlock_context_textures( struct gl_context *ctx )
{
assert(ctx->Shared->TextureStateStamp == ctx->TextureStateTimestamp);
- mtx_unlock(&ctx->Shared->TexMutex);
+ full_mtx_unlock(&ctx->Shared->TexMutex);
}
diff --git a/src/mesa/main/texobj.h b/src/mesa/main/texobj.h
index ec5ccb2..1319565 100644
--- a/src/mesa/main/texobj.h
+++ b/src/mesa/main/texobj.h
@@ -107,7 +107,7 @@ _mesa_reference_texobj(struct gl_texture_object **ptr,
static inline void
_mesa_lock_texture(struct gl_context *ctx, struct gl_texture_object *texObj)
{
- mtx_lock(&ctx->Shared->TexMutex);
+ full_mtx_lock(&ctx->Shared->TexMutex);
ctx->Shared->TextureStateStamp++;
(void) texObj;
}
@@ -116,7 +116,7 @@ static inline void
_mesa_unlock_texture(struct gl_context *ctx, struct gl_texture_object *texObj)
{
(void) texObj;
- mtx_unlock(&ctx->Shared->TexMutex);
+ full_mtx_unlock(&ctx->Shared->TexMutex);
}
--
2.2.1
More information about the mesa-dev
mailing list