[Xcb] [PATCH 2/3] fix deadlock with xcb_take_socket/return_socket v3

[Christian König]

From: Christian König <christian.koenig at amd.com>

To prevent different threads from stealing the socket from each other the
caller of "xcb_take_socket" must hold a lock that is also acquired in
"return_socket". Unfortunately xcb tries to prevent calling return_socket
from multiple threads and this can lead to a deadlock situation.

A simple example:
- X11 has taken the socket
- Thread A has locked the display.
- Thread B does xcb_no_operation() and thus ends up in libX11's return_socket(),
  waiting for the display lock.
- Thread A calls e.g. xcb_no_operation(), too, ends up in return_socket() and
  because socket_moving == 1, ends up waiting for thread B
=> Deadlock

This patch allows calling return_socket from different threads at the same time
an so resolves the deadlock situation.

Partially fixes: https://bugs.freedesktop.org/show_bug.cgi?id=20708

v2: fixes additional pthread_cond_wait dependencies,
    rework comments and patch description

v3: separate pthread_cond_wait dependencies and unrelated whitespace
    change into their own patch, use unsigned for socket_seq

Signed-off-by: Christian König <christian.koenig at amd.com>
---
 src/xcb_out.c |   38 ++++++++++++++++++++------------------
 src/xcbext.h  |    1 +
 src/xcbint.h  |    7 ++++---
 3 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/src/xcb_out.c b/src/xcb_out.c
index a4f52fb..429fa99 100644
--- a/src/xcb_out.c
+++ b/src/xcb_out.c
@@ -86,21 +86,24 @@ static void send_sync(xcb_connection_t *c)
 
 static void get_socket_back(xcb_connection_t *c)
 {
-    while(c->out.return_socket && c->out.socket_moving)
-        pthread_cond_wait(&c->out.socket_cond, &c->iolock);
-    if(!c->out.return_socket)
-        return;
-
-    c->out.socket_moving = 1;
-    pthread_mutex_unlock(&c->iolock);
-    c->out.return_socket(c->out.socket_closure);
-    pthread_mutex_lock(&c->iolock);
-    c->out.socket_moving = 0;
-
-    pthread_cond_broadcast(&c->out.socket_cond);
-    c->out.return_socket = 0;
-    c->out.socket_closure = 0;
-    _xcb_in_replies_done(c);
+    while (c->out.return_socket) {
+        /* we are about to release the lock,
+           so make a copy of the current status */
+        xcb_return_socket_func_t return_socket = c->out.return_socket;
+        void *socket_closure = c->out.socket_closure;
+        int socket_seq = c->out.socket_seq;
+
+        pthread_mutex_unlock(&c->iolock);
+        return_socket(socket_closure);
+        pthread_mutex_lock(&c->iolock);
+
+        /* make sure nobody else has acquired the socket */
+        if (socket_seq == c->out.socket_seq) {
+            c->out.return_socket = 0;
+            c->out.socket_closure = 0;
+            _xcb_in_replies_done(c);
+        }
+    }
 }
 
 /* Public interface */
@@ -278,6 +281,7 @@ int xcb_take_socket(xcb_connection_t *c, void (*return_socket)(void *closure), v
     {
         c->out.return_socket = return_socket;
         c->out.socket_closure = closure;
+        ++c->out.socket_seq;
         if(flags)
             _xcb_in_expect_reply(c, c->out.request, WORKAROUND_EXTERNAL_SOCKET_OWNER, flags);
         assert(c->out.request == c->out.request_written);
@@ -314,11 +318,9 @@ int xcb_flush(xcb_connection_t *c)
 
 int _xcb_out_init(_xcb_out *out)
 {
-    if(pthread_cond_init(&out->socket_cond, 0))
-        return 0;
     out->return_socket = 0;
     out->socket_closure = 0;
-    out->socket_moving = 0;
+    out->socket_seq = 0;
 
     if(pthread_cond_init(&out->cond, 0))
         return 0;
diff --git a/src/xcbext.h b/src/xcbext.h
index 98b3c93..4e1f2f7 100644
--- a/src/xcbext.h
+++ b/src/xcbext.h
@@ -66,6 +66,7 @@ unsigned int xcb_send_request(xcb_connection_t *c, int flags, struct iovec *vect
  * callback which XCB can call when it wants the write side of the
  * socket back to make a request. This callback synchronizes with the
  * external socket owner and flushes any output queues if appropriate.
+ * The callback might be called from different threads at the same time.
  * If you are sending requests which won't cause a reply, please note the
  * comment for xcb_writev which explains some sequence number wrap issues.
  * */
diff --git a/src/xcbint.h b/src/xcbint.h
index f9e5a52..7f9ab28 100644
--- a/src/xcbint.h
+++ b/src/xcbint.h
@@ -79,14 +79,15 @@ void *_xcb_map_remove(_xcb_map *q, unsigned int key);
 
 /* xcb_out.c */
 
+typedef void (*xcb_return_socket_func_t)(void *closure);
+
 typedef struct _xcb_out {
     pthread_cond_t cond;
     int writing;
 
-    pthread_cond_t socket_cond;
-    void (*return_socket)(void *closure);
+    xcb_return_socket_func_t return_socket;
     void *socket_closure;
-    int socket_moving;
+    unsigned int socket_seq;
 
     char queue[XCB_QUEUE_BUFFER_SIZE];
     int queue_len;
-- 
1.7.9.5