Mesa (main): vulkan/queue: Rework submit thread enabling

[GitLab Mirror]

Module: Mesa
Branch: main
Commit: e0ffdc8ce0b5813940c20f2055477d0d95ba1a1a
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=e0ffdc8ce0b5813940c20f2055477d0d95ba1a1a

Author: Jason Ekstrand <jason.ekstrand at collabora.com>
Date:   Thu Mar 24 17:33:57 2022 -0500

vulkan/queue: Rework submit thread enabling

Now that we have a threading mode in the device, we can set that based
on the environment variable instead of delaying it to submit time.  This
allows us to avoid the static variable trickery we use to avoid reading
environment variables over and over again.  We also move the enabling of
the submit thread up a level or two and give it a bit more obvious
condition.

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin at intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15566>

---

 src/vulkan/runtime/vk_device.c |  6 ++++-
 src/vulkan/runtime/vk_queue.c  | 55 ++++++++++++++++++++++++------------------
 2 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/src/vulkan/runtime/vk_device.c b/src/vulkan/runtime/vk_device.c
index 8ced6b425d7..517820fdc48 100644
--- a/src/vulkan/runtime/vk_device.c
+++ b/src/vulkan/runtime/vk_device.c
@@ -150,7 +150,11 @@ vk_device_init(struct vk_device *device,
       break;
 
    case VK_DEVICE_TIMELINE_MODE_ASSISTED:
-      device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND;
+      if (env_var_as_boolean("MESA_VK_ENABLE_SUBMIT_THREAD", false)) {
+         device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED;
+      } else {
+         device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND;
+      }
       break;
 
    default:
diff --git a/src/vulkan/runtime/vk_queue.c b/src/vulkan/runtime/vk_queue.c
index f650cfefdd1..7f422692c43 100644
--- a/src/vulkan/runtime/vk_queue.c
+++ b/src/vulkan/runtime/vk_queue.c
@@ -43,6 +43,9 @@
 
 #include "vulkan/wsi/wsi_common.h"
 
+static VkResult
+vk_queue_enable_submit_thread(struct vk_queue *queue);
+
 VkResult
 vk_queue_init(struct vk_queue *queue, struct vk_device *device,
               const VkDeviceQueueCreateInfo *pCreateInfo,
@@ -86,11 +89,19 @@ vk_queue_init(struct vk_queue *queue, struct vk_device *device,
       goto fail_pop;
    }
 
+   if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED) {
+      result = vk_queue_enable_submit_thread(queue);
+      if (result != VK_SUCCESS)
+         goto fail_thread;
+   }
+
    util_dynarray_init(&queue->labels, NULL);
    queue->region_begin = true;
 
    return VK_SUCCESS;
 
+fail_thread:
+   cnd_destroy(&queue->submit.pop);
 fail_pop:
    cnd_destroy(&queue->submit.push);
 fail_push:
@@ -559,6 +570,7 @@ static VkResult
 vk_queue_submit(struct vk_queue *queue,
                 const struct vulkan_submit_info *info)
 {
+   struct vk_device *device = queue->base.device;
    VkResult result;
    uint32_t sparse_memory_bind_entry_count = 0;
    uint32_t sparse_memory_image_bind_entry_count = 0;
@@ -778,31 +790,28 @@ vk_queue_submit(struct vk_queue *queue,
 
    assert(signal_count == submit->signal_count);
 
-   switch (queue->base.device->timeline_mode) {
-   case VK_DEVICE_TIMELINE_MODE_ASSISTED:
-      if (queue->submit.mode != VK_QUEUE_SUBMIT_MODE_THREADED) {
-         static int force_submit_thread = -1;
-         if (unlikely(force_submit_thread < 0)) {
-            force_submit_thread =
-               env_var_as_boolean("MESA_VK_ENABLE_SUBMIT_THREAD", false);
-         }
+   /* If this device supports threaded submit, we can't rely on the client
+    * ordering requirements to ensure submits happen in the right order.  Even
+    * if this queue doesn't have a submit thread, another queue (possibly in a
+    * different process) may and that means we our dependencies may not have
+    * been submitted to the kernel yet.  Do a quick zero-timeout WAIT_PENDING
+    * on all the wait semaphores to see if we need to start up our own thread.
+    */
+   if (device->submit_mode == VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND &&
+       queue->submit.mode != VK_QUEUE_SUBMIT_MODE_THREADED) {
+      assert(queue->submit.mode == VK_QUEUE_SUBMIT_MODE_IMMEDIATE);
 
-         if (unlikely(force_submit_thread)) {
-            result = vk_queue_enable_submit_thread(queue);
-         } else {
-            /* Otherwise, only enable the submit thread if we need it in order
-             * to resolve timeline semaphore wait-before-signal issues.
-             */
-            result = vk_sync_wait_many(queue->base.device,
-                                       submit->wait_count, submit->waits,
-                                       VK_SYNC_WAIT_PENDING, 0);
-            if (result == VK_TIMEOUT)
-               result = vk_queue_enable_submit_thread(queue);
-         }
-         if (unlikely(result != VK_SUCCESS))
-            goto fail;
-      }
+      result = vk_sync_wait_many(queue->base.device,
+                                 submit->wait_count, submit->waits,
+                                 VK_SYNC_WAIT_PENDING, 0);
+      if (result == VK_TIMEOUT)
+         result = vk_queue_enable_submit_thread(queue);
+      if (unlikely(result != VK_SUCCESS))
+         goto fail;
+   }
 
+   switch (queue->base.device->timeline_mode) {
+   case VK_DEVICE_TIMELINE_MODE_ASSISTED:
       if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED) {
          if (has_binary_permanent_semaphore_wait) {
             for (uint32_t i = 0; i < info->wait_count; i++) {