Mesa (main): tu: Use incoherent CCU write for buffer accesses

[GitLab Mirror]

Module: Mesa
Branch: main
Commit: 67c9ca2319730975025532114930b64ea56214f8
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=67c9ca2319730975025532114930b64ea56214f8

Author: Connor Abbott <cwabbott0 at gmail.com>
Date:   Wed Jun 22 19:24:58 2022 +0200

tu: Use incoherent CCU write for buffer accesses

Unlike image writes, buffer writes may access the same memory in
different ways, which we've seen in the past can cause problems. Use an
incoherent access to force flush/invalidate between accesses to the same
buffer, unless we know the barrier applies to images only.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17193>

---

 src/freedreno/vulkan/tu_cmd_buffer.c               | 28 ++++++++++++++--------
 src/gallium/drivers/zink/ci/zink-tu-a630-fails.txt |  1 -
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/freedreno/vulkan/tu_cmd_buffer.c b/src/freedreno/vulkan/tu_cmd_buffer.c
index e2419a4f608..4ad53ee3220 100644
--- a/src/freedreno/vulkan/tu_cmd_buffer.c
+++ b/src/freedreno/vulkan/tu_cmd_buffer.c
@@ -3031,7 +3031,7 @@ gfx_write_access(VkAccessFlags2 flags, VkPipelineStageFlags2 stages,
                               tu_stages | VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT);
 }
 static enum tu_cmd_access_mask
-vk2tu_access(VkAccessFlags2 flags, VkPipelineStageFlags2 stages, bool gmem)
+vk2tu_access(VkAccessFlags2 flags, VkPipelineStageFlags2 stages, bool image_only, bool gmem)
 {
    enum tu_cmd_access_mask mask = 0;
 
@@ -3142,8 +3142,16 @@ vk2tu_access(VkAccessFlags2 flags, VkPipelineStageFlags2 stages, bool gmem)
                            VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT)) {
       if (gmem) {
          mask |= TU_ACCESS_SYSMEM_WRITE;
-      } else {
+      } else if (image_only) {
+         /* Because we always split up blits/copies of images involving
+          * multiple layers, we always access each layer in the same way, with
+          * the same base address, same format, etc. This means we can avoid
+          * flushing between multiple writes to the same image. This elides
+          * flushes between e.g. multiple blits to the same image.
+          */
          mask |= TU_ACCESS_CCU_COLOR_WRITE;
+      } else {
+         mask |= TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE;
       }
    }
 
@@ -3507,9 +3515,9 @@ tu_subpass_barrier(struct tu_cmd_buffer *cmd_buffer,
    VkPipelineStageFlags2 dst_stage_vk =
       sanitize_dst_stage(barrier->dst_stage_mask);
    enum tu_cmd_access_mask src_flags =
-      vk2tu_access(barrier->src_access_mask, src_stage_vk, false);
+      vk2tu_access(barrier->src_access_mask, src_stage_vk, false, false);
    enum tu_cmd_access_mask dst_flags =
-      vk2tu_access(barrier->dst_access_mask, dst_stage_vk, false);
+      vk2tu_access(barrier->dst_access_mask, dst_stage_vk, false, false);
 
    if (barrier->incoherent_ccu_color)
       src_flags |= TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE;
@@ -4753,9 +4761,9 @@ tu_barrier(struct tu_cmd_buffer *cmd,
       VkPipelineStageFlags2 sanitized_dst_stage =
          sanitize_dst_stage(dep_info->pMemoryBarriers[i].dstStageMask);
       src_flags |= vk2tu_access(dep_info->pMemoryBarriers[i].srcAccessMask,
-                                sanitized_src_stage, gmem);
+                                sanitized_src_stage, false, gmem);
       dst_flags |= vk2tu_access(dep_info->pMemoryBarriers[i].dstAccessMask,
-                                sanitized_dst_stage, gmem);
+                                sanitized_dst_stage, false, gmem);
       srcStage |= sanitized_src_stage;
       dstStage |= sanitized_dst_stage;
    }
@@ -4766,9 +4774,9 @@ tu_barrier(struct tu_cmd_buffer *cmd,
       VkPipelineStageFlags2 sanitized_dst_stage =
          sanitize_dst_stage(dep_info->pBufferMemoryBarriers[i].dstStageMask);
       src_flags |= vk2tu_access(dep_info->pBufferMemoryBarriers[i].srcAccessMask,
-                                sanitized_src_stage, gmem);
+                                sanitized_src_stage, false, gmem);
       dst_flags |= vk2tu_access(dep_info->pBufferMemoryBarriers[i].dstAccessMask,
-                                sanitized_dst_stage, gmem);
+                                sanitized_dst_stage, false, gmem);
       srcStage |= sanitized_src_stage;
       dstStage |= sanitized_dst_stage;
    }
@@ -4790,9 +4798,9 @@ tu_barrier(struct tu_cmd_buffer *cmd,
       VkPipelineStageFlags2 sanitized_dst_stage =
          sanitize_dst_stage(dep_info->pImageMemoryBarriers[i].dstStageMask);
       src_flags |= vk2tu_access(dep_info->pImageMemoryBarriers[i].srcAccessMask,
-                                sanitized_src_stage, gmem);
+                                sanitized_src_stage, true, gmem);
       dst_flags |= vk2tu_access(dep_info->pImageMemoryBarriers[i].dstAccessMask,
-                                sanitized_dst_stage, gmem);
+                                sanitized_dst_stage, true, gmem);
       srcStage |= sanitized_src_stage;
       dstStage |= sanitized_dst_stage;
    }
diff --git a/src/gallium/drivers/zink/ci/zink-tu-a630-fails.txt b/src/gallium/drivers/zink/ci/zink-tu-a630-fails.txt
index 891835e81be..18a4d75dc2d 100644
--- a/src/gallium/drivers/zink/ci/zink-tu-a630-fails.txt
+++ b/src/gallium/drivers/zink/ci/zink-tu-a630-fails.txt
@@ -7,7 +7,6 @@ GTF-GL46.gtf30.GL3Tests.sgis_texture_lod.sgis_texture_lod_basic_lod_selection,Fa
 GTF-GL46.gtf32.GL3Tests.draw_elements_base_vertex.draw_elements_base_vertex_invalid_mode,Fail
 KHR-GL46.buffer_storage.map_persistent_draw,Fail
 KHR-GL46.copy_image.functional,Fail
-KHR-GL46.direct_state_access.buffers_functional,Fail
 KHR-GL46.gpu_shader_fp64.builtin.mod_dvec2,Fail
 KHR-GL46.gpu_shader_fp64.builtin.mod_dvec3,Fail
 KHR-GL46.gpu_shader_fp64.builtin.mod_dvec4,Fail