Mesa (master): radeonsi: add CP DMA flags for greater control over synchronization
Marek Olšák
mareko at kemper.freedesktop.org
Fri Jan 6 20:06:16 UTC 2017
Module: Mesa
Branch: master
Commit: a131dacb1443039b284aef3ba0a47e2ba20a13a6
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=a131dacb1443039b284aef3ba0a47e2ba20a13a6
Author: Marek Olšák <marek.olsak at amd.com>
Date: Thu Oct 13 12:09:36 2016 +0200
radeonsi: add CP DMA flags for greater control over synchronization
for L2 prefetch
Reviewed-by: Edward O'Callaghan <funfunctor at folklore1984.net>
Reviewed-by: Nicolai Hähnle <nicolai.haehnle at amd.com>
---
src/gallium/drivers/radeonsi/si_blit.c | 2 +-
src/gallium/drivers/radeonsi/si_cp_dma.c | 37 ++++++++++++++++++++------------
src/gallium/drivers/radeonsi/si_pipe.h | 8 ++++++-
3 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c
index e15fe40..5b53373 100644
--- a/src/gallium/drivers/radeonsi/si_blit.c
+++ b/src/gallium/drivers/radeonsi/si_blit.c
@@ -848,7 +848,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
/* Handle buffers first. */
if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
- si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
+ si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, 0);
return;
}
diff --git a/src/gallium/drivers/radeonsi/si_cp_dma.c b/src/gallium/drivers/radeonsi/si_cp_dma.c
index 1ca9a05..934fd8d 100644
--- a/src/gallium/drivers/radeonsi/si_cp_dma.c
+++ b/src/gallium/drivers/radeonsi/si_cp_dma.c
@@ -130,7 +130,7 @@ static unsigned get_tc_l2_flag(struct si_context *sctx, enum r600_coherency cohe
static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
struct pipe_resource *src, unsigned byte_count,
- uint64_t remaining_size,
+ uint64_t remaining_size, unsigned user_flags,
bool *is_first, unsigned *packet_flags)
{
/* Count memory usage in so that need_cs_space can take it into account. */
@@ -138,7 +138,8 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
if (src)
r600_context_add_resource_size(&sctx->b.b, src);
- si_need_cs_space(sctx);
+ if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
+ si_need_cs_space(sctx);
/* This must be done after need_cs_space. */
radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
@@ -152,10 +153,10 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
/* Flush the caches for the first copy only.
* Also wait for the previous CP DMA operations.
*/
- if (sctx->b.flags)
+ if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->b.flags)
si_emit_cache_flush(sctx);
- if (*is_first)
+ if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first)
*packet_flags |= CP_DMA_RAW_WAIT;
*is_first = false;
@@ -163,7 +164,8 @@ static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst
/* Do the synchronization after the last dma, so that all data
* is written to memory.
*/
- if (byte_count == remaining_size)
+ if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) &&
+ byte_count == remaining_size)
*packet_flags |= CP_DMA_SYNC;
}
@@ -227,7 +229,7 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
unsigned dma_flags = tc_l2_flag | CP_DMA_CLEAR;
- si_cp_dma_prepare(sctx, dst, NULL, byte_count, size,
+ si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, 0,
&is_first, &dma_flags);
/* Emit the clear packet. */
@@ -252,7 +254,7 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
* \param size Remaining size to the CP DMA alignment.
*/
static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
- bool *is_first)
+ unsigned user_flags, bool *is_first)
{
uint64_t va;
unsigned dma_flags = 0;
@@ -275,7 +277,7 @@ static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
}
si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b,
- &sctx->scratch_buffer->b.b, size, size,
+ &sctx->scratch_buffer->b.b, size, size, user_flags,
is_first, &dma_flags);
va = sctx->scratch_buffer->gpu_address;
@@ -283,9 +285,15 @@ static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
R600_COHERENCY_SHADER);
}
+/**
+ * Do memcpy between buffers using CP DMA.
+ *
+ * \param user_flags bitmask of SI_CPDMA_*
+ */
void si_copy_buffer(struct si_context *sctx,
struct pipe_resource *dst, struct pipe_resource *src,
- uint64_t dst_offset, uint64_t src_offset, unsigned size)
+ uint64_t dst_offset, uint64_t src_offset, unsigned size,
+ unsigned user_flags)
{
uint64_t main_dst_offset, main_src_offset;
unsigned skipped_size = 0;
@@ -329,8 +337,9 @@ void si_copy_buffer(struct si_context *sctx,
}
/* Flush the caches. */
- sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
- SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
+ if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC))
+ sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
+ SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
/* This is the main part doing the copying. Src is always aligned. */
main_dst_offset = dst_offset + skipped_size;
@@ -342,7 +351,7 @@ void si_copy_buffer(struct si_context *sctx,
si_cp_dma_prepare(sctx, dst, src, byte_count,
size + skipped_size + realign_size,
- &is_first, &dma_flags);
+ user_flags, &is_first, &dma_flags);
si_emit_cp_dma(sctx, main_dst_offset, main_src_offset,
byte_count, dma_flags, R600_COHERENCY_SHADER);
@@ -357,7 +366,7 @@ void si_copy_buffer(struct si_context *sctx,
unsigned dma_flags = tc_l2_flag;
si_cp_dma_prepare(sctx, dst, src, skipped_size,
- skipped_size + realign_size,
+ skipped_size + realign_size, user_flags,
&is_first, &dma_flags);
si_emit_cp_dma(sctx, dst_offset, src_offset, skipped_size,
@@ -366,7 +375,7 @@ void si_copy_buffer(struct si_context *sctx,
/* Finally, realign the engine if the size wasn't aligned. */
if (realign_size)
- si_cp_dma_realign_engine(sctx, realign_size,
+ si_cp_dma_realign_engine(sctx, realign_size, user_flags,
&is_first);
if (tc_l2_flag)
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index 9f79c2a..dc37c8d 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -372,9 +372,15 @@ void si_resource_copy_region(struct pipe_context *ctx,
const struct pipe_box *src_box);
/* si_cp_dma.c */
+#define SI_CPDMA_SKIP_CHECK_CS_SPACE (1 << 0) /* don't call need_cs_space */
+#define SI_CPDMA_SKIP_SYNC_AFTER (1 << 1) /* don't wait for DMA after the copy */
+#define SI_CPDMA_SKIP_SYNC_BEFORE (1 << 2) /* don't wait for DMA before the copy (RAW hazards) */
+#define SI_CPDMA_SKIP_GFX_SYNC (1 << 3) /* don't flush caches and don't wait for PS/CS */
+
void si_copy_buffer(struct si_context *sctx,
struct pipe_resource *dst, struct pipe_resource *src,
- uint64_t dst_offset, uint64_t src_offset, unsigned size);
+ uint64_t dst_offset, uint64_t src_offset, unsigned size,
+ unsigned user_flags);
void si_init_cp_dma_functions(struct si_context *sctx);
/* si_debug.c */
More information about the mesa-commit
mailing list