[Mesa-dev] [PATCH 2/3] radeonsi: add CP DMA flags for greater control over synchronization
Marek Olšák
maraeo at gmail.com
Mon Jan 2 20:18:37 UTC 2017
From: Marek Olšák <marek.olsak at amd.com>
for L2 prefetch
---
src/gallium/drivers/radeonsi/si_blit.c | 2 +-
src/gallium/drivers/radeonsi/si_cp_dma.c | 37 ++++++++++++++++++++------------
src/gallium/drivers/radeonsi/si_pipe.h | 8 ++++++-
3 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c
index e15fe40..5b53373 100644
--- a/src/gallium/drivers/radeonsi/si_blit.c
+++ b/src/gallium/drivers/radeonsi/si_blit.c
@@ -841,21 +841,21 @@ void si_resource_copy_region(struct pipe_context *ctx,
struct si_context *sctx = (struct si_context *)ctx;
struct r600_texture *rsrc = (struct r600_texture*)src;
struct pipe_surface *dst_view, dst_templ;
struct pipe_sampler_view src_templ, *src_view;
unsigned dst_width, dst_height, src_width0, src_height0;
unsigned src_force_level = 0;
struct pipe_box sbox, dstbox;
/* Handle buffers first. */
if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
- si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
+ si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, 0);
return;
}
assert(u_max_sample(dst) == u_max_sample(src));
/* The driver doesn't decompress resources automatically while
* u_blitter is rendering. */
si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level,
src_box->z, src_box->z + src_box->depth - 1);
diff --git a/src/gallium/drivers/radeonsi/si_cp_dma.c b/src/gallium/drivers/radeonsi/si_cp_dma.c
index 7e5c4b2..653021e 100644
--- a/src/gallium/drivers/radeonsi/si_cp_dma.c
+++ b/src/gallium/drivers/radeonsi/si_cp_dma.c
@@ -123,54 +123,56 @@ static unsigned get_flush_flags(struct si_context *sctx, enum r600_coherency coh
}
static unsigned get_tc_l2_flag(struct si_context *sctx, enum r600_coherency coher)
{
return coher == R600_COHERENCY_SHADER &&
sctx->b.chip_class >= CIK ? CP_DMA_USE_L2 : 0;
}
static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst,
struct pipe_resource *src, unsigned byte_count,
- uint64_t remaining_size,
+ uint64_t remaining_size, unsigned user_flags,
bool *is_first, unsigned *packet_flags)
{
/* Count memory usage in so that need_cs_space can take it into account. */
r600_context_add_resource_size(&sctx->b.b, dst);
if (src)
r600_context_add_resource_size(&sctx->b.b, src);
- si_need_cs_space(sctx);
+ if (!(user_flags & SI_CPDMA_SKIP_CHECK_CS_SPACE))
+ si_need_cs_space(sctx);
/* This must be done after need_cs_space. */
radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
(struct r600_resource*)dst,
RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
if (src)
radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
(struct r600_resource*)src,
RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
/* Flush the caches for the first copy only.
* Also wait for the previous CP DMA operations.
*/
- if (sctx->b.flags)
+ if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC) && sctx->b.flags)
si_emit_cache_flush(sctx);
- if (*is_first)
+ if (!(user_flags & SI_CPDMA_SKIP_SYNC_BEFORE) && *is_first)
*packet_flags |= CP_DMA_RAW_WAIT;
*is_first = false;
/* Do the synchronization after the last dma, so that all data
* is written to memory.
*/
- if (byte_count == remaining_size)
+ if (!(user_flags & SI_CPDMA_SKIP_SYNC_AFTER) &&
+ byte_count == remaining_size)
*packet_flags |= CP_DMA_SYNC;
}
static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
uint64_t offset, uint64_t size, unsigned value,
enum r600_coherency coher)
{
struct si_context *sctx = (struct si_context*)ctx;
unsigned tc_l2_flag = get_tc_l2_flag(sctx, coher);
unsigned flush_flags = get_flush_flags(sctx, coher);
@@ -201,21 +203,21 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
uint64_t va = r600_resource(dst)->gpu_address + offset;
/* Flush the caches. */
sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
while (size) {
unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
unsigned dma_flags = tc_l2_flag | CP_DMA_CLEAR;
- si_cp_dma_prepare(sctx, dst, NULL, byte_count, size,
+ si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, 0,
&is_first, &dma_flags);
/* Emit the clear packet. */
si_emit_cp_dma(sctx, va, value, byte_count, dma_flags, coher);
size -= byte_count;
va += byte_count;
}
if (tc_l2_flag)
@@ -226,21 +228,21 @@ static void si_clear_buffer(struct pipe_context *ctx, struct pipe_resource *dst,
sctx->b.num_cp_dma_calls++;
}
/**
* Realign the CP DMA engine. This must be done after a copy with an unaligned
* size.
*
* \param size Remaining size to the CP DMA alignment.
*/
static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
- bool *is_first)
+ unsigned user_flags, bool *is_first)
{
uint64_t va;
unsigned dma_flags = 0;
unsigned scratch_size = CP_DMA_ALIGNMENT * 2;
assert(size < CP_DMA_ALIGNMENT);
/* Use the scratch buffer as the dummy buffer. The 3D engine should be
* idle at this point.
*/
@@ -249,31 +251,37 @@ static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size,
r600_resource_reference(&sctx->scratch_buffer, NULL);
sctx->scratch_buffer = (struct r600_resource*)
pipe_buffer_create(&sctx->screen->b.b, 0,
PIPE_USAGE_DEFAULT, scratch_size);
if (!sctx->scratch_buffer)
return;
sctx->emit_scratch_reloc = true;
}
si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b,
- &sctx->scratch_buffer->b.b, size, size,
+ &sctx->scratch_buffer->b.b, size, size, user_flags,
is_first, &dma_flags);
va = sctx->scratch_buffer->gpu_address;
si_emit_cp_dma(sctx, va, va + CP_DMA_ALIGNMENT, size, dma_flags,
R600_COHERENCY_SHADER);
}
+/**
+ * Do memcpy between buffers using CP DMA.
+ *
+ * \param user_flags bitmask of SI_CPDMA_*
+ */
void si_copy_buffer(struct si_context *sctx,
struct pipe_resource *dst, struct pipe_resource *src,
- uint64_t dst_offset, uint64_t src_offset, unsigned size)
+ uint64_t dst_offset, uint64_t src_offset, unsigned size,
+ unsigned user_flags)
{
uint64_t main_dst_offset, main_src_offset;
unsigned skipped_size = 0;
unsigned realign_size = 0;
unsigned tc_l2_flag = get_tc_l2_flag(sctx, R600_COHERENCY_SHADER);
unsigned flush_flags = get_flush_flags(sctx, R600_COHERENCY_SHADER);
bool is_first = true;
if (!size)
return;
@@ -303,58 +311,59 @@ void si_copy_buffer(struct si_context *sctx,
*/
if (src_offset % CP_DMA_ALIGNMENT) {
skipped_size = CP_DMA_ALIGNMENT - (src_offset % CP_DMA_ALIGNMENT);
/* The main part will be skipped if the size is too small. */
skipped_size = MIN2(skipped_size, size);
size -= skipped_size;
}
}
/* Flush the caches. */
- sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
- SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
+ if (!(user_flags & SI_CPDMA_SKIP_GFX_SYNC))
+ sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
+ SI_CONTEXT_CS_PARTIAL_FLUSH | flush_flags;
/* This is the main part doing the copying. Src is always aligned. */
main_dst_offset = dst_offset + skipped_size;
main_src_offset = src_offset + skipped_size;
while (size) {
unsigned dma_flags = tc_l2_flag;
unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
si_cp_dma_prepare(sctx, dst, src, byte_count,
size + skipped_size + realign_size,
- &is_first, &dma_flags);
+ user_flags, &is_first, &dma_flags);
si_emit_cp_dma(sctx, main_dst_offset, main_src_offset,
byte_count, dma_flags, R600_COHERENCY_SHADER);
size -= byte_count;
main_src_offset += byte_count;
main_dst_offset += byte_count;
}
/* Copy the part we skipped because src wasn't aligned. */
if (skipped_size) {
unsigned dma_flags = tc_l2_flag;
si_cp_dma_prepare(sctx, dst, src, skipped_size,
- skipped_size + realign_size,
+ skipped_size + realign_size, user_flags,
&is_first, &dma_flags);
si_emit_cp_dma(sctx, dst_offset, src_offset, skipped_size,
dma_flags, R600_COHERENCY_SHADER);
}
/* Finally, realign the engine if the size wasn't aligned. */
if (realign_size)
- si_cp_dma_realign_engine(sctx, realign_size,
+ si_cp_dma_realign_engine(sctx, realign_size, user_flags,
&is_first);
if (tc_l2_flag)
r600_resource(dst)->TC_L2_dirty = true;
/* If it's not a prefetch... */
if (dst_offset != src_offset)
sctx->b.num_cp_dma_calls++;
}
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index 9f79c2a..dc37c8d 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -365,23 +365,29 @@ void si_decompress_graphics_textures(struct si_context *sctx);
void si_decompress_compute_textures(struct si_context *sctx);
void si_resource_copy_region(struct pipe_context *ctx,
struct pipe_resource *dst,
unsigned dst_level,
unsigned dstx, unsigned dsty, unsigned dstz,
struct pipe_resource *src,
unsigned src_level,
const struct pipe_box *src_box);
/* si_cp_dma.c */
+#define SI_CPDMA_SKIP_CHECK_CS_SPACE (1 << 0) /* don't call need_cs_space */
+#define SI_CPDMA_SKIP_SYNC_AFTER (1 << 1) /* don't wait for DMA after the copy */
+#define SI_CPDMA_SKIP_SYNC_BEFORE (1 << 2) /* don't wait for DMA before the copy (RAW hazards) */
+#define SI_CPDMA_SKIP_GFX_SYNC (1 << 3) /* don't flush caches and don't wait for PS/CS */
+
void si_copy_buffer(struct si_context *sctx,
struct pipe_resource *dst, struct pipe_resource *src,
- uint64_t dst_offset, uint64_t src_offset, unsigned size);
+ uint64_t dst_offset, uint64_t src_offset, unsigned size,
+ unsigned user_flags);
void si_init_cp_dma_functions(struct si_context *sctx);
/* si_debug.c */
void si_init_debug_functions(struct si_context *sctx);
void si_check_vm_faults(struct r600_common_context *ctx,
struct radeon_saved_cs *saved, enum ring_type ring);
bool si_replace_shader(unsigned num, struct radeon_shader_binary *binary);
/* si_dma.c */
void si_init_dma_functions(struct si_context *sctx);
--
2.7.4
More information about the mesa-dev
mailing list