[Mesa-dev] [PATCH 4/4] radv: enable out-of-order rasterization when it's safe on VI+
Samuel Pitoiset
samuel.pitoiset at gmail.com
Thu Mar 29 13:25:02 UTC 2018
This can be disabled with RADV_DEBUG=nooutoforder.
No CTS regressions on Polaris, and all Vulkan games I tested
look good as well.
Expect small performance improvements for applications where
out-of-order rasterization can be enabled by the driver.
Loosely based on RadeonSI.
Signed-off-by: Samuel Pitoiset <samuel.pitoiset at gmail.com>
---
src/amd/vulkan/radv_cmd_buffer.c | 28 +++++
src/amd/vulkan/radv_debug.h | 1 +
src/amd/vulkan/radv_device.c | 7 ++
src/amd/vulkan/radv_extensions.py | 2 +-
src/amd/vulkan/radv_pipeline.c | 240 +++++++++++++++++++++++++++++++++++++-
src/amd/vulkan/radv_private.h | 4 +
6 files changed, 279 insertions(+), 3 deletions(-)
diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c
index cadb06f0af..c915d4142d 100644
--- a/src/amd/vulkan/radv_cmd_buffer.c
+++ b/src/amd/vulkan/radv_cmd_buffer.c
@@ -1171,10 +1171,24 @@ radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer)
void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer)
{
+ struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
+ uint32_t pa_sc_mode_cntl_1 =
+ pipeline ? pipeline->graphics.ms.pa_sc_mode_cntl_1 : 0;
uint32_t db_count_control;
if(!cmd_buffer->state.active_occlusion_queries) {
if (cmd_buffer->device->physical_device->rad_info.chip_class >= CIK) {
+ if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) &&
+ pipeline->graphics.disable_out_of_order_rast_for_occlusion) {
+ /* Re-enable out-of-order rasterization if the
+ * bound pipeline supports it and if it's has
+ * been disabled before starting occlusion
+ * queries.
+ */
+ radeon_set_context_reg(cmd_buffer->cs,
+ R_028A4C_PA_SC_MODE_CNTL_1,
+ pa_sc_mode_cntl_1);
+ }
db_count_control = 0;
} else {
db_count_control = S_028004_ZPASS_INCREMENT_DISABLE(1);
@@ -1186,6 +1200,20 @@ void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer)
S_028004_ZPASS_ENABLE(1) |
S_028004_SLICE_EVEN_ENABLE(1) |
S_028004_SLICE_ODD_ENABLE(1);
+
+ if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) &&
+ pipeline->graphics.disable_out_of_order_rast_for_occlusion) {
+ /* If the bound pipeline has enabled
+ * out-of-order rasterization, we should
+ * disable it before starting occlusion
+ * queries.
+ */
+ pa_sc_mode_cntl_1 &= C_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE;
+
+ radeon_set_context_reg(cmd_buffer->cs,
+ R_028A4C_PA_SC_MODE_CNTL_1,
+ pa_sc_mode_cntl_1);
+ }
} else {
db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) |
S_028004_SAMPLE_RATE(0); /* TODO: set this to the number of samples of the current framebuffer */
diff --git a/src/amd/vulkan/radv_debug.h b/src/amd/vulkan/radv_debug.h
index 08877676b5..65bc61b1d4 100644
--- a/src/amd/vulkan/radv_debug.h
+++ b/src/amd/vulkan/radv_debug.h
@@ -43,6 +43,7 @@ enum {
RADV_DEBUG_SYNC_SHADERS = 0x2000,
RADV_DEBUG_NO_SISCHED = 0x4000,
RADV_DEBUG_PREOPTIR = 0x8000,
+ RADV_DEBUG_NO_OUT_OF_ORDER = 0x10000,
};
enum {
diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c
index 4acdf3d416..9af6bf8059 100644
--- a/src/amd/vulkan/radv_device.c
+++ b/src/amd/vulkan/radv_device.c
@@ -307,6 +307,12 @@ radv_physical_device_init(struct radv_physical_device *device,
device->has_scissor_bug = device->rad_info.family == CHIP_VEGA10 ||
device->rad_info.family == CHIP_RAVEN;
+ /* Out-of-order primitive rasterization. */
+ device->has_out_of_order_rast = device->rad_info.chip_class >= VI &&
+ device->rad_info.max_se >= 2;
+ device->out_of_order_rast_allowed = device->has_out_of_order_rast &&
+ !(device->instance->debug_flags & RADV_DEBUG_NO_OUT_OF_ORDER);
+
radv_physical_device_init_mem_types(device);
radv_fill_device_extension_table(device, &device->supported_extensions);
@@ -376,6 +382,7 @@ static const struct debug_control radv_debug_options[] = {
{"syncshaders", RADV_DEBUG_SYNC_SHADERS},
{"nosisched", RADV_DEBUG_NO_SISCHED},
{"preoptir", RADV_DEBUG_PREOPTIR},
+ {"nooutoforder", RADV_DEBUG_NO_OUT_OF_ORDER},
{NULL, 0}
};
diff --git a/src/amd/vulkan/radv_extensions.py b/src/amd/vulkan/radv_extensions.py
index cbb02f138a..aa284428e3 100644
--- a/src/amd/vulkan/radv_extensions.py
+++ b/src/amd/vulkan/radv_extensions.py
@@ -94,7 +94,7 @@ EXTENSIONS = [
Extension('VK_EXT_sampler_filter_minmax', 1, 'device->rad_info.chip_class >= CIK'),
Extension('VK_AMD_draw_indirect_count', 1, True),
Extension('VK_AMD_gcn_shader', 1, True),
- Extension('VK_AMD_rasterization_order', 1, 'device->rad_info.chip_class >= VI && device->rad_info.max_se >= 2'),
+ Extension('VK_AMD_rasterization_order', 1, 'device->has_out_of_order_rast'),
Extension('VK_AMD_shader_info', 1, True),
Extension('VK_AMD_shader_trinary_minmax', 1, True),
]
diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c
index e259b16318..0841cb96d7 100644
--- a/src/amd/vulkan/radv_pipeline.c
+++ b/src/amd/vulkan/radv_pipeline.c
@@ -63,10 +63,25 @@ struct radv_blend_state {
uint32_t cb_shader_mask;
uint32_t db_alpha_to_mask;
+ uint32_t commutative_4bit;
+
bool single_cb_enable;
bool mrt0_is_dual_src;
};
+struct radv_dsa_order_invariance {
+ /* Whether the final result in Z/S buffers is guaranteed to be
+ * invariant under changes to the order in which fragments arrive.
+ */
+ bool zs;
+
+ /* Whether the set of fragments that pass the combined Z/S test is
+ * guaranteed to be invariant under changes to the order in which
+ * fragments arrive.
+ */
+ bool pass_set;
+};
+
struct radv_tessellation_state {
uint32_t ls_hs_config;
unsigned num_patches;
@@ -530,6 +545,40 @@ radv_pipeline_compute_get_int_clamp(const VkGraphicsPipelineCreateInfo *pCreateI
}
}
+static void
+radv_blend_check_commutativity(struct radv_blend_state *blend,
+ VkBlendOp op, VkBlendFactor src,
+ VkBlendFactor dst, unsigned chanmask)
+{
+ /* Src factor is allowed when it does not depend on Dst. */
+ static const uint32_t src_allowed =
+ (1u << VK_BLEND_FACTOR_ONE) |
+ (1u << VK_BLEND_FACTOR_SRC_COLOR) |
+ (1u << VK_BLEND_FACTOR_SRC_ALPHA) |
+ (1u << VK_BLEND_FACTOR_SRC_ALPHA_SATURATE) |
+ (1u << VK_BLEND_FACTOR_CONSTANT_COLOR) |
+ (1u << VK_BLEND_FACTOR_CONSTANT_ALPHA) |
+ (1u << VK_BLEND_FACTOR_SRC1_COLOR) |
+ (1u << VK_BLEND_FACTOR_SRC1_ALPHA) |
+ (1u << VK_BLEND_FACTOR_ZERO) |
+ (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR) |
+ (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA) |
+ (1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR) |
+ (1u << VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA) |
+ (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR) |
+ (1u << VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA);
+
+ if (dst == VK_BLEND_FACTOR_ONE &&
+ (src_allowed && (1u << src))) {
+ /* Addition is commutative, but floating point addition isn't
+ * associative: subtle changes can be introduced via different
+ * rounding. Be conservative, only enable for min and max.
+ */
+ if (op == VK_BLEND_OP_MAX || op == VK_BLEND_OP_MIN)
+ blend->commutative_4bit |= chanmask;
+ }
+}
+
static struct radv_blend_state
radv_pipeline_init_blend_state(struct radv_pipeline *pipeline,
const VkGraphicsPipelineCreateInfo *pCreateInfo,
@@ -600,6 +649,11 @@ radv_pipeline_init_blend_state(struct radv_pipeline *pipeline,
dstA = VK_BLEND_FACTOR_ONE;
}
+ radv_blend_check_commutativity(&blend, eqRGB, srcRGB, dstRGB,
+ 0x7 << (4 * i));
+ radv_blend_check_commutativity(&blend, eqA, srcA, dstA,
+ 0x8 << (4 * i));
+
/* Blending optimizations for RB+.
* These transformations don't change the behavior.
*
@@ -736,13 +790,182 @@ static uint8_t radv_pipeline_get_ps_iter_samples(const VkPipelineMultisampleStat
return ps_iter_samples;
}
+static bool
+radv_is_depth_write_enabled(const VkPipelineDepthStencilStateCreateInfo *pCreateInfo)
+{
+ return pCreateInfo->depthTestEnable &&
+ pCreateInfo->depthWriteEnable &&
+ pCreateInfo->depthCompareOp != VK_COMPARE_OP_NEVER;
+}
+
+static bool
+radv_writes_stencil(const VkStencilOpState *state)
+{
+ return state->writeMask &&
+ (state->failOp != VK_STENCIL_OP_KEEP ||
+ state->passOp != VK_STENCIL_OP_KEEP ||
+ state->depthFailOp != VK_STENCIL_OP_KEEP);
+}
+
+static bool
+radv_is_stencil_write_enabled(const VkPipelineDepthStencilStateCreateInfo *pCreateInfo)
+{
+ return pCreateInfo->stencilTestEnable &&
+ (radv_writes_stencil(&pCreateInfo->front) ||
+ radv_writes_stencil(&pCreateInfo->back));
+}
+
+static bool
+radv_is_ds_write_enabled(const VkPipelineDepthStencilStateCreateInfo *pCreateInfo)
+{
+ return radv_is_depth_write_enabled(pCreateInfo) ||
+ radv_is_stencil_write_enabled(pCreateInfo);
+}
+
+static bool
+radv_order_invariant_stencil_op(VkStencilOp op)
+{
+ /* REPLACE is normally order invariant, except when the stencil
+ * reference value is written by the fragment shader. Tracking this
+ * interaction does not seem worth the effort, so be conservative.
+ */
+ return op != VK_STENCIL_OP_INCREMENT_AND_CLAMP &&
+ op != VK_STENCIL_OP_DECREMENT_AND_CLAMP &&
+ op != VK_STENCIL_OP_REPLACE;
+}
+
+static bool
+radv_order_invariant_stencil_state(const VkStencilOpState *state)
+{
+ /* Compute whether, assuming Z writes are disabled, this stencil state
+ * is order invariant in the sense that the set of passing fragments as
+ * well as the final stencil buffer result does not depend on the order
+ * of fragments.
+ */
+ return !state->writeMask ||
+ /* The following assumes that Z writes are disabled. */
+ (state->compareOp == VK_COMPARE_OP_ALWAYS &&
+ radv_order_invariant_stencil_op(state->passOp) &&
+ radv_order_invariant_stencil_op(state->depthFailOp)) ||
+ (state->compareOp == VK_COMPARE_OP_NEVER &&
+ radv_order_invariant_stencil_op(state->failOp));
+}
+
+static bool
+radv_pipeline_out_of_order_rast(struct radv_pipeline *pipeline,
+ struct radv_blend_state *blend,
+ const VkGraphicsPipelineCreateInfo *pCreateInfo)
+{
+ RADV_FROM_HANDLE(radv_render_pass, pass, pCreateInfo->renderPass);
+ struct radv_subpass *subpass = pass->subpasses + pCreateInfo->subpass;
+ unsigned colormask = blend->cb_target_enabled_4bit;
+
+ if (!pipeline->device->physical_device->out_of_order_rast_allowed)
+ return false;
+
+ /* Be conservative if a logic operation is enabled with color buffers. */
+ if (colormask && pCreateInfo->pColorBlendState->logicOpEnable)
+ return false;
+
+ /* Default depth/stencil invariance when no attachment is bound. */
+ struct radv_dsa_order_invariance dsa_order_invariant = {
+ .zs = true, .pass_set = true
+ };
+
+ if (pCreateInfo->pDepthStencilState &&
+ subpass->depth_stencil_attachment.attachment != VK_ATTACHMENT_UNUSED) {
+ const VkPipelineDepthStencilStateCreateInfo *vkds =
+ pCreateInfo->pDepthStencilState;
+ struct radv_render_pass_attachment *attachment =
+ pass->attachments + subpass->depth_stencil_attachment.attachment;
+ bool has_stencil = vk_format_is_stencil(attachment->format);
+ struct radv_dsa_order_invariance order_invariance[2];
+ struct radv_shader_variant *ps =
+ pipeline->shaders[MESA_SHADER_FRAGMENT];
+
+ /* Compute depth/stencil order invariance in order to know if
+ * it's safe to enable out-of-order.
+ */
+ bool zfunc_is_ordered =
+ vkds->depthCompareOp == VK_COMPARE_OP_NEVER ||
+ vkds->depthCompareOp == VK_COMPARE_OP_LESS ||
+ vkds->depthCompareOp == VK_COMPARE_OP_LESS_OR_EQUAL ||
+ vkds->depthCompareOp == VK_COMPARE_OP_GREATER ||
+ vkds->depthCompareOp == VK_COMPARE_OP_GREATER_OR_EQUAL;
+
+ bool nozwrite_and_order_invariant_stencil =
+ !radv_is_ds_write_enabled(vkds) ||
+ (!radv_is_depth_write_enabled(vkds) &&
+ radv_order_invariant_stencil_state(&vkds->front) &&
+ radv_order_invariant_stencil_state(&vkds->back));
+
+ order_invariance[1].zs =
+ nozwrite_and_order_invariant_stencil ||
+ (!radv_is_stencil_write_enabled(vkds) &&
+ zfunc_is_ordered);
+ order_invariance[0].zs =
+ !radv_is_depth_write_enabled(vkds) || zfunc_is_ordered;
+
+ order_invariance[1].pass_set =
+ nozwrite_and_order_invariant_stencil ||
+ (!radv_is_stencil_write_enabled(vkds) &&
+ (vkds->depthCompareOp == VK_COMPARE_OP_ALWAYS ||
+ vkds->depthCompareOp == VK_COMPARE_OP_NEVER));
+ order_invariance[0].pass_set =
+ !radv_is_depth_write_enabled(vkds) ||
+ (vkds->depthCompareOp == VK_COMPARE_OP_ALWAYS ||
+ vkds->depthCompareOp == VK_COMPARE_OP_NEVER);
+
+ dsa_order_invariant = order_invariance[has_stencil];
+ if (!dsa_order_invariant.zs)
+ return false;
+
+ /* The set of PS invocations is always order invariant,
+ * except when early Z/S tests are requested.
+ */
+ if (ps &&
+ ps->info.info.ps.writes_memory &&
+ ps->info.fs.early_fragment_test &&
+ !dsa_order_invariant.pass_set)
+ return false;
+
+ /* Determine if out-of-order rasterization should be disabled
+ * when occlusion queries are used.
+ */
+ pipeline->graphics.disable_out_of_order_rast_for_occlusion =
+ !dsa_order_invariant.pass_set;
+ }
+
+ /* No color buffers are enabled for writing. */
+ if (!colormask)
+ return true;
+
+ unsigned blendmask = colormask & blend->blend_enable_4bit;
+
+ if (blendmask) {
+ /* Only commutative blending. */
+ if (blendmask & ~blend->commutative_4bit)
+ return false;
+
+ if (!dsa_order_invariant.pass_set)
+ return false;
+ }
+
+ if (colormask & ~blendmask)
+ return false;
+
+ return true;
+}
+
static void
radv_pipeline_init_multisample_state(struct radv_pipeline *pipeline,
+ struct radv_blend_state *blend,
const VkGraphicsPipelineCreateInfo *pCreateInfo)
{
const VkPipelineMultisampleStateCreateInfo *vkms = pCreateInfo->pMultisampleState;
struct radv_multisample_state *ms = &pipeline->graphics.ms;
unsigned num_tile_pipes = pipeline->device->physical_device->rad_info.num_tile_pipes;
+ bool out_of_order_rast = false;
int ps_iter_samples = 1;
uint32_t mask = 0xffff;
@@ -794,8 +1017,21 @@ radv_pipeline_init_multisample_state(struct radv_pipeline *pipeline,
const struct VkPipelineRasterizationStateRasterizationOrderAMD *raster_order =
vk_find_struct_const(pCreateInfo->pRasterizationState->pNext, PIPELINE_RASTERIZATION_STATE_RASTERIZATION_ORDER_AMD);
if (raster_order && raster_order->rasterizationOrder == VK_RASTERIZATION_ORDER_RELAXED_AMD) {
+ /* Out-of-order rasterization is explicitly enabled by the
+ * application.
+ */
+ out_of_order_rast = true;
+ } else {
+ /* Determine if the driver can enable out-of-order
+ * rasterization internally.
+ */
+ out_of_order_rast =
+ radv_pipeline_out_of_order_rast(pipeline, blend, pCreateInfo);
+ }
+
+ if (out_of_order_rast) {
ms->pa_sc_mode_cntl_1 |= S_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(1) |
- S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7);
+ S_028A4C_OUT_OF_ORDER_WATER_MARK(0x7);
}
if (vkms && vkms->pSampleMask) {
@@ -3075,7 +3311,7 @@ radv_pipeline_init(struct radv_pipeline *pipeline,
pStages);
pipeline->graphics.spi_baryc_cntl = S_0286E0_FRONT_FACE_ALL_BITS(1);
- radv_pipeline_init_multisample_state(pipeline, pCreateInfo);
+ radv_pipeline_init_multisample_state(pipeline, &blend, pCreateInfo);
uint32_t gs_out;
uint32_t prim = si_translate_prim(pCreateInfo->pInputAssemblyState->topology);
diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h
index ce2e487bdb..a05df07845 100644
--- a/src/amd/vulkan/radv_private.h
+++ b/src/amd/vulkan/radv_private.h
@@ -291,6 +291,9 @@ struct radv_physical_device {
bool cpdma_prefetch_writes_memory;
bool has_scissor_bug;
+ bool has_out_of_order_rast;
+ bool out_of_order_rast_allowed;
+
/* This is the drivers on-disk cache used as a fallback as opposed to
* the pipeline cache defined by apps.
*/
@@ -1232,6 +1235,7 @@ struct radv_pipeline {
struct radv_prim_vertex_count prim_vertex_count;
bool can_use_guardband;
uint32_t needed_dynamic_state;
+ bool disable_out_of_order_rast_for_occlusion;
} graphics;
};
--
2.16.3
More information about the mesa-dev
mailing list