Mesa (master): iris: Store a list of shader variants in the shader itself
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Fri Jan 29 06:33:34 UTC 2021
Module: Mesa
Branch: master
Commit: 1afed51445c4d0fa9420ab3fb9e5b82638a1a304
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=1afed51445c4d0fa9420ab3fb9e5b82638a1a304
Author: Kenneth Graunke <kenneth at whitecape.org>
Date: Mon Nov 16 13:17:08 2020 -0800
iris: Store a list of shader variants in the shader itself
We've traditionally stored shader variants in a per-context hash table,
based on a key with many per-stage fields. On older hardware supported
by i965, there were potentially quite a few variants, as many features
had to be emulated in shaders, including things like texture swizzling.
However, on the modern hardware targeted by iris, our NOS dependencies
are much smaller. We almost always guess the correct state when doing
the initial precompile, and so we have maybe 1-3 variants. iris NOS
keys are also dramatically smaller (4 to 24 bytes) than i965's.
Unlike the classic world, Gallium also provides a single kind of object
for API shaders---pipe_shader_state aka iris_uncompiled_shader. We can
simply store a list of shader variants there. This makes it possible
to access shader variants across contexts, rather than compiling them
separately for each context, which better matches how the APIs work.
To look up variants, we simply walk the list and memcmp the keys.
Since the list is almost always singular (and rarely ever long),
and the keys are tiny, this should be quite low overhead.
We continue storing internally generated shaders for BLORP and
passthrough TCS in the per-context hash table, as they don't have
an associated pipe_shader_state / iris_uncompiled_shader object.
(There can also be many BLORP shaders, and the blit keys are large,
so having a hash table rather than a list makes sense there.)
Because iris_uncompiled_shaders are shared across multiple contexts,
we do require locking when accessing this list. Fortunately, this
is a per-shader lock, rather than a global one. Additionally, since
we only append variants to the list, and generate the first one at
precompile time (while only one context has the uncompiled shader),
we can assume that it is safe to access that first entry without
locking the list. This means that we only have to lock when we
have multiple variants, which is relatively uncommon.
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/7668>
---
src/gallium/drivers/iris/iris_context.h | 29 +++++++--
src/gallium/drivers/iris/iris_disk_cache.c | 4 +-
src/gallium/drivers/iris/iris_program.c | 93 +++++++++++++++++++++------
src/gallium/drivers/iris/iris_program_cache.c | 52 +++++++++------
4 files changed, 131 insertions(+), 47 deletions(-)
diff --git a/src/gallium/drivers/iris/iris_context.h b/src/gallium/drivers/iris/iris_context.h
index 5fd4cdb5e11..eb7699a841f 100644
--- a/src/gallium/drivers/iris/iris_context.h
+++ b/src/gallium/drivers/iris/iris_context.h
@@ -265,6 +265,17 @@ struct iris_cs_prog_key {
struct iris_base_prog_key base;
};
+union iris_any_prog_key {
+ struct iris_base_prog_key base;
+ struct iris_vue_prog_key vue;
+ struct iris_vs_prog_key vs;
+ struct iris_tcs_prog_key tcs;
+ struct iris_tes_prog_key tes;
+ struct iris_gs_prog_key gs;
+ struct iris_fs_prog_key fs;
+ struct iris_cs_prog_key cs;
+};
+
/** @} */
struct iris_depth_stencil_alpha_state;
@@ -389,6 +400,12 @@ struct iris_uncompiled_shader {
/** Size (in bytes) of the local (shared) data passed as kernel inputs */
unsigned kernel_shared_size;
+
+ /** List of iris_compiled_shader variants */
+ struct list_head variants;
+
+ /** Lock for the variants list */
+ simple_mtx_t lock;
};
enum iris_surface_group {
@@ -431,6 +448,12 @@ struct iris_binding_table {
struct iris_compiled_shader {
struct pipe_reference ref;
+ /** Link in the iris_uncompiled_shader::variants list */
+ struct list_head link;
+
+ /** Key for this variant (but not for BLORP programs) */
+ union iris_any_prog_key key;
+
/** Reference to the uploaded assembly. */
struct iris_state_ref assembly;
@@ -881,7 +904,7 @@ void iris_disk_cache_store(struct disk_cache *cache,
uint32_t prog_key_size);
struct iris_compiled_shader *
iris_disk_cache_retrieve(struct iris_context *ice,
- const struct iris_uncompiled_shader *ish,
+ struct iris_uncompiled_shader *ish,
const void *prog_key,
uint32_t prog_key_size);
@@ -894,6 +917,7 @@ struct iris_compiled_shader *iris_find_cached_shader(struct iris_context *ice,
uint32_t key_size,
const void *key);
struct iris_compiled_shader *iris_upload_shader(struct iris_context *ice,
+ struct iris_uncompiled_shader *,
enum iris_program_cache_id,
uint32_t key_size,
const void *key,
@@ -905,9 +929,6 @@ struct iris_compiled_shader *iris_upload_shader(struct iris_context *ice,
unsigned kernel_input_size,
unsigned num_cbufs,
const struct iris_binding_table *bt);
-const void *iris_find_previous_compile(const struct iris_context *ice,
- enum iris_program_cache_id cache_id,
- unsigned program_string_id);
void iris_delete_shader_variant(struct iris_compiled_shader *shader);
static inline void
diff --git a/src/gallium/drivers/iris/iris_disk_cache.c b/src/gallium/drivers/iris/iris_disk_cache.c
index 6e8abe20ce3..cbf5e7e6525 100644
--- a/src/gallium/drivers/iris/iris_disk_cache.c
+++ b/src/gallium/drivers/iris/iris_disk_cache.c
@@ -143,7 +143,7 @@ static const enum iris_program_cache_id cache_id_for_stage[] = {
*/
struct iris_compiled_shader *
iris_disk_cache_retrieve(struct iris_context *ice,
- const struct iris_uncompiled_shader *ish,
+ struct iris_uncompiled_shader *ish,
const void *prog_key,
uint32_t key_size)
{
@@ -245,7 +245,7 @@ iris_disk_cache_retrieve(struct iris_context *ice,
* return it to the caller.
*/
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, cache_id, key_size, prog_key, assembly,
+ iris_upload_shader(ice, ish, cache_id, key_size, prog_key, assembly,
prog_data, so_decls, system_values,
num_system_values, kernel_input_size, num_cbufs, &bt);
diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c
index c53b808b600..47559b91dbd 100644
--- a/src/gallium/drivers/iris/iris_program.c
+++ b/src/gallium/drivers/iris/iris_program.c
@@ -1025,13 +1025,9 @@ iris_debug_recompile(struct iris_context *ice,
struct iris_uncompiled_shader *ish,
const struct brw_base_prog_key *key)
{
- if (!ish)
- return;
-
- if (!ish->compiled_once) {
- ish->compiled_once = true;
+ if (!ish || list_is_empty(&ish->variants)
+ || list_is_singular(&ish->variants))
return;
- }
struct iris_screen *screen = (struct iris_screen *) ice->ctx.screen;
const struct gen_device_info *devinfo = &screen->devinfo;
@@ -1043,8 +1039,9 @@ iris_debug_recompile(struct iris_context *ice,
info->name ? info->name : "(no identifier)",
info->label ? info->label : "");
- const void *old_iris_key =
- iris_find_previous_compile(ice, info->stage, key->program_string_id);
+ struct iris_compiled_shader *shader =
+ list_first_entry(&ish->variants, struct iris_compiled_shader, link);
+ const void *old_iris_key = &shader->key;
union brw_any_prog_key old_key;
@@ -1110,6 +1107,49 @@ last_vue_stage(struct iris_context *ice)
return MESA_SHADER_VERTEX;
}
+static inline struct iris_compiled_shader *
+find_variant(const struct iris_screen *screen,
+ struct iris_uncompiled_shader *ish,
+ const void *key, unsigned key_size)
+{
+ struct list_head *start = ish->variants.next;
+
+ if (screen->precompile) {
+ /* Check the first list entry. There will always be at least one
+ * variant in the list (most likely the precompile variant), and
+ * other contexts only append new variants, so we can safely check
+ * it without locking, saving that cost in the common case.
+ */
+ struct iris_compiled_shader *first =
+ list_first_entry(&ish->variants, struct iris_compiled_shader, link);
+
+ if (memcmp(&first->key, key, key_size) == 0)
+ return first;
+
+ /* Skip this one in the loop below */
+ start = first->link.next;
+ }
+
+ struct iris_compiled_shader *variant = NULL;
+
+ /* If it doesn't match, we have to walk the list; other contexts may be
+ * concurrently appending shaders to it, so we need to lock here.
+ */
+ simple_mtx_lock(&ish->lock);
+
+ list_for_each_entry_from(struct iris_compiled_shader, v, start,
+ &ish->variants, link) {
+ if (memcmp(&v->key, key, key_size) == 0) {
+ variant = v;
+ break;
+ }
+ }
+
+ simple_mtx_unlock(&ish->lock);
+
+ return variant;
+}
+
/**
* Compile a vertex shader, and upload the assembly.
*/
@@ -1176,7 +1216,7 @@ iris_compile_vs(struct iris_context *ice,
&vue_prog_data->vue_map);
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_VS, sizeof(*key), key, program,
+ iris_upload_shader(ice, ish, IRIS_CACHE_VS, sizeof(*key), key, program,
prog_data, so_decls, system_values, num_system_values,
0, num_cbufs, &bt);
@@ -1204,7 +1244,7 @@ iris_update_compiled_vs(struct iris_context *ice)
struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_VS];
struct iris_compiled_shader *shader =
- iris_find_cached_shader(ice, IRIS_CACHE_VS, sizeof(key), &key);
+ find_variant(screen, ish, &key, sizeof(key));
if (!shader)
shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
@@ -1360,7 +1400,7 @@ iris_compile_tcs(struct iris_context *ice,
iris_debug_recompile(ice, ish, &brw_key.base);
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_TCS, sizeof(*key), key, program,
+ iris_upload_shader(ice, ish, IRIS_CACHE_TCS, sizeof(*key), key, program,
prog_data, NULL, system_values, num_system_values,
0, num_cbufs, &bt);
@@ -1403,6 +1443,7 @@ iris_update_compiled_tcs(struct iris_context *ice)
struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_TCS];
struct iris_compiled_shader *shader =
+ tcs ? find_variant(screen, tcs, &key, sizeof(key)) :
iris_find_cached_shader(ice, IRIS_CACHE_TCS, sizeof(key), &key);
if (tcs && !shader)
@@ -1489,7 +1530,7 @@ iris_compile_tes(struct iris_context *ice,
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_TES, sizeof(*key), key, program,
+ iris_upload_shader(ice, ish, IRIS_CACHE_TES, sizeof(*key), key, program,
prog_data, so_decls, system_values, num_system_values,
0, num_cbufs, &bt);
@@ -1518,7 +1559,7 @@ iris_update_compiled_tes(struct iris_context *ice)
struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_TES];
struct iris_compiled_shader *shader =
- iris_find_cached_shader(ice, IRIS_CACHE_TES, sizeof(key), &key);
+ find_variant(screen, ish, &key, sizeof(key));
if (!shader)
shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
@@ -1610,7 +1651,7 @@ iris_compile_gs(struct iris_context *ice,
&vue_prog_data->vue_map);
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_GS, sizeof(*key), key, program,
+ iris_upload_shader(ice, ish, IRIS_CACHE_GS, sizeof(*key), key, program,
prog_data, so_decls, system_values, num_system_values,
0, num_cbufs, &bt);
@@ -1639,8 +1680,7 @@ iris_update_compiled_gs(struct iris_context *ice)
struct iris_gs_prog_key key = { KEY_ID(vue.base) };
screen->vtbl.populate_gs_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
- shader =
- iris_find_cached_shader(ice, IRIS_CACHE_GS, sizeof(key), &key);
+ shader = find_variant(screen, ish, &key, sizeof(key));
if (!shader)
shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
@@ -1726,7 +1766,7 @@ iris_compile_fs(struct iris_context *ice,
iris_debug_recompile(ice, ish, &brw_key.base);
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_FS, sizeof(*key), key, program,
+ iris_upload_shader(ice, ish, IRIS_CACHE_FS, sizeof(*key), key, program,
prog_data, NULL, system_values, num_system_values,
0, num_cbufs, &bt);
@@ -1756,7 +1796,7 @@ iris_update_compiled_fs(struct iris_context *ice)
struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_FS];
struct iris_compiled_shader *shader =
- iris_find_cached_shader(ice, IRIS_CACHE_FS, sizeof(key), &key);
+ find_variant(screen, ish, &key, sizeof(key));
if (!shader)
shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
@@ -1978,7 +2018,7 @@ iris_compile_cs(struct iris_context *ice,
iris_debug_recompile(ice, ish, &brw_key.base);
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_CS, sizeof(*key), key, program,
+ iris_upload_shader(ice, ish, IRIS_CACHE_CS, sizeof(*key), key, program,
prog_data, NULL, system_values, num_system_values,
ish->kernel_input_size, num_cbufs, &bt);
@@ -2001,7 +2041,7 @@ iris_update_compiled_cs(struct iris_context *ice)
struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_CS];
struct iris_compiled_shader *shader =
- iris_find_cached_shader(ice, IRIS_CACHE_CS, sizeof(key), &key);
+ find_variant(screen, ish, &key, sizeof(key));
if (!shader)
shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
@@ -2142,6 +2182,9 @@ iris_create_uncompiled_shader(struct pipe_context *ctx,
if (!ish)
return NULL;
+ list_inithead(&ish->variants);
+ simple_mtx_init(&ish->lock, mtx_plain);
+
NIR_PASS(ish->needs_edge_flag, nir, iris_fix_edge_flags);
brw_preprocess_nir(screen->compiler, nir, NULL);
@@ -2415,6 +2458,16 @@ iris_delete_shader_state(struct pipe_context *ctx, void *state, gl_shader_stage
ice->state.stage_dirty |= IRIS_STAGE_DIRTY_UNCOMPILED_VS << stage;
}
+ /* No need to take ish->lock; we hold the last reference to ish */
+ list_for_each_entry_safe(struct iris_compiled_shader, shader,
+ &ish->variants, link) {
+ list_del(&shader->link);
+
+ iris_shader_variant_reference(&shader, NULL);
+ }
+
+ simple_mtx_destroy(&ish->lock);
+
ralloc_free(ish->nir);
free(ish);
}
diff --git a/src/gallium/drivers/iris/iris_program_cache.c b/src/gallium/drivers/iris/iris_program_cache.c
index 15f524ff628..adb0b38c2e0 100644
--- a/src/gallium/drivers/iris/iris_program_cache.c
+++ b/src/gallium/drivers/iris/iris_program_cache.c
@@ -99,23 +99,6 @@ iris_find_cached_shader(struct iris_context *ice,
return entry ? entry->data : NULL;
}
-const void *
-iris_find_previous_compile(const struct iris_context *ice,
- enum iris_program_cache_id cache_id,
- unsigned program_string_id)
-{
- hash_table_foreach(ice->shaders.cache, entry) {
- const struct keybox *keybox = entry->key;
- const struct brw_base_prog_key *key = (const void *)keybox->data;
- if (keybox->cache_id == cache_id &&
- key->program_string_id == program_string_id) {
- return keybox->data;
- }
- }
-
- return NULL;
-}
-
void
iris_delete_shader_variant(struct iris_compiled_shader *shader)
{
@@ -125,6 +108,7 @@ iris_delete_shader_variant(struct iris_compiled_shader *shader)
struct iris_compiled_shader *
iris_upload_shader(struct iris_context *ice,
+ struct iris_uncompiled_shader *ish,
enum iris_program_cache_id cache_id,
uint32_t key_size,
const void *key,
@@ -138,9 +122,10 @@ iris_upload_shader(struct iris_context *ice,
const struct iris_binding_table *bt)
{
struct hash_table *cache = ice->shaders.cache;
+ void *mem_ctx = ish ? NULL : (void *) cache;
struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
struct iris_compiled_shader *shader =
- rzalloc_size(cache, sizeof(struct iris_compiled_shader) +
+ rzalloc_size(mem_ctx, sizeof(struct iris_compiled_shader) +
screen->vtbl.derived_program_state_size(cache_id));
pipe_reference_init(&shader->ref, 1);
@@ -187,8 +172,33 @@ iris_upload_shader(struct iris_context *ice,
/* Store the 3DSTATE shader packets and other derived state. */
screen->vtbl.store_derived_program_state(ice, cache_id, shader);
- struct keybox *keybox = make_keybox(shader, cache_id, key, key_size);
- _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
+ if (ish) {
+ assert(key_size <= sizeof(union iris_any_prog_key));
+ memcpy(&shader->key, key, key_size);
+
+ simple_mtx_lock(&ish->lock);
+
+ /* While unlikely, it's possible that another thread concurrently
+ * compiled the same variant. Make sure no one beat us to it; if
+ * they did, return the existing one and discard our new one.
+ */
+ list_for_each_entry(struct iris_compiled_shader, existing,
+ &ish->variants, link) {
+ if (memcmp(&existing->key, key, key_size) == 0) {
+ iris_delete_shader_variant(shader);
+ simple_mtx_unlock(&ish->lock);
+ return existing;
+ }
+ }
+
+ /* Append our new variant to the shader's variant list. */
+ list_addtail(&shader->link, &ish->variants);
+
+ simple_mtx_unlock(&ish->lock);
+ } else {
+ struct keybox *keybox = make_keybox(shader, cache_id, key, key_size);
+ _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
+ }
return shader;
}
@@ -236,7 +246,7 @@ iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
memset(&bt, 0, sizeof(bt));
struct iris_compiled_shader *shader =
- iris_upload_shader(ice, IRIS_CACHE_BLORP, key_size, key, kernel,
+ iris_upload_shader(ice, NULL, IRIS_CACHE_BLORP, key_size, key, kernel,
prog_data, NULL, NULL, 0, 0, 0, &bt);
struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
More information about the mesa-commit
mailing list