[Mesa-dev] [PATCH 2/6] amd/rtld: layout and relocate LDS symbols
Nicolai Hähnle
nhaehnle at gmail.com
Sat May 4 13:39:18 UTC 2019
From: Nicolai Hähnle <nicolai.haehnle at amd.com>
Upcoming changes to LLVM will emit LDS objects as symbols in the ELF
symbol table, with relocations that will be resolved with this change.
Callers will also be able to define LDS symbols that are shared between
shader parts. This will be used by radeonsi for the ESGS ring in gfx9+
merged shaders.
---
src/amd/common/ac_rtld.c | 210 ++++++++++++++++--
src/amd/common/ac_rtld.h | 39 +++-
src/gallium/drivers/radeonsi/si_compute.c | 9 +-
src/gallium/drivers/radeonsi/si_debug.c | 22 +-
src/gallium/drivers/radeonsi/si_shader.c | 61 +++--
src/gallium/drivers/radeonsi/si_shader.h | 5 +-
.../drivers/radeonsi/si_state_shaders.c | 2 +-
7 files changed, 296 insertions(+), 52 deletions(-)
diff --git a/src/amd/common/ac_rtld.c b/src/amd/common/ac_rtld.c
index 4e0468d2062..3df7b3ba51f 100644
--- a/src/amd/common/ac_rtld.c
+++ b/src/amd/common/ac_rtld.c
@@ -24,25 +24,31 @@
#include "ac_rtld.h"
#include <gelf.h>
#include <libelf.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "ac_binary.h"
+#include "ac_gpu_info.h"
+#include "util/u_dynarray.h"
#include "util/u_math.h"
// Old distributions may not have this enum constant
#define MY_EM_AMDGPU 224
+#ifndef STT_AMDGPU_LDS
+#define STT_AMDGPU_LDS 13
+#endif
+
#ifndef R_AMDGPU_NONE
#define R_AMDGPU_NONE 0
#define R_AMDGPU_ABS32_LO 1
#define R_AMDGPU_ABS32_HI 2
#define R_AMDGPU_ABS64 3
#define R_AMDGPU_REL32 4
#define R_AMDGPU_REL64 5
#define R_AMDGPU_ABS32 6
#define R_AMDGPU_GOTPCREL 7
#define R_AMDGPU_GOTPCREL32_LO 8
@@ -97,41 +103,155 @@ static void report_elf_errorf(const char *fmt, ...) PRINTFLIKE(1, 2);
static void report_elf_errorf(const char *fmt, ...)
{
va_list va;
va_start(va, fmt);
report_erroraf(fmt, va);
va_end(va);
fprintf(stderr, "ELF error: %s\n", elf_errmsg(elf_errno()));
}
+/**
+ * Find a symbol in a dynarray of struct ac_rtld_symbol by \p name and shader
+ * \p part_idx.
+ */
+static const struct ac_rtld_symbol *find_symbol(const struct util_dynarray *symbols,
+ const char *name, unsigned part_idx)
+{
+ util_dynarray_foreach(symbols, struct ac_rtld_symbol, symbol) {
+ if ((symbol->part_idx == ~0u || symbol->part_idx == part_idx) &&
+ !strcmp(name, symbol->name))
+ return symbol;
+ }
+ return 0;
+}
+
+static int compare_symbol_by_align(const void *lhsp, const void *rhsp)
+{
+ const struct ac_rtld_symbol *lhs = lhsp;
+ const struct ac_rtld_symbol *rhs = rhsp;
+ if (rhs->align > lhs->align)
+ return -1;
+ if (rhs->align < lhs->align)
+ return 1;
+ return 0;
+}
+
+/**
+ * Sort the given symbol list by decreasing alignment and assign offsets.
+ */
+static bool layout_symbols(struct ac_rtld_symbol *symbols, unsigned num_symbols,
+ uint64_t *ptotal_size)
+{
+ qsort(symbols, num_symbols, sizeof(*symbols), compare_symbol_by_align);
+
+ uint64_t total_size = *ptotal_size;
+
+ for (unsigned i = 0; i < num_symbols; ++i) {
+ struct ac_rtld_symbol *s = &symbols[i];
+ assert(util_is_power_of_two_nonzero(s->align));
+
+ total_size = align64(total_size, s->align);
+ s->offset = total_size;
+
+ if (total_size + s->size < total_size) {
+ report_errorf("%s: size overflow", __FUNCTION__);
+ return false;
+ }
+
+ total_size += s->size;
+ }
+
+ *ptotal_size = total_size;
+ return true;
+}
+
+/**
+ * Read LDS symbols from the given \p section of the ELF of \p part and append
+ * them to the LDS symbols list.
+ *
+ * Shared LDS symbols are filtered out.
+ */
+static bool read_private_lds_symbols(struct ac_rtld_binary *binary,
+ unsigned part_idx,
+ Elf_Scn *section,
+ uint32_t *lds_end_align)
+{
+#define report_elf_if(cond) \
+ do { \
+ if ((cond)) { \
+ report_errorf(#cond); \
+ return false; \
+ } \
+ } while (false)
+
+ struct ac_rtld_part *part = &binary->parts[part_idx];
+ Elf64_Shdr *shdr = elf64_getshdr(section);
+ uint32_t strtabidx = shdr->sh_link;
+ Elf_Data *symbols_data = elf_getdata(section, NULL);
+ report_elf_if(!symbols_data);
+
+ const Elf64_Sym *symbol = symbols_data->d_buf;
+ size_t num_symbols = symbols_data->d_size / sizeof(Elf64_Sym);
+
+ for (size_t j = 0; j < num_symbols; ++j, ++symbol) {
+ if (ELF64_ST_TYPE(symbol->st_info) != STT_AMDGPU_LDS)
+ continue;
+
+ report_elf_if(symbol->st_size > 1u << 29);
+
+ struct ac_rtld_symbol s = {};
+ s.name = elf_strptr(part->elf, strtabidx, symbol->st_name);
+ s.size = symbol->st_size;
+ s.align = MIN2(1u << (symbol->st_other >> 3), 1u << 16);
+ s.part_idx = part_idx;
+
+ if (!strcmp(s.name, "__lds_end")) {
+ report_elf_if(s.size != 0);
+ *lds_end_align = MAX2(*lds_end_align, s.align);
+ continue;
+ }
+
+ const struct ac_rtld_symbol *shared =
+ find_symbol(&binary->lds_symbols, s.name, part_idx);
+ if (shared) {
+ report_elf_if(s.align > shared->align);
+ report_elf_if(s.size > shared->size);
+ continue;
+ }
+
+ util_dynarray_append(&binary->lds_symbols, struct ac_rtld_symbol, s);
+ }
+
+ return true;
+
+#undef report_elf_if
+}
+
/**
* Open a binary consisting of one or more shader parts.
*
* \param binary the uninitialized struct
- * \param num_parts number of shader parts
- * \param elf_ptrs pointers to the in-memory ELF objects for each shader part
- * \param elf_sizes sizes (in bytes) of the in-memory ELF objects
+ * \param i binary opening parameters
*/
-bool ac_rtld_open(struct ac_rtld_binary *binary, unsigned num_parts,
- const char * const *elf_ptrs,
- const uint64_t *elf_sizes)
+bool ac_rtld_open(struct ac_rtld_binary *binary,
+ struct ac_rtld_open_info i)
{
/* One of the libelf implementations
* (http://www.mr511.de/software/english.htm) requires calling
* elf_version() before elf_memory().
*/
elf_version(EV_CURRENT);
memset(binary, 0, sizeof(*binary));
- binary->num_parts = num_parts;
- binary->parts = calloc(sizeof(*binary->parts), num_parts);
+ binary->num_parts = i.num_parts;
+ binary->parts = calloc(sizeof(*binary->parts), i.num_parts);
if (!binary->parts)
return false;
uint64_t pasted_text_size = 0;
uint64_t rx_align = 1;
uint64_t rx_size = 0;
#define report_if(cond) \
do { \
if ((cond)) { \
@@ -140,25 +260,44 @@ bool ac_rtld_open(struct ac_rtld_binary *binary, unsigned num_parts,
} \
} while (false)
#define report_elf_if(cond) \
do { \
if ((cond)) { \
report_elf_errorf(#cond); \
goto fail; \
} \
} while (false)
- /* First pass over all parts: open ELFs and determine the placement of
- * sections in the memory image. */
- for (unsigned i = 0; i < num_parts; ++i) {
- struct ac_rtld_part *part = &binary->parts[i];
- part->elf = elf_memory((char *)elf_ptrs[i], elf_sizes[i]);
+ /* Copy and layout shared LDS symbols. */
+ util_dynarray_resize(&binary->lds_symbols, struct ac_rtld_symbol, i.num_shared_lds_symbols);
+ memcpy(binary->lds_symbols.data, i.shared_lds_symbols, binary->lds_symbols.size);
+
+ util_dynarray_foreach(&binary->lds_symbols, struct ac_rtld_symbol, symbol)
+ symbol->part_idx = ~0u;
+
+ unsigned max_lds_size = i.info->chip_class >= CIK ? 64 * 1024 : 32 * 1024;
+ uint64_t shared_lds_size = 0;
+ if (!layout_symbols(binary->lds_symbols.data, i.num_shared_lds_symbols, &shared_lds_size))
+ goto fail;
+ report_if(shared_lds_size > max_lds_size);
+ binary->lds_size = shared_lds_size;
+
+ /* First pass over all parts: open ELFs, pre-determine the placement of
+ * sections in the memory image, and collect and layout private LDS symbols. */
+ uint32_t lds_end_align = 0;
+
+ for (unsigned part_idx = 0; part_idx < i.num_parts; ++part_idx) {
+ struct ac_rtld_part *part = &binary->parts[part_idx];
+ unsigned part_lds_symbols_begin =
+ util_dynarray_num_elements(&binary->lds_symbols, struct ac_rtld_symbol);
+
+ part->elf = elf_memory((char *)i.elf_ptrs[part_idx], i.elf_sizes[part_idx]);
report_elf_if(!part->elf);
const Elf64_Ehdr *ehdr = elf64_getehdr(part->elf);
report_elf_if(!ehdr);
report_if(ehdr->e_machine != MY_EM_AMDGPU);
size_t section_str_index;
size_t num_shdrs;
report_elf_if(elf_getshdrstrndx(part->elf, §ion_str_index) < 0);
report_elf_if(elf_getshdrnum(part->elf, &num_shdrs) < 0);
@@ -196,33 +335,62 @@ bool ac_rtld_open(struct ac_rtld_binary *binary, unsigned num_parts,
if (s->is_pasted_text) {
s->offset = pasted_text_size;
pasted_text_size += shdr->sh_size;
} else {
rx_align = align(rx_align, sh_align);
rx_size = align(rx_size, sh_align);
s->offset = rx_size;
rx_size += shdr->sh_size;
}
+ } else if (shdr->sh_type == SHT_SYMTAB) {
+ if (!read_private_lds_symbols(binary, part_idx, section, &lds_end_align))
+ goto fail;
}
}
+
+ uint64_t part_lds_size = shared_lds_size;
+ if (!layout_symbols(
+ util_dynarray_element(&binary->lds_symbols, struct ac_rtld_symbol, part_lds_symbols_begin),
+ util_dynarray_num_elements(&binary->lds_symbols, struct ac_rtld_symbol) - part_lds_symbols_begin,
+ &part_lds_size))
+ goto fail;
+ binary->lds_size = MAX2(binary->lds_size, part_lds_size);
}
binary->rx_end_markers = pasted_text_size;
pasted_text_size += 4 * DEBUGGER_NUM_MARKERS;
+ /* __lds_end is a special symbol that points at the end of the memory
+ * occupied by other LDS symbols. Its alignment is taken as the
+ * maximum of its alignment over all shader parts where it occurs.
+ */
+ if (lds_end_align) {
+ binary->lds_size = align(binary->lds_size, lds_end_align);
+
+ struct ac_rtld_symbol *lds_end =
+ util_dynarray_grow(&binary->lds_symbols, struct ac_rtld_symbol, 1);
+ lds_end->name = "__lds_end";
+ lds_end->size = 0;
+ lds_end->align = lds_end_align;
+ lds_end->offset = binary->lds_size;
+ lds_end->part_idx = ~0u;
+ }
+
+ report_elf_if(binary->lds_size > max_lds_size);
+
/* Second pass: Adjust offsets of non-pasted text sections. */
binary->rx_size = pasted_text_size;
binary->rx_size = align(binary->rx_size, rx_align);
- for (unsigned i = 0; i < num_parts; ++i) {
- struct ac_rtld_part *part = &binary->parts[i];
+ for (unsigned part_idx = 0; part_idx < i.num_parts; ++part_idx) {
+ struct ac_rtld_part *part = &binary->parts[part_idx];
size_t num_shdrs;
elf_getshdrnum(part->elf, &num_shdrs);
for (unsigned j = 0; j < num_shdrs; ++j) {
struct ac_rtld_section *s = &part->sections[j];
if (s->is_rx && !s->is_pasted_text)
s->offset += binary->rx_size;
}
}
@@ -239,20 +407,21 @@ fail:
}
void ac_rtld_close(struct ac_rtld_binary *binary)
{
for (unsigned i = 0; i < binary->num_parts; ++i) {
struct ac_rtld_part *part = &binary->parts[i];
free(part->sections);
elf_end(part->elf);
}
+ util_dynarray_fini(&binary->lds_symbols);
free(binary->parts);
binary->parts = NULL;
binary->num_parts = 0;
}
static bool get_section_by_name(struct ac_rtld_part *part, const char *name,
const char **data, uint64_t *nbytes)
{
for (unsigned i = 0; i < part->num_sections; ++i) {
struct ac_rtld_section *s = &part->sections[i];
@@ -318,20 +487,28 @@ bool ac_rtld_read_config(struct ac_rtld_binary *binary,
}
return true;
}
static bool resolve_symbol(const struct ac_rtld_upload_info *u,
unsigned part_idx, const Elf64_Sym *sym,
const char *name, uint64_t *value)
{
if (sym->st_shndx == SHN_UNDEF) {
+ const struct ac_rtld_symbol *lds_sym =
+ find_symbol(&u->binary->lds_symbols, name, part_idx);
+
+ if (lds_sym) {
+ *value = lds_sym->offset;
+ return true;
+ }
+
/* TODO: resolve from other parts */
if (u->get_external_symbol(u->cb_data, name, value))
return true;
report_errorf("symbol %s: unknown", name);
return false;
}
struct ac_rtld_part *part = &u->binary->parts[part_idx];
@@ -498,23 +675,24 @@ bool ac_rtld_upload(struct ac_rtld_upload_info *u)
} \
} while (false)
#define report_elf_if(cond) \
do { \
if ((cond)) { \
report_errorf(#cond); \
return false; \
} \
} while (false)
- /* First pass: upload raw section data. */
+ /* First pass: upload raw section data and lay out private LDS symbols. */
for (unsigned i = 0; i < u->binary->num_parts; ++i) {
struct ac_rtld_part *part = &u->binary->parts[i];
+
Elf_Scn *section = NULL;
while ((section = elf_nextscn(part->elf, section))) {
Elf64_Shdr *shdr = elf64_getshdr(section);
struct ac_rtld_section *s = &part->sections[elf_ndxscn(section)];
if (!s->is_rx)
continue;
report_if(shdr->sh_type != SHT_PROGBITS);
diff --git a/src/amd/common/ac_rtld.h b/src/amd/common/ac_rtld.h
index 17b85deaacc..aaa98297ce6 100644
--- a/src/amd/common/ac_rtld.h
+++ b/src/amd/common/ac_rtld.h
@@ -20,49 +20,82 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef AC_RTLD_H
#define AC_RTLD_H
#include <stdbool.h>
#include <stdint.h>
+#include "util/u_dynarray.h"
+
struct ac_rtld_part;
struct ac_shader_config;
+struct radeon_info;
+
+struct ac_rtld_symbol {
+ const char *name;
+ uint32_t size;
+ uint32_t align;
+ uint64_t offset; /* filled in by ac_rtld_open */
+ unsigned part_idx; /* shader part in which this symbol appears */
+};
/* Lightweight wrapper around underlying ELF objects. */
struct ac_rtld_binary {
/* Required buffer sizes, currently read/executable only. */
uint64_t rx_size;
uint64_t rx_end_markers;
unsigned num_parts;
struct ac_rtld_part *parts;
+
+ struct util_dynarray lds_symbols;
+ uint32_t lds_size;
};
/**
* Callback function type used during upload to resolve external symbols that
* are not defined in any of the ELF binaries available to the linker.
*
* \param cb_data caller-defined data
* \param symbol NUL-terminated symbol name
* \param value to be filled in by the callback
* \return whether the symbol was found successfully
*/
typedef bool (*ac_rtld_get_external_symbol_cb)(
void *cb_data, const char *symbol, uint64_t *value);
-bool ac_rtld_open(struct ac_rtld_binary *binary, unsigned num_parts,
- const char * const *elf_ptrs,
- const uint64_t *elf_sizes);
+/**
+ * Lifetimes of \ref info, in-memory ELF objects, and the names of
+ * \ref shared_lds_symbols must extend until \ref ac_rtld_close is called on
+ * the opened binary.
+ */
+struct ac_rtld_open_info {
+ const struct radeon_info *info;
+
+ unsigned num_parts;
+ const char * const *elf_ptrs; /* in-memory ELF objects of each part */
+ const uint64_t *elf_sizes; /* sizes of corresponding in-memory ELF objects in bytes */
+
+ /* Shared LDS symbols are layouted such that they are accessible from
+ * all shader parts. Non-shared (private) LDS symbols of one part may
+ * overlap private LDS symbols of another shader part.
+ */
+ unsigned num_shared_lds_symbols;
+ const struct ac_rtld_symbol *shared_lds_symbols;
+};
+
+bool ac_rtld_open(struct ac_rtld_binary *binary,
+ struct ac_rtld_open_info i);
void ac_rtld_close(struct ac_rtld_binary *binary);
bool ac_rtld_get_section_by_name(struct ac_rtld_binary *binary, const char *name,
const char **data, uint64_t *nbytes);
bool ac_rtld_read_config(struct ac_rtld_binary *binary,
struct ac_shader_config *config);
struct ac_rtld_upload_info {
diff --git a/src/gallium/drivers/radeonsi/si_compute.c b/src/gallium/drivers/radeonsi/si_compute.c
index e4ef138db33..0dfe16b6afd 100644
--- a/src/gallium/drivers/radeonsi/si_compute.c
+++ b/src/gallium/drivers/radeonsi/si_compute.c
@@ -57,22 +57,25 @@ struct dispatch_packet {
static const amd_kernel_code_t *si_compute_get_code_object(
const struct si_compute *program,
uint64_t symbol_offset)
{
if (!program->use_code_object_v2) {
return NULL;
}
struct ac_rtld_binary rtld;
- if (!ac_rtld_open(&rtld, 1, &program->shader.binary.elf_buffer,
- &program->shader.binary.elf_size))
+ if (!ac_rtld_open(&rtld, (struct ac_rtld_open_info){
+ .info = &program->screen->info,
+ .num_parts = 1,
+ .elf_ptrs = &program->shader.binary.elf_buffer,
+ .elf_sizes = &program->shader.binary.elf_size }))
return NULL;
const amd_kernel_code_t *result = NULL;
const char *text;
size_t size;
if (!ac_rtld_get_section_by_name(&rtld, ".text", &text, &size))
goto out;
if (symbol_offset + sizeof(amd_kernel_code_t) > size)
goto out;
@@ -152,21 +155,21 @@ static void si_create_compute_state_async(void *job, int thread_index)
void *ir_binary = si_get_ir_binary(&sel);
/* Try to load the shader from the shader cache. */
mtx_lock(&sscreen->shader_cache_mutex);
if (ir_binary &&
si_shader_cache_load_shader(sscreen, ir_binary, shader)) {
mtx_unlock(&sscreen->shader_cache_mutex);
- si_shader_dump_stats_for_shader_db(shader, debug);
+ si_shader_dump_stats_for_shader_db(sscreen, shader, debug);
si_shader_dump(sscreen, shader, debug, PIPE_SHADER_COMPUTE,
stderr, true);
if (!si_shader_binary_upload(sscreen, shader, 0))
program->shader.compilation_failed = true;
} else {
mtx_unlock(&sscreen->shader_cache_mutex);
if (!si_shader_create(sscreen, compiler, &program->shader, debug)) {
program->shader.compilation_failed = true;
diff --git a/src/gallium/drivers/radeonsi/si_debug.c b/src/gallium/drivers/radeonsi/si_debug.c
index ae7ac2f0442..1f48a19a633 100644
--- a/src/gallium/drivers/radeonsi/si_debug.c
+++ b/src/gallium/drivers/radeonsi/si_debug.c
@@ -93,21 +93,21 @@ void si_clear_saved_cs(struct radeon_saved_cs *saved)
void si_destroy_saved_cs(struct si_saved_cs *scs)
{
si_clear_saved_cs(&scs->gfx);
si_resource_reference(&scs->trace_buf, NULL);
free(scs);
}
static void si_dump_shader(struct si_screen *sscreen,
enum pipe_shader_type processor,
- const struct si_shader *shader, FILE *f)
+ struct si_shader *shader, FILE *f)
{
if (shader->shader_log)
fwrite(shader->shader_log, shader->shader_log_size, 1, f);
else
si_shader_dump(sscreen, shader, NULL, processor, f, false);
if (shader->bo && sscreen->options.dump_shader_binary) {
unsigned size = shader->bo->b.b.width0;
fprintf(f, "BO: VA=%"PRIx64" Size=%u\n", shader->bo->gpu_address, size);
@@ -901,27 +901,32 @@ struct si_shader_inst {
/**
* Open the given \p binary as \p rtld_binary and split the contained
* disassembly string into instructions and add them to the array
* pointed to by \p instructions, which must be sufficiently large.
*
* Labels are considered to be part of the following instruction.
*
* The caller must keep \p rtld_binary alive as long as \p instructions are
* used and then close it afterwards.
*/
-static void si_add_split_disasm(struct ac_rtld_binary *rtld_binary,
+static void si_add_split_disasm(struct si_screen *screen,
+ struct ac_rtld_binary *rtld_binary,
struct si_shader_binary *binary,
uint64_t *addr,
unsigned *num,
struct si_shader_inst *instructions)
{
- if (!ac_rtld_open(rtld_binary, 1, &binary->elf_buffer, &binary->elf_size))
+ if (!ac_rtld_open(rtld_binary, (struct ac_rtld_open_info){
+ .info = &screen->info,
+ .num_parts = 1,
+ .elf_ptrs = &binary->elf_buffer,
+ .elf_sizes = &binary->elf_size }))
return;
const char *disasm;
uint64_t nbytes;
if (!ac_rtld_get_section_by_name(rtld_binary, ".AMDGPU.disasm",
&disasm, &nbytes))
return;
const char *end = disasm + nbytes;
while (disasm < end) {
@@ -953,20 +958,21 @@ static void si_add_split_disasm(struct ac_rtld_binary *rtld_binary,
* execute them. This is most useful during a GPU hang.
*/
static void si_print_annotated_shader(struct si_shader *shader,
struct ac_wave_info *waves,
unsigned num_waves,
FILE *f)
{
if (!shader)
return;
+ struct si_screen *screen = shader->selector->screen;
uint64_t start_addr = shader->bo->gpu_address;
uint64_t end_addr = start_addr + shader->bo->b.b.width0;
unsigned i;
/* See if any wave executes the shader. */
for (i = 0; i < num_waves; i++) {
if (start_addr <= waves[i].pc && waves[i].pc <= end_addr)
break;
}
if (i == num_waves)
@@ -979,35 +985,35 @@ static void si_print_annotated_shader(struct si_shader *shader,
/* Get the list of instructions.
* Buffer size / 4 is the upper bound of the instruction count.
*/
unsigned num_inst = 0;
uint64_t inst_addr = start_addr;
struct ac_rtld_binary rtld_binaries[5] = {};
struct si_shader_inst *instructions =
calloc(shader->bo->b.b.width0 / 4, sizeof(struct si_shader_inst));
if (shader->prolog) {
- si_add_split_disasm(&rtld_binaries[0], &shader->prolog->binary,
+ si_add_split_disasm(screen, &rtld_binaries[0], &shader->prolog->binary,
&inst_addr, &num_inst, instructions);
}
if (shader->previous_stage) {
- si_add_split_disasm(&rtld_binaries[1], &shader->previous_stage->binary,
+ si_add_split_disasm(screen, &rtld_binaries[1], &shader->previous_stage->binary,
&inst_addr, &num_inst, instructions);
}
if (shader->prolog2) {
- si_add_split_disasm(&rtld_binaries[2], &shader->prolog2->binary,
+ si_add_split_disasm(screen, &rtld_binaries[2], &shader->prolog2->binary,
&inst_addr, &num_inst, instructions);
}
- si_add_split_disasm(&rtld_binaries[3], &shader->binary,
+ si_add_split_disasm(screen, &rtld_binaries[3], &shader->binary,
&inst_addr, &num_inst, instructions);
if (shader->epilog) {
- si_add_split_disasm(&rtld_binaries[4], &shader->epilog->binary,
+ si_add_split_disasm(screen, &rtld_binaries[4], &shader->epilog->binary,
&inst_addr, &num_inst, instructions);
}
fprintf(f, COLOR_YELLOW "%s - annotated disassembly:" COLOR_RESET "\n",
si_get_shader_name(shader, shader->selector->type));
/* Print instructions with annotations. */
for (i = 0; i < num_inst; i++) {
struct si_shader_inst *inst = &instructions[i];
diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c
index 528c34aecba..6968038d4d0 100644
--- a/src/gallium/drivers/radeonsi/si_shader.c
+++ b/src/gallium/drivers/radeonsi/si_shader.c
@@ -4968,21 +4968,22 @@ static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
row = ac_to_integer(&ctx->ac, row);
bit = LLVMBuildLShr(builder, row, address[0], "");
bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
ac_build_kill_if_false(&ctx->ac, bit);
}
/* For the UMR disassembler. */
#define DEBUGGER_END_OF_CODE_MARKER 0xbf9f0000 /* invalid instruction */
#define DEBUGGER_NUM_MARKERS 5
-static bool si_shader_binary_open(const struct si_shader *shader,
+static bool si_shader_binary_open(struct si_screen *screen,
+ struct si_shader *shader,
struct ac_rtld_binary *rtld)
{
const char *part_elfs[5];
uint64_t part_sizes[5];
unsigned num_parts = 0;
#define add_part(shader_or_part) \
if (shader_or_part) { \
part_elfs[num_parts] = (shader_or_part)->binary.elf_buffer; \
part_sizes[num_parts] = (shader_or_part)->binary.elf_size; \
@@ -4990,27 +4991,39 @@ static bool si_shader_binary_open(const struct si_shader *shader,
}
add_part(shader->prolog);
add_part(shader->previous_stage);
add_part(shader->prolog2);
add_part(shader);
add_part(shader->epilog);
#undef add_part
- return ac_rtld_open(rtld, num_parts, part_elfs, part_sizes);
+ bool ok = ac_rtld_open(rtld, (struct ac_rtld_open_info){
+ .info = &screen->info,
+ .num_parts = num_parts,
+ .elf_ptrs = part_elfs,
+ .elf_sizes = part_sizes });
+
+ if (rtld->lds_size > 0) {
+ unsigned alloc_granularity = screen->info.chip_class >= CIK ? 512 : 256;
+ shader->config.lds_size =
+ align(rtld->lds_size, alloc_granularity) / alloc_granularity;
+ }
+
+ return ok;
}
-static unsigned si_get_shader_binary_size(const struct si_shader *shader)
+static unsigned si_get_shader_binary_size(struct si_screen *screen, struct si_shader *shader)
{
struct ac_rtld_binary rtld;
- si_shader_binary_open(shader, &rtld);
+ si_shader_binary_open(screen, shader, &rtld);
return rtld.rx_size;
}
static bool si_get_external_symbol(void *data, const char *name, uint64_t *value)
{
uint64_t *scratch_va = data;
if (!strcmp(scratch_rsrc_dword0_symbol, name)) {
*value = (uint32_t)*scratch_va;
@@ -5028,21 +5041,21 @@ static bool si_get_external_symbol(void *data, const char *name, uint64_t *value
return true;
}
return false;
}
bool si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader,
uint64_t scratch_va)
{
struct ac_rtld_binary binary;
- if (!si_shader_binary_open(shader, &binary))
+ if (!si_shader_binary_open(sscreen, shader, &binary))
return false;
si_resource_reference(&shader->bo, NULL);
shader->bo = si_aligned_buffer_create(&sscreen->b,
sscreen->cpdma_prefetch_writes_memory ?
0 : SI_RESOURCE_FLAG_READ_ONLY,
PIPE_USAGE_IMMUTABLE,
align(binary.rx_size, SI_CPDMA_ALIGNMENT),
256);
if (!shader->bo)
@@ -5060,27 +5073,32 @@ bool si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader
RADEON_TRANSFER_TEMPORARY);
if (!u.rx_ptr)
return false;
bool ok = ac_rtld_upload(&u);
sscreen->ws->buffer_unmap(shader->bo->buf);
return ok;
}
-static void si_shader_dump_disassembly(const struct si_shader_binary *binary,
+static void si_shader_dump_disassembly(struct si_screen *screen,
+ const struct si_shader_binary *binary,
struct pipe_debug_callback *debug,
const char *name, FILE *file)
{
struct ac_rtld_binary rtld_binary;
- if (!ac_rtld_open(&rtld_binary, 1, &binary->elf_buffer, &binary->elf_size))
+ if (!ac_rtld_open(&rtld_binary, (struct ac_rtld_open_info){
+ .info = &screen->info,
+ .num_parts = 1,
+ .elf_ptrs = &binary->elf_buffer,
+ .elf_sizes = &binary->elf_size }))
return;
const char *disasm;
uint64_t nbytes;
if (!ac_rtld_get_section_by_name(&rtld_binary, ".AMDGPU.disasm", &disasm, &nbytes))
goto out;
fprintf(file, "Shader %s disassembly:\n", name);
if (nbytes > INT_MAX) {
@@ -5170,38 +5188,39 @@ static void si_calculate_max_simd_waves(struct si_shader *shader)
max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
/* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
* 16KB makes some SIMDs unoccupied). */
if (lds_per_wave)
max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
shader->max_simd_waves = max_simd_waves;
}
-void si_shader_dump_stats_for_shader_db(const struct si_shader *shader,
+void si_shader_dump_stats_for_shader_db(struct si_screen *screen,
+ struct si_shader *shader,
struct pipe_debug_callback *debug)
{
const struct ac_shader_config *conf = &shader->config;
pipe_debug_message(debug, SHADER_INFO,
"Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
"LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
"Spilled VGPRs: %d PrivMem VGPRs: %d",
conf->num_sgprs, conf->num_vgprs,
- si_get_shader_binary_size(shader),
+ si_get_shader_binary_size(screen, shader),
conf->lds_size, conf->scratch_bytes_per_wave,
shader->max_simd_waves, conf->spilled_sgprs,
conf->spilled_vgprs, shader->private_mem_vgprs);
}
static void si_shader_dump_stats(struct si_screen *sscreen,
- const struct si_shader *shader,
+ struct si_shader *shader,
unsigned processor,
FILE *file,
bool check_debug_option)
{
const struct ac_shader_config *conf = &shader->config;
if (!check_debug_option ||
si_can_dump_shader(sscreen, processor)) {
if (processor == PIPE_SHADER_FRAGMENT) {
fprintf(file, "*** SHADER CONFIG ***\n"
@@ -5217,21 +5236,21 @@ static void si_shader_dump_stats(struct si_screen *sscreen,
"Spilled VGPRs: %d\n"
"Private memory VGPRs: %d\n"
"Code Size: %d bytes\n"
"LDS: %d blocks\n"
"Scratch: %d bytes per wave\n"
"Max Waves: %d\n"
"********************\n\n\n",
conf->num_sgprs, conf->num_vgprs,
conf->spilled_sgprs, conf->spilled_vgprs,
shader->private_mem_vgprs,
- si_get_shader_binary_size(shader),
+ si_get_shader_binary_size(sscreen, shader),
conf->lds_size, conf->scratch_bytes_per_wave,
shader->max_simd_waves);
}
}
const char *si_get_shader_name(const struct si_shader *shader, unsigned processor)
{
switch (processor) {
case PIPE_SHADER_VERTEX:
if (shader->key.as_es)
@@ -5254,21 +5273,21 @@ const char *si_get_shader_name(const struct si_shader *shader, unsigned processo
return "Geometry Shader";
case PIPE_SHADER_FRAGMENT:
return "Pixel Shader";
case PIPE_SHADER_COMPUTE:
return "Compute Shader";
default:
return "Unknown Shader";
}
}
-void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
+void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
struct pipe_debug_callback *debug, unsigned processor,
FILE *file, bool check_debug_option)
{
if (!check_debug_option ||
si_can_dump_shader(sscreen, processor))
si_dump_shader_key(processor, shader, file);
if (!check_debug_option && shader->binary.llvm_ir_string) {
if (shader->previous_stage &&
shader->previous_stage->binary.llvm_ir_string) {
@@ -5281,33 +5300,33 @@ void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
si_get_shader_name(shader, processor));
fprintf(file, "%s\n", shader->binary.llvm_ir_string);
}
if (!check_debug_option ||
(si_can_dump_shader(sscreen, processor) &&
!(sscreen->debug_flags & DBG(NO_ASM)))) {
fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
if (shader->prolog)
- si_shader_dump_disassembly(&shader->prolog->binary,
+ si_shader_dump_disassembly(sscreen, &shader->prolog->binary,
debug, "prolog", file);
if (shader->previous_stage)
- si_shader_dump_disassembly(&shader->previous_stage->binary,
+ si_shader_dump_disassembly(sscreen, &shader->previous_stage->binary,
debug, "previous stage", file);
if (shader->prolog2)
- si_shader_dump_disassembly(&shader->prolog2->binary,
+ si_shader_dump_disassembly(sscreen, &shader->prolog2->binary,
debug, "prolog2", file);
- si_shader_dump_disassembly(&shader->binary, debug, "main", file);
+ si_shader_dump_disassembly(sscreen, &shader->binary, debug, "main", file);
if (shader->epilog)
- si_shader_dump_disassembly(&shader->epilog->binary,
+ si_shader_dump_disassembly(sscreen, &shader->epilog->binary,
debug, "epilog", file);
fprintf(file, "\n");
}
si_shader_dump_stats(sscreen, shader, processor, file,
check_debug_option);
}
static int si_compile_llvm(struct si_screen *sscreen,
struct si_shader_binary *binary,
@@ -5338,21 +5357,25 @@ static int si_compile_llvm(struct si_screen *sscreen,
}
if (!si_replace_shader(count, binary)) {
unsigned r = si_llvm_compile(mod, binary, compiler, debug,
less_optimized);
if (r)
return r;
}
struct ac_rtld_binary rtld;
- if (!ac_rtld_open(&rtld, 1, &binary->elf_buffer, &binary->elf_size))
+ if (!ac_rtld_open(&rtld, (struct ac_rtld_open_info){
+ .info = &sscreen->info,
+ .num_parts = 1,
+ .elf_ptrs = &binary->elf_buffer,
+ .elf_sizes = &binary->elf_size }))
return -1;
bool ok = ac_rtld_read_config(&rtld, conf);
ac_rtld_close(&rtld);
if (!ok)
return -1;
/* Enable 64-bit and 16-bit denormals, because there is no performance
* cost.
*
@@ -6796,21 +6819,21 @@ int si_compile_tgsi_shader(struct si_screen *sscreen,
shader->info.ancillary_vgpr_index = shader->info.num_input_vgprs;
shader->info.num_input_vgprs += 1;
}
if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
shader->info.num_input_vgprs += 1;
if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
shader->info.num_input_vgprs += 1;
}
si_calculate_max_simd_waves(shader);
- si_shader_dump_stats_for_shader_db(shader, debug);
+ si_shader_dump_stats_for_shader_db(sscreen, shader, debug);
return 0;
}
/**
* Create, compile and return a shader part (prolog or epilog).
*
* \param sscreen screen
* \param list list of shader parts of the same category
* \param type shader type
* \param key shader part key
diff --git a/src/gallium/drivers/radeonsi/si_shader.h b/src/gallium/drivers/radeonsi/si_shader.h
index ef9f5c379d3..0d018cef2d2 100644
--- a/src/gallium/drivers/radeonsi/si_shader.h
+++ b/src/gallium/drivers/radeonsi/si_shader.h
@@ -677,24 +677,25 @@ int si_compile_tgsi_shader(struct si_screen *sscreen,
struct pipe_debug_callback *debug);
bool si_shader_create(struct si_screen *sscreen, struct ac_llvm_compiler *compiler,
struct si_shader *shader,
struct pipe_debug_callback *debug);
void si_shader_destroy(struct si_shader *shader);
unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index);
unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index,
unsigned is_varying);
bool si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader,
uint64_t scratch_va);
-void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
+void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
struct pipe_debug_callback *debug, unsigned processor,
FILE *f, bool check_debug_option);
-void si_shader_dump_stats_for_shader_db(const struct si_shader *shader,
+void si_shader_dump_stats_for_shader_db(struct si_screen *screen,
+ struct si_shader *shader,
struct pipe_debug_callback *debug);
void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
unsigned *lds_size);
const char *si_get_shader_name(const struct si_shader *shader, unsigned processor);
void si_shader_binary_clean(struct si_shader_binary *binary);
/* si_shader_nir.c */
void si_nir_scan_shader(const struct nir_shader *nir,
struct tgsi_shader_info *info);
void si_nir_scan_tess_ctrl(const struct nir_shader *nir,
diff --git a/src/gallium/drivers/radeonsi/si_state_shaders.c b/src/gallium/drivers/radeonsi/si_state_shaders.c
index 041ed1c0c62..aec818c23a7 100644
--- a/src/gallium/drivers/radeonsi/si_state_shaders.c
+++ b/src/gallium/drivers/radeonsi/si_state_shaders.c
@@ -2090,21 +2090,21 @@ static void si_init_shader_selector_async(void *job, int thread_index)
if (sel->tokens || sel->nir)
ir_binary = si_get_ir_binary(sel);
/* Try to load the shader from the shader cache. */
mtx_lock(&sscreen->shader_cache_mutex);
if (ir_binary &&
si_shader_cache_load_shader(sscreen, ir_binary, shader)) {
mtx_unlock(&sscreen->shader_cache_mutex);
- si_shader_dump_stats_for_shader_db(shader, debug);
+ si_shader_dump_stats_for_shader_db(sscreen, shader, debug);
} else {
mtx_unlock(&sscreen->shader_cache_mutex);
/* Compile the shader if it hasn't been loaded from the cache. */
if (si_compile_tgsi_shader(sscreen, compiler, shader,
debug) != 0) {
FREE(shader);
FREE(ir_binary);
fprintf(stderr, "radeonsi: can't compile a main shader part\n");
return;
--
2.20.1
More information about the mesa-dev
mailing list