[Beignet] [PATCH V5] Backend: Refine block read/write buffer
Xiuli Pan
xiuli.pan at intel.com
Fri Jun 24 03:07:39 UTC 2016
From: Pan Xiuli <xiuli.pan at intel.com>
Using max to 8 OWORD as read/write size for high profermance.
V4: Reuse tmp for oword read for small and less vector.
V5: Move all tmp in dst
Signed-off-by: Pan Xiuli <xiuli.pan at intel.com>
---
backend/src/backend/gen_context.cpp | 154 ++++++++++++++++++++++++++---
backend/src/backend/gen_encoder.cpp | 6 +-
backend/src/backend/gen_insn_selection.cpp | 86 +++++++++++-----
backend/src/libocl/tmpl/ocl_simd.tmpl.cl | 41 +++-----
backend/src/llvm/llvm_gen_backend.cpp | 46 ++++++---
backend/src/llvm/llvm_gen_ocl_function.hxx | 6 ++
backend/src/llvm/llvm_scalarize.cpp | 12 +++
7 files changed, 264 insertions(+), 87 deletions(-)
diff --git a/backend/src/backend/gen_context.cpp b/backend/src/backend/gen_context.cpp
index 081033a..5ddf084 100644
--- a/backend/src/backend/gen_context.cpp
+++ b/backend/src/backend/gen_context.cpp
@@ -3488,11 +3488,17 @@ namespace gbe
}
void GenContext::emitOBReadInstruction(const SelectionInstruction &insn) {
- const GenRegister dst = ra->genReg(insn.dst(0));
+ const GenRegister dst= GenRegister::retype(ra->genReg(insn.dst(1)), GEN_TYPE_UD);
const GenRegister addr = GenRegister::toUniform(ra->genReg(insn.src(0)), GEN_TYPE_UD);
- GenRegister header = GenRegister::retype(ra->genReg(insn.src(1)), GEN_TYPE_UD);
+ const GenRegister header = GenRegister::retype(ra->genReg(insn.dst(0)), GEN_TYPE_UD);
+ const GenRegister headeraddr = GenRegister::offset(header, 0, 2*4);
+ const uint32_t vec_size = insn.extra.elem;
+ const GenRegister tmp = GenRegister::retype(ra->genReg(insn.dst(1 + vec_size)), GEN_TYPE_UD);
+ const uint32_t simdWidth = p->curr.execWidth;
+ // Make header
p->push();
+ {
// Copy r0 into the header first
p->curr.execWidth = 8;
p->curr.predicate = GEN_PREDICATE_NONE;
@@ -3501,23 +3507,81 @@ namespace gbe
// Update the header with the current address
p->curr.execWidth = 1;
- p->SHR(GenRegister::offset(header, 0, 2*4), addr, GenRegister::immud(4));
+ p->SHR(headeraddr, addr, GenRegister::immud(4));
// Put zero in the general state base address
- p->MOV(GenRegister::offset(header, 0, 5*4), GenRegister::immud(0));
-
+ p->MOV(GenRegister::offset(header, 0, 5 * 4), GenRegister::immud(0));
+ }
p->pop();
- // Now read the data
- p->OBREAD(dst, header, insn.getbti(), insn.extra.elem);
+ // Now read the data, oword block read can only work with simd16 and no mask
+ if (vec_size == 1) {
+ p->push();
+ {
+ p->curr.execWidth = 16;
+ p->curr.noMask = 1;
+ p->OBREAD(dst, header, insn.getbti(), simdWidth / 4);
+ }
+ p->pop();
+ } else if (vec_size == 2) {
+ p->push();
+ {
+ p->curr.execWidth = 16;
+ p->curr.noMask = 1;
+ p->OBREAD(tmp, header, insn.getbti(), simdWidth / 2);
+ }
+ p->pop();
+ p->MOV(ra->genReg(insn.dst(1)), GenRegister::offset(tmp, 0));
+ p->MOV(ra->genReg(insn.dst(2)), GenRegister::offset(tmp, simdWidth / 8));
+ } else if (vec_size == 4 || vec_size == 8) {
+ if (simdWidth == 8) {
+ for (uint32_t i = 0; i < vec_size / 4; i++) {
+ if (i > 0) {
+ p->push();
+ {
+ // Update the address in header
+ p->curr.execWidth = 1;
+ p->ADD(headeraddr, headeraddr, GenRegister::immud(8));
+ }
+ p->pop();
+ }
+ p->push();
+ {
+ p->curr.execWidth = 16;
+ p->curr.noMask = 1;
+ p->OBREAD(tmp, header, insn.getbti(), 8);
+ }
+ p->pop();
+ for (uint32_t j = 0; j < 4; j++)
+ p->MOV(ra->genReg(insn.dst(1 + j + i * 4)), GenRegister::offset(tmp, j));
+ }
+ } else {
+ for (uint32_t i = 0; i < vec_size / 2; i++) {
+ if (i > 0) {
+ p->push();
+ {
+ // Update the address in header
+ p->curr.execWidth = 1;
+ p->ADD(headeraddr, headeraddr, GenRegister::immud(8));
+ }
+ p->pop();
+ }
+ p->OBREAD(tmp, header, insn.getbti(), 8);
+ for (uint32_t j = 0; j < 2; j++)
+ p->MOV(ra->genReg(insn.dst(1 + j + i * 2)), GenRegister::offset(tmp, j*2));
+ }
+ }
+ } else NOT_SUPPORTED;
}
void GenContext::emitOBWriteInstruction(const SelectionInstruction &insn) {
- const GenRegister addr = GenRegister::toUniform(ra->genReg(insn.src(2)), GEN_TYPE_UD);
- GenRegister header;
- if (simdWidth == 8)
- header = GenRegister::retype(ra->genReg(insn.src(0)), GEN_TYPE_UD);
- else
- header = GenRegister::retype(GenRegister::Qn(ra->genReg(insn.src(0)),1), GEN_TYPE_UD);
+ const GenRegister addr = GenRegister::toUniform(ra->genReg(insn.src(0)), GEN_TYPE_UD);
+ const GenRegister header = GenRegister::retype(ra->genReg(insn.dst(0)), GEN_TYPE_UD);
+ const GenRegister headeraddr = GenRegister::offset(header, 0, 2*4);
+ const uint32_t vec_size = insn.extra.elem;
+ const GenRegister tmp = GenRegister::offset(header, 1);
+ const uint32_t simdWidth = p->curr.execWidth;
+ uint32_t tmp_size = simdWidth * vec_size / 8;
+ tmp_size = tmp_size > 4 ? 4 : tmp_size;
p->push();
// Copy r0 into the header first
@@ -3528,14 +3592,72 @@ namespace gbe
// Update the header with the current address
p->curr.execWidth = 1;
- p->SHR(GenRegister::offset(header, 0, 2*4), addr, GenRegister::immud(4));
+ p->SHR(headeraddr, addr, GenRegister::immud(4));
// Put zero in the general state base address
p->MOV(GenRegister::offset(header, 0, 5*4), GenRegister::immud(0));
p->pop();
- // Now write the data
- p->OBWRITE(header, insn.getbti(), insn.extra.elem);
+ // Now write the data, oword block write can only work with simd16 and no mask
+ if (vec_size == 1) {
+ p->MOV(tmp, ra->genReg(insn.src(1)));
+ p->push();
+ {
+ p->curr.execWidth = 16;
+ p->curr.noMask = 1;
+ p->OBWRITE(header, insn.getbti(), simdWidth / 4);
+ }
+ p->pop();
+ } else if (vec_size == 2) {
+ p->MOV(GenRegister::offset(tmp, 0), ra->genReg(insn.src(1))) ;
+ p->MOV(GenRegister::offset(tmp, simdWidth / 8), ra->genReg(insn.src(2))) ;
+ p->push();
+ {
+ p->curr.execWidth = 16;
+ p->curr.noMask = 1;
+ p->OBWRITE(header, insn.getbti(), simdWidth / 2);
+ }
+ p->pop();
+ } else if (vec_size == 4 || vec_size == 8) {
+ if (simdWidth == 8) {
+ for (uint32_t i = 0; i < vec_size / 4; i++) {
+ for (uint32_t j = 0; j < 4; j++)
+ p->MOV(GenRegister::offset(tmp, j), ra->genReg(insn.src(1 + j + i*4))) ;
+ if (i > 0) {
+ p->push();
+ {
+ // Update the address in header
+ p->curr.execWidth = 1;
+ p->ADD(headeraddr, headeraddr, GenRegister::immud(8));
+ }
+ p->pop();
+ }
+ p->push();
+ {
+ p->curr.execWidth = 16;
+ p->curr.noMask = 1;
+ p->OBWRITE(header, insn.getbti(), 8);
+ }
+ p->pop();
+ }
+ } else {
+ for (uint32_t i = 0; i < vec_size / 2; i++) {
+ for (uint32_t j = 0; j < 2; j++)
+ p->MOV(GenRegister::offset(tmp, j * 2), ra->genReg(insn.src(1 + j + i*2))) ;
+ if (i > 0) {
+ p->push();
+ {
+ // Update the address in header
+ p->curr.execWidth = 1;
+ p->ADD(headeraddr, headeraddr, GenRegister::immud(8));
+ }
+ p->pop();
+ }
+ p->OBWRITE(header, insn.getbti(), 8);
+ }
+ }
+ } else NOT_SUPPORTED;
+
}
void GenContext::emitMBReadInstruction(const SelectionInstruction &insn) {
diff --git a/backend/src/backend/gen_encoder.cpp b/backend/src/backend/gen_encoder.cpp
index eb9fbeb..f8c99b2 100644
--- a/backend/src/backend/gen_encoder.cpp
+++ b/backend/src/backend/gen_encoder.cpp
@@ -269,10 +269,10 @@ namespace gbe
{
const GenMessageTarget sfid = GEN_SFID_DATAPORT_DATA;
p->setMessageDescriptor(insn, sfid, msg_length, response_length);
- assert(size == 2 || size == 4);
+ assert(size == 2 || size == 4 || size == 8);
insn->bits3.gen7_oblock_rw.msg_type = msg_type;
insn->bits3.gen7_oblock_rw.bti = bti;
- insn->bits3.gen7_oblock_rw.block_size = size == 2 ? 2 : 3;
+ insn->bits3.gen7_oblock_rw.block_size = size == 2 ? 2 : (size == 4 ? 3 : 4);
insn->bits3.gen7_oblock_rw.header_present = 1;
}
@@ -1261,7 +1261,7 @@ namespace gbe
void GenEncoder::OBREAD(GenRegister dst, GenRegister header, uint32_t bti, uint32_t size) {
GenNativeInstruction *insn = this->next(GEN_OPCODE_SEND);
const uint32_t msg_length = 1;
- const uint32_t response_length = size / 2; // Size is in owords
+ const uint32_t response_length = size / 2; // Size is in regs
this->setHeader(insn);
this->setDst(insn, GenRegister::uw16grf(dst.nr, 0));
this->setSrc0(insn, GenRegister::ud8grf(header.nr, 0));
diff --git a/backend/src/backend/gen_insn_selection.cpp b/backend/src/backend/gen_insn_selection.cpp
index 9a5567d..788a69a 100644
--- a/backend/src/backend/gen_insn_selection.cpp
+++ b/backend/src/backend/gen_insn_selection.cpp
@@ -702,9 +702,9 @@ namespace gbe
void SUBGROUP_OP(uint32_t wg_op, Reg dst, GenRegister src,
GenRegister tmpData1, GenRegister tmpData2);
/*! Oblock read */
- void OBREAD(GenRegister dst, GenRegister addr, GenRegister header, uint32_t bti, uint32_t size);
+ void OBREAD(GenRegister* dsts, uint32_t vec_size, GenRegister addr, GenRegister header, uint32_t bti, GenRegister* tmp, uint32_t tmp_size);
/*! Oblock write */
- void OBWRITE(GenRegister addr, GenRegister value, GenRegister header, uint32_t bti, uint32_t size);
+ void OBWRITE(GenRegister addr, GenRegister* values, uint32_t vec_size, GenRegister header, uint32_t bti, GenRegister* tmp, uint32_t tmp_size);
/*! Media block read */
void MBREAD(GenRegister* dsts, GenRegister coordx, GenRegister coordy, GenRegister header, GenRegister* tmp, uint32_t bti, uint32_t vec_size);
/*! Media block write */
@@ -2027,38 +2027,54 @@ namespace gbe
insn->src(0) = src;
insn->src(1) = tmpData2;
}
- void Selection::Opaque::OBREAD(GenRegister dst,
+ void Selection::Opaque::OBREAD(GenRegister* dsts,
+ uint32_t vec_size,
GenRegister addr,
GenRegister header,
uint32_t bti,
- uint32_t size) {
- SelectionInstruction *insn = this->appendInsn(SEL_OP_OBREAD, 1, 2);
- insn->dst(0) = dst;
+ GenRegister* tmp,
+ uint32_t tmp_size) {
+ SelectionInstruction *insn = this->appendInsn(SEL_OP_OBREAD, 1 + vec_size + tmp_size, 1);
+ SelectionVector *vector = this->appendVector();
+ insn->dst(0) = header;
+ for (uint32_t i = 0; i < vec_size; ++i)
+ insn->dst(1 + i) = dsts[i];
+ for (uint32_t i = 0; i < tmp_size; ++i)
+ insn->dst(1 + i + vec_size) = tmp[i];
insn->src(0) = addr;
- insn->src(1) = header;
insn->setbti(bti);
- insn->extra.elem = size / sizeof(int[4]); // number of owords
+ insn->extra.elem = vec_size; // number of vector size
+
+ // tmp regs for OWORD read dst
+ vector->regNum = tmp_size;
+ vector->reg = &insn->dst(1 + vec_size);
+ vector->offsetID = 1 + vec_size;
+ vector->isSrc = 0;
}
void Selection::Opaque::OBWRITE(GenRegister addr,
- GenRegister value,
+ GenRegister* values,
+ uint32_t vec_size,
GenRegister header,
uint32_t bti,
- uint32_t size) {
- SelectionInstruction *insn = this->appendInsn(SEL_OP_OBWRITE, 0, 3);
+ GenRegister* tmp,
+ uint32_t tmp_size) {
+ SelectionInstruction *insn = this->appendInsn(SEL_OP_OBWRITE, tmp_size + 1, vec_size + 1);
SelectionVector *vector = this->appendVector();
- insn->src(0) = header;
- insn->src(1) = value;
- insn->src(2) = addr;
- insn->state = this->curr;
+ insn->src(0) = addr;
+ for (uint32_t i = 0; i < vec_size; ++i)
+ insn->src(i + 1) = values[i];
+ insn->dst(0) = header;
+ for (uint32_t i = 0; i < tmp_size; ++i)
+ insn->dst(i + 1) = tmp[i];
insn->setbti(bti);
- insn->extra.elem = size / sizeof(int[4]); // number of owords
+ insn->extra.elem = vec_size; // number of vector_size
- // We need to put the header and the data together
- vector->regNum = 2;
- vector->reg = &insn->src(0);
+ // tmp regs for OWORD read dst
+ vector->regNum = tmp_size + 1;
+ vector->reg = &insn->dst(0);
vector->offsetID = 0;
- vector->isSrc = 1;
+ vector->isSrc = 0;
}
void Selection::Opaque::MBREAD(GenRegister* dsts,
@@ -4113,10 +4129,19 @@ extern bool OCL_DEBUGINFO; // first defined by calling BVAR in program.cpp
ir::BTI bti) const
{
using namespace ir;
- const GenRegister header = sel.selReg(sel.reg(FAMILY_DWORD), TYPE_U32);
- const GenRegister value = sel.selReg(insn.getValue(0), TYPE_U32);
+ const uint32_t vec_size = insn.getValueNum();
const uint32_t simdWidth = sel.ctx.getSimdWidth();
- sel.OBREAD(value, address, header, bti.imm, simdWidth * sizeof(int));
+ const GenRegister header = GenRegister::retype(GenRegister::f8grf(sel.reg(FAMILY_DWORD)), TYPE_U32);
+ vector<GenRegister> valuesVec;
+ for(uint32_t i = 0; i < vec_size; i++)
+ valuesVec.push_back(sel.selReg(insn.getValue(i), TYPE_U32));
+ // check tmp_size for OWORD read need, max 8 OWROD thus 4 regs
+ uint32_t tmp_size = simdWidth * vec_size / 8;
+ tmp_size = tmp_size > 4 ? 4 : tmp_size;
+ vector<GenRegister> tmpVec;
+ for(uint32_t i = 0; i < tmp_size; i++)
+ tmpVec.push_back(GenRegister::retype(GenRegister::f8grf(sel.reg(FAMILY_DWORD)), TYPE_U32));
+ sel.OBREAD(&valuesVec[0], vec_size, address, header, bti.imm, &tmpVec[0], tmp_size);
}
// check whether all binded table index point to constant memory
@@ -4289,10 +4314,19 @@ extern bool OCL_DEBUGINFO; // first defined by calling BVAR in program.cpp
ir::BTI bti) const
{
using namespace ir;
- const GenRegister header = sel.selReg(sel.reg(FAMILY_DWORD), TYPE_U32);
- const GenRegister value = sel.selReg(insn.getValue(0), TYPE_U32);
+ const uint32_t vec_size = insn.getValueNum();
const uint32_t simdWidth = sel.ctx.getSimdWidth();
- sel.OBWRITE(address, value, header, bti.imm, simdWidth * sizeof(int));
+ const GenRegister header = GenRegister::retype(GenRegister::f8grf(sel.reg(FAMILY_DWORD)), TYPE_U32);
+ vector<GenRegister> valuesVec;
+ for(uint32_t i = 0; i < vec_size; i++)
+ valuesVec.push_back(sel.selReg(insn.getValue(i), TYPE_U32));
+ // check tmp_size for OWORD write need, max 8 OWROD thus 4 regs
+ uint32_t tmp_size = simdWidth * vec_size / 8;
+ tmp_size = tmp_size > 4 ? 4 : tmp_size;
+ vector<GenRegister> tmpVec;
+ for(uint32_t i = 0; i < tmp_size; i++)
+ tmpVec.push_back(GenRegister::retype(GenRegister::f8grf(sel.reg(FAMILY_DWORD)), TYPE_U32));
+ sel.OBWRITE(address, &valuesVec[0], vec_size, header, bti.imm, &tmpVec[0], tmp_size);
}
virtual bool emit(Selection::Opaque &sel, SelectionDAG &dag) const
diff --git a/backend/src/libocl/tmpl/ocl_simd.tmpl.cl b/backend/src/libocl/tmpl/ocl_simd.tmpl.cl
index 5d3d0bb..b066502 100644
--- a/backend/src/libocl/tmpl/ocl_simd.tmpl.cl
+++ b/backend/src/libocl/tmpl/ocl_simd.tmpl.cl
@@ -134,63 +134,46 @@ RANGE_OP(scan_exclusive, max, double, true)
#undef RANGE_OP
PURE CONST uint __gen_ocl_sub_group_block_read_mem(const global uint* p);
+PURE CONST uint2 __gen_ocl_sub_group_block_read_mem2(const global uint* p);
+PURE CONST uint4 __gen_ocl_sub_group_block_read_mem4(const global uint* p);
+PURE CONST uint8 __gen_ocl_sub_group_block_read_mem8(const global uint* p);
OVERLOADABLE uint intel_sub_group_block_read(const global uint* p)
{
return __gen_ocl_sub_group_block_read_mem(p);
}
OVERLOADABLE uint2 intel_sub_group_block_read2(const global uint* p)
{
- return (uint2)(intel_sub_group_block_read(p),
- intel_sub_group_block_read(p + get_simd_size()));
+ return __gen_ocl_sub_group_block_read_mem2(p);
}
OVERLOADABLE uint4 intel_sub_group_block_read4(const global uint* p)
{
- return (uint4)(intel_sub_group_block_read(p),
- intel_sub_group_block_read(p + get_simd_size()),
- intel_sub_group_block_read(p + get_simd_size() * 2),
- intel_sub_group_block_read(p + get_simd_size() * 3));
+ return __gen_ocl_sub_group_block_read_mem4(p);
}
OVERLOADABLE uint8 intel_sub_group_block_read8(const global uint* p)
{
- return (uint8)(intel_sub_group_block_read(p),
- intel_sub_group_block_read(p + get_simd_size()),
- intel_sub_group_block_read(p + get_simd_size() * 2),
- intel_sub_group_block_read(p + get_simd_size() * 3),
- intel_sub_group_block_read(p + get_simd_size() * 4),
- intel_sub_group_block_read(p + get_simd_size() * 5),
- intel_sub_group_block_read(p + get_simd_size() * 6),
- intel_sub_group_block_read(p + get_simd_size() * 7));
+ return __gen_ocl_sub_group_block_read_mem8(p);
}
-
void __gen_ocl_sub_group_block_write_mem(const global uint* p, uint data);
+void __gen_ocl_sub_group_block_write_mem2(const global uint* p, uint2 data);
+void __gen_ocl_sub_group_block_write_mem4(const global uint* p, uint4 data);
+void __gen_ocl_sub_group_block_write_mem8(const global uint* p, uint8 data);
OVERLOADABLE void intel_sub_group_block_write(const global uint* p, uint data)
{
__gen_ocl_sub_group_block_write_mem(p, data);
}
OVERLOADABLE void intel_sub_group_block_write2(const global uint* p, uint2 data)
{
- intel_sub_group_block_write(p, data.s0);
- intel_sub_group_block_write(p + get_simd_size(), data.s1);
+ __gen_ocl_sub_group_block_write_mem2(p, data);
}
OVERLOADABLE void intel_sub_group_block_write4(const global uint* p,uint4 data)
{
- intel_sub_group_block_write(p, data.s0);
- intel_sub_group_block_write(p + get_simd_size(), data.s1);
- intel_sub_group_block_write(p + get_simd_size() * 2, data.s2);
- intel_sub_group_block_write(p + get_simd_size() * 3, data.s3);
+ __gen_ocl_sub_group_block_write_mem4(p, data);
}
OVERLOADABLE void intel_sub_group_block_write8(const global uint* p,uint8 data)
{
- intel_sub_group_block_write(p, data.s0);
- intel_sub_group_block_write(p + get_simd_size(), data.s1);
- intel_sub_group_block_write(p + get_simd_size() * 2, data.s2);
- intel_sub_group_block_write(p + get_simd_size() * 3, data.s3);
- intel_sub_group_block_write(p + get_simd_size() * 4, data.s4);
- intel_sub_group_block_write(p + get_simd_size() * 5, data.s5);
- intel_sub_group_block_write(p + get_simd_size() * 6, data.s6);
- intel_sub_group_block_write(p + get_simd_size() * 7, data.s7);
+ __gen_ocl_sub_group_block_write_mem8(p, data);
}
PURE CONST uint __gen_ocl_sub_group_block_read_image(image2d_t p, int x, int y);
diff --git a/backend/src/llvm/llvm_gen_backend.cpp b/backend/src/llvm/llvm_gen_backend.cpp
index 419f585..074391f 100644
--- a/backend/src/llvm/llvm_gen_backend.cpp
+++ b/backend/src/llvm/llvm_gen_backend.cpp
@@ -698,7 +698,7 @@ namespace gbe
// Emit subgroup instructions
void emitSubGroupInst(CallInst &I, CallSite &CS, ir::WorkGroupOps opcode);
// Emit subgroup instructions
- void emitBlockReadWriteMemInst(CallInst &I, CallSite &CS, bool isWrite);
+ void emitBlockReadWriteMemInst(CallInst &I, CallSite &CS, bool isWrite, uint8_t vec_size);
void emitBlockReadWriteImageInst(CallInst &I, CallSite &CS, bool isWrite, uint8_t vec_size);
uint8_t appendSampler(CallSite::arg_iterator AI);
@@ -3726,6 +3726,9 @@ namespace gbe
case GEN_OCL_SUB_GROUP_SCAN_INCLUSIVE_MIN:
case GEN_OCL_LRP:
case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM:
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM2:
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM4:
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM8:
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE:
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE2:
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE4:
@@ -3747,6 +3750,9 @@ namespace gbe
case GEN_OCL_STORE_PROFILING:
case GEN_OCL_DEBUGWAIT:
case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM:
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM2:
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM4:
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM8:
case GEN_OCL_SUB_GROUP_BLOCK_WRITE_IMAGE:
case GEN_OCL_SUB_GROUP_BLOCK_WRITE_IMAGE2:
case GEN_OCL_SUB_GROUP_BLOCK_WRITE_IMAGE4:
@@ -3945,13 +3951,12 @@ namespace gbe
GBE_ASSERT(AI == AE);
}
- void GenWriter::emitBlockReadWriteMemInst(CallInst &I, CallSite &CS, bool isWrite) {
+ void GenWriter::emitBlockReadWriteMemInst(CallInst &I, CallSite &CS, bool isWrite, uint8_t vec_size) {
CallSite::arg_iterator AI = CS.arg_begin();
CallSite::arg_iterator AE = CS.arg_end();
GBE_ASSERT(AI != AE);
Value *llvmPtr = *(AI++);
- Value *llvmValues;
ir::AddressSpace addrSpace = addressSpaceLLVMToGen(llvmPtr->getType()->getPointerAddressSpace());
GBE_ASSERT(addrSpace == ir::MEM_GLOBAL);
ir::Register pointer = this->getRegister(llvmPtr);
@@ -3986,15 +3991,18 @@ namespace gbe
GBE_ASSERT(AM != ir::AM_DynamicBti);
if(isWrite){
- llvmValues = *(AI++);
- const ir::Register values = getRegister(llvmValues);
- const ir::Tuple tuple = ctx.arrayTuple(&values, 1);
- ctx.STORE(type, tuple, ptr, addrSpace, 1, true, AM, SurfaceIndex, true);
+ Value *llvmValues = *(AI++);
+ vector<ir::Register> srcTupleData;
+ for(int i = 0;i < vec_size; i++)
+ srcTupleData.push_back(getRegister(llvmValues, i));
+ const ir::Tuple tuple = ctx.arrayTuple(&srcTupleData[0], vec_size);
+ ctx.STORE(type, tuple, ptr, addrSpace, vec_size, true, AM, SurfaceIndex, true);
} else {
- llvmValues = &I;
- const ir::Register values = getRegister(llvmValues);
- const ir::Tuple tuple = ctx.arrayTuple(&values, 1);
- ctx.LOAD(type, tuple, ptr, addrSpace, 1, true, AM, SurfaceIndex, true);
+ vector<ir::Register> dstTupleData;
+ for(int i = 0;i < vec_size; i++)
+ dstTupleData.push_back(getRegister(&I, i));
+ const ir::Tuple tuple = ctx.arrayTuple(&dstTupleData[0], vec_size);
+ ctx.LOAD(type, tuple, ptr, addrSpace, vec_size, true, AM, SurfaceIndex, true);
}
GBE_ASSERT(AI == AE);
@@ -4858,9 +4866,21 @@ namespace gbe
break;
}
case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM:
- this->emitBlockReadWriteMemInst(I, CS, false); break;
+ this->emitBlockReadWriteMemInst(I, CS, false, 1); break;
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM2:
+ this->emitBlockReadWriteMemInst(I, CS, false, 2); break;
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM4:
+ this->emitBlockReadWriteMemInst(I, CS, false, 4); break;
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM8:
+ this->emitBlockReadWriteMemInst(I, CS, false, 8); break;
case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM:
- this->emitBlockReadWriteMemInst(I, CS, true); break;
+ this->emitBlockReadWriteMemInst(I, CS, true, 1); break;
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM2:
+ this->emitBlockReadWriteMemInst(I, CS, true, 2); break;
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM4:
+ this->emitBlockReadWriteMemInst(I, CS, true, 4); break;
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM8:
+ this->emitBlockReadWriteMemInst(I, CS, true, 8); break;
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE:
this->emitBlockReadWriteImageInst(I, CS, false, 1); break;
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE2:
diff --git a/backend/src/llvm/llvm_gen_ocl_function.hxx b/backend/src/llvm/llvm_gen_ocl_function.hxx
index 456ab58..48a72d1 100644
--- a/backend/src/llvm/llvm_gen_ocl_function.hxx
+++ b/backend/src/llvm/llvm_gen_ocl_function.hxx
@@ -218,7 +218,13 @@ DECL_LLVM_GEN_FUNCTION(SUB_GROUP_SCAN_INCLUSIVE_MAX, __gen_ocl_sub_group_scan_in
DECL_LLVM_GEN_FUNCTION(SUB_GROUP_SCAN_INCLUSIVE_MIN, __gen_ocl_sub_group_scan_inclusive_min)
DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_MEM, __gen_ocl_sub_group_block_read_mem)
+DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_MEM2, __gen_ocl_sub_group_block_read_mem2)
+DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_MEM4, __gen_ocl_sub_group_block_read_mem4)
+DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_MEM8, __gen_ocl_sub_group_block_read_mem8)
DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_WRITE_MEM, __gen_ocl_sub_group_block_write_mem)
+DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_WRITE_MEM2, __gen_ocl_sub_group_block_write_mem2)
+DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_WRITE_MEM4, __gen_ocl_sub_group_block_write_mem4)
+DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_WRITE_MEM8, __gen_ocl_sub_group_block_write_mem8)
DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_IMAGE, __gen_ocl_sub_group_block_read_image)
DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_IMAGE2, __gen_ocl_sub_group_block_read_image2)
DECL_LLVM_GEN_FUNCTION(SUB_GROUP_BLOCK_READ_IMAGE4, __gen_ocl_sub_group_block_read_image4)
diff --git a/backend/src/llvm/llvm_scalarize.cpp b/backend/src/llvm/llvm_scalarize.cpp
index e60bf4b..615fb50 100644
--- a/backend/src/llvm/llvm_scalarize.cpp
+++ b/backend/src/llvm/llvm_scalarize.cpp
@@ -693,7 +693,19 @@ namespace gbe {
*CI = InsertToVector(call, *CI);
break;
}
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM:
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM2:
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM4:
+ case GEN_OCL_SUB_GROUP_BLOCK_WRITE_MEM8:
+ {
+ if ((*CI)->getType()->isVectorTy())
+ *CI = InsertToVector(call, *CI);
+ break;
+ }
case GEN_OCL_VME:
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM2:
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM4:
+ case GEN_OCL_SUB_GROUP_BLOCK_READ_MEM8:
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE2:
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE4:
case GEN_OCL_SUB_GROUP_BLOCK_READ_IMAGE8:
--
2.7.4
More information about the Beignet
mailing list