Skip to content

Commit 8b7e3b2

Browse files
Enigmopablodelara
Enigmo
authored and
pablodelara
committed
erasure_code: optimize SVE/SVE2 dot product with XOR for coefficient=1
Optimize the GF(2^8) vector dot product implementation for ARM SVE/SVE2 by using direct XOR operations when the encoding coefficient is 1, eliminating unnecessary table lookups. Technical Details: - Add is_dest_coeff_all_one() helper to detect coefficient=1 blocks - Pre-compute coefficient status for each destination vector - For coefficient=1 blocks: use direct XOR (src_data XOR accumulator) - For other coefficients: use traditional nibble-based table lookup - Optimization applied uniformly inside gf_nvect_dot_prod_sve_unrolled() The optimization leverages the property that EC encoding matrices often have coefficient 1 for the first parity block, making XOR operations sufficient instead of full GF multiplication via table lookup. Performance Results: Kunpeng-920 (SVE): +------------------+------------+------------+--------+ | Test Case | Before | After | Speedup| +------------------+------------+------------+--------+ | 10+1 encode | 15 GB/s | 79 GB/s | 5.3x | | 10+1 decode(1) | 15 GB/s | 78 GB/s | 5.2x | | 4+2 encode | 19 GB/s | 23 GB/s | 1.2x | | 4+2 decode(1) | 24 GB/s | 85 GB/s | 3.5x | | 8+3 encode | 13 GB/s | 14 GB/s | 1.1x | | 8+3 decode(1) | 21 GB/s | 77 GB/s | 3.7x | +------------------+------------+------------+--------+ Kunpeng-950 (SVE2): +------------------+------------+------------+--------+ | Test Case | Before | After | Speedup| +------------------+------------+------------+--------+ | 10+1 encode | 13 GB/s | 84 GB/s | 6.5x | | 10+1 decode(1) | 13 GB/s | 84 GB/s | 6.5x | | 4+2 encode | 17 GB/s | 22 GB/s | 1.3x | | 4+2 decode(1) | 24 GB/s | 85 GB/s | 3.5x | | 8+3 encode | 12 GB/s | 14 GB/s | 1.2x | | 8+3 decode(1) | 12 GB/s | 88 GB/s | 7.3x | +------------------+------------+------------+--------+ Measured using erasure_code_perf tool on Huawei Kunpeng platforms. Signed-off-by: Chenxuqiang <chenxuqiang3@hisilicon.com> Signed-off-by: Enigmo <guotaowei4@huawei.com>
1 parent adcbb2a commit 8b7e3b2

1 file changed

Lines changed: 68 additions & 34 deletions

File tree

‎erasure_code/aarch64/gf_nvect_dot_prod_sve.c‎

Lines changed: 68 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@
5757
// can do this automatically in optimization so a separate implementation isn't required.
5858
// We simply allow the compiler to generate SVE2 versions as well.
5959

60+
staticinlineint
61+
is_dest_coeff_all_one(unsigned char*gftbls, intvlen, intd)
62+
{
63+
for (intv=0; v<vlen; v++) {
64+
if (gftbls[d*vlen*32+v*32+1] !=0x1) {
65+
return0;
66+
}
67+
}
68+
return1;
69+
}
70+
6071
#ifdef__APPLE__
6172
__attribute__((target("+sme"), always_inline))
6273
#else
@@ -75,6 +86,11 @@ gf_nvect_dot_prod_sve_unrolled(int len, int vlen, unsigned char *gftbls, unsigne
7586
intpos=0;
7687

7788
// 4x unrolled main loop - SVE predicates handle ALL remaining data automatically
89+
intis_all_one[7] = { 0 };
90+
for (intd=0; d<nvect&&d<7; d++) {
91+
is_all_one[d] =is_dest_coeff_all_one(gftbls, vlen, d);
92+
}
93+
7894
while (pos<len) {
7995
// Create predicates for 4 batches - SVE masks beyond array bounds
8096
svbool_tpredicate_0=svwhilelt_b8_s32(pos+sve_len*0, len);
@@ -129,43 +145,61 @@ gf_nvect_dot_prod_sve_unrolled(int len, int vlen, unsigned char *gftbls, unsigne
129145
svuint8_tsrc_data2=svld1_u8(predicate_2, &src[v][pos+sve_len*2]);
130146
svuint8_tsrc_data3=svld1_u8(predicate_3, &src[v][pos+sve_len*3]);
131147

132-
// Extract nibbles for all batches
133-
svuint8_tsrc_lo0=svand_x(predicate_0, src_data0, mask0f);
134-
svuint8_tsrc_hi0=svlsr_x(predicate_0, src_data0, 4);
135-
svuint8_tsrc_lo1=svand_x(predicate_1, src_data1, mask0f);
136-
svuint8_tsrc_hi1=svlsr_x(predicate_1, src_data1, 4);
137-
svuint8_tsrc_lo2=svand_x(predicate_2, src_data2, mask0f);
138-
svuint8_tsrc_hi2=svlsr_x(predicate_2, src_data2, 4);
139-
svuint8_tsrc_lo3=svand_x(predicate_3, src_data3, mask0f);
140-
svuint8_tsrc_hi3=svlsr_x(predicate_3, src_data3, 4);
148+
svuint8_tsrc_lo0=svdup_u8(0), src_hi0=svdup_u8(0);
149+
svuint8_tsrc_lo1=svdup_u8(0), src_hi1=svdup_u8(0);
150+
svuint8_tsrc_lo2=svdup_u8(0), src_hi2=svdup_u8(0);
151+
svuint8_tsrc_lo3=svdup_u8(0), src_hi3=svdup_u8(0);
152+
for (intd=0; d<nvect; d++) {
153+
if (!is_all_one[d]) {
154+
src_lo0=svand_x(predicate_0, src_data0, mask0f);
155+
src_hi0=svlsr_x(predicate_0, src_data0, 4);
156+
src_lo1=svand_x(predicate_1, src_data1, mask0f);
157+
src_hi1=svlsr_x(predicate_1, src_data1, 4);
158+
src_lo2=svand_x(predicate_2, src_data2, mask0f);
159+
src_hi2=svlsr_x(predicate_2, src_data2, 4);
160+
src_lo3=svand_x(predicate_3, src_data3, mask0f);
161+
src_hi3=svlsr_x(predicate_3, src_data3, 4);
162+
break;
163+
}
164+
}
141165

142166
// Process each destination with unrolled batches
143167
for (intd=0; d<nvect; d++) {
144-
unsigned char*tbl_base=&gftbls[d*vlen*32+v*32];
145-
svuint8_ttbl_lo=svld1_u8(predicate_true, tbl_base);
146-
svuint8_ttbl_hi=svld1_u8(predicate_true, tbl_base+16);
147-
148-
// Batch 0
149-
svuint8_tgf_lo0=svtbl_u8(tbl_lo, src_lo0);
150-
svuint8_tgf_hi0=svtbl_u8(tbl_hi, src_hi0);
151-
152-
// Batch 1
153-
svuint8_tgf_lo1=svtbl_u8(tbl_lo, src_lo1);
154-
svuint8_tgf_hi1=svtbl_u8(tbl_hi, src_hi1);
155-
156-
// Batch 2
157-
svuint8_tgf_lo2=svtbl_u8(tbl_lo, src_lo2);
158-
svuint8_tgf_hi2=svtbl_u8(tbl_hi, src_hi2);
159-
160-
// Batch 3
161-
svuint8_tgf_lo3=svtbl_u8(tbl_lo, src_lo3);
162-
svuint8_tgf_hi3=svtbl_u8(tbl_hi, src_hi3);
163-
164-
svuint8_tgf_result0=sveor_x(predicate_0, gf_lo0, gf_hi0);
165-
svuint8_tgf_result1=sveor_x(predicate_1, gf_lo1, gf_hi1);
166-
svuint8_tgf_result2=sveor_x(predicate_2, gf_lo2, gf_hi2);
167-
svuint8_tgf_result3=sveor_x(predicate_3, gf_lo3, gf_hi3);
168-
168+
svuint8_tgf_result0, gf_result1, gf_result2, gf_result3;
169+
170+
if (is_all_one[d]) {
171+
gf_result0=src_data0;
172+
gf_result1=src_data1;
173+
gf_result2=src_data2;
174+
gf_result3=src_data3;
175+
} else {
176+
constunsigned char*consttbl_base=
177+
&gftbls[d*vlen*32+v*32];
178+
constsvuint8_ttbl_lo=svld1_u8(predicate_true, tbl_base);
179+
constsvuint8_ttbl_hi=
180+
svld1_u8(predicate_true, tbl_base+16);
181+
182+
// Batch 0
183+
constsvuint8_tgf_lo0=svtbl_u8(tbl_lo, src_lo0);
184+
constsvuint8_tgf_hi0=svtbl_u8(tbl_hi, src_hi0);
185+
186+
// Batch 1
187+
constsvuint8_tgf_lo1=svtbl_u8(tbl_lo, src_lo1);
188+
constsvuint8_tgf_hi1=svtbl_u8(tbl_hi, src_hi1);
189+
190+
// Batch 2
191+
constsvuint8_tgf_lo2=svtbl_u8(tbl_lo, src_lo2);
192+
constsvuint8_tgf_hi2=svtbl_u8(tbl_hi, src_hi2);
193+
194+
// Batch 3
195+
constsvuint8_tgf_lo3=svtbl_u8(tbl_lo, src_lo3);
196+
constsvuint8_tgf_hi3=svtbl_u8(tbl_hi, src_hi3);
197+
198+
gf_result0=sveor_x(predicate_0, gf_lo0, gf_hi0);
199+
gf_result1=sveor_x(predicate_1, gf_lo1, gf_hi1);
200+
gf_result2=sveor_x(predicate_2, gf_lo2, gf_hi2);
201+
gf_result3=sveor_x(predicate_3, gf_lo3, gf_hi3);
202+
}
169203
// Accumulate results
170204
switch (d) {
171205
case0:

0 commit comments

Comments
 (0)