Enable EVEX feature: embedded broadcast for Vector128/256/512.Add() in limited cases

src/coreclr/jit/codegeninterface.h

@@ -127,7 +127,9 @@ class CodeGenInterface
 #define INST_FP 0x01 // is it a FP instruction?
 public:
  static bool instIsFP(instruction ins);
-
+#if defined(TARGET_XARCH)
+ static bool instIsEmbeddedBroadcastCompatible(instruction ins);
+#endif // TARGET_XARCH
  //-------------------------------------------------------------------------
  // Liveness-related fields & methods
 public:
@@ -764,6 +766,10 @@ class CodeGenInterface
 
  virtual const char* siStackVarName(size_t offs, size_t size, unsigned reg, unsigned stkOffs) = 0;
 #endif // LATE_DISASM
+
+#if defined(TARGET_XARCH)
+ bool IsEmbeddedBroadcastEnabled(instruction ins, GenTree* op);
+#endif
 };
 
 #endif // _CODEGEN_INTERFACE_H_

src/coreclr/jit/emit.h

@@ -781,6 +781,9 @@ class emitter
  unsigned _idCallRegPtr : 1; // IL indirect calls: addr in reg
  unsigned _idCallAddr : 1; // IL indirect calls: can make a direct call to iiaAddr
  unsigned _idNoGC : 1; // Some helpers don't get recorded in GC tables
+#if defined(TARGET_XARCH)
+ unsigned _idEvexbContext : 1; // does EVEX.b need to be set.
+#endif // TARGET_XARCH
 
 #ifdef TARGET_ARM64
  opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16
@@ -814,8 +817,8 @@ class emitter
 
  ////////////////////////////////////////////////////////////////////////
  // Space taken up to here:
- // x86: 46 bits
- // amd64: 46 bits
+ // x86: 47 bits
+ // amd64: 47 bits
  // arm: 48 bits
  // arm64: 50 bits
  // loongarch64: 46 bits
@@ -830,8 +833,10 @@ class emitter
 #define ID_EXTRA_BITFIELD_BITS (16)
 #elif defined(TARGET_ARM64)
 #define ID_EXTRA_BITFIELD_BITS (18)
-#elif defined(TARGET_XARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
+#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
 #define ID_EXTRA_BITFIELD_BITS (14)
+#elif defined(TARGET_XARCH)
+#define ID_EXTRA_BITFIELD_BITS (15)
 #else
 #error Unsupported or unset target architecture
 #endif
@@ -866,8 +871,8 @@ class emitter
 
  ////////////////////////////////////////////////////////////////////////
  // Space taken up to here (with/without prev offset, assuming host==target):
- // x86: 52/48 bits
- // amd64: 53/48 bits
+ // x86: 53/49 bits
+ // amd64: 54/49 bits
  // arm: 54/50 bits
  // arm64: 57/52 bits
  // loongarch64: 53/48 bits
@@ -1529,6 +1534,19 @@ class emitter
  _idNoGC = val;
  }
 
+#ifdef TARGET_XARCH
+ bool idIsEvexbContext() const
+ {
+ return _idEvexbContext != 0;
+ }
+ void idSetEvexbContext()
+ {
+ assert(_idEvexbContext == 0);
+ _idEvexbContext = 1;
+ assert(_idEvexbContext == 1);
+ }
+#endif
+
 #ifdef TARGET_ARMARCH
  bool idIsLclVar() const
  {
@@ -3655,9 +3673,25 @@ inline unsigned emitter::emitGetInsCIargs(instrDesc* id)
 //
 emitAttr emitter::emitGetMemOpSize(instrDesc* id) const
 {
+
  emitAttr defaultSize = id->idOpSize();
  instruction ins = id->idIns();
+ if (id->idIsEvexbContext())
+ {
+ // should have the assumption that Evex.b now stands for the embedded broadcast context.
+ // reference: Section 2.7.5 in Intel 64 and ia-32 architectures software developer's manual volume 2.
+ ssize_t inputSize = GetInputSizeInBytes(id);
+ switch (inputSize)
+ {
+ case 4:
+ return EA_4BYTE;
+ case 8:
+ return EA_8BYTE;
 
+ default:
+ unreached();
+ }
+ }
  switch (ins)
  {
  case INS_pextrb:

src/coreclr/jit/emitxarch.cpp

@@ -1231,9 +1231,10 @@ bool emitter::TakesEvexPrefix(const instrDesc* id) const
 #define DEFAULT_BYTE_EVEX_PREFIX_MASK 0xFFFFFFFF00000000ULL
 #define LBIT_IN_BYTE_EVEX_PREFIX 0x0000002000000000ULL
 #define LPRIMEBIT_IN_BYTE_EVEX_PREFIX 0x0000004000000000ULL
+#define EVEX_B_BIT 0x0000001000000000ULL
 
 //------------------------------------------------------------------------
-// AddEvexPrefix: Add default EVEX perfix with only LL' bits set.
+// AddEvexPrefix: Add default EVEX prefix with only LL' bits set.
 //
 // Arguments:
 // ins -- processor instruction to check.
@@ -1268,6 +1269,22 @@ emitter::code_t emitter::AddEvexPrefix(instruction ins, code_t code, emitAttr at
  return code;
 }
 
+//------------------------------------------------------------------------
+// AddEvexPrefix: set Evex.b bit if EvexbContext is set in instruction descritor.
+//
+// Arguments:
+// code -- opcode bits.
+//
+// Return Value:
+// encoded code with Evex.b set if needed.
+//
+emitter::code_t emitter::AddEvexbBit(code_t code)
+{
+ assert(hasEvexPrefix(code));
+ code |= EVEX_B_BIT;
+ return code;
+}
+
 // Returns true if this instruction requires a VEX prefix
 // All AVX instructions require a VEX prefix
 bool emitter::TakesVexPrefix(instruction ins) const
@@ -6667,7 +6684,8 @@ void emitter::emitIns_R_S_I(instruction ins, emitAttr attr, regNumber reg1, int
  emitCurIGsize += sz;
 }
 
-void emitter::emitIns_R_R_A(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, GenTreeIndir* indir)
+void emitter::emitIns_R_R_A(
+ instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, GenTreeIndir* indir, insOpts instOptions)
 {
  assert(IsAvx512OrPriorInstruction(ins));
  assert(IsThreeOperandAVXInstruction(ins));
@@ -6678,6 +6696,11 @@ void emitter::emitIns_R_R_A(instruction ins, emitAttr attr, regNumber reg1, regN
  id->idIns(ins);
  id->idReg1(reg1);
  id->idReg2(reg2);
+ if (instOptions == INS_OPTS_EVEX_b)
+ {
+ assert(UseEvexEncoding());
+ id->idSetEvexbContext();
+ }
 
  emitHandleMemOp(indir, id, (ins == INS_mulx) ? IF_RWR_RWR_ARD : emitInsModeFormat(ins, IF_RRD_RRD_ARD), ins);
 
@@ -6778,8 +6801,13 @@ void emitter::emitIns_R_AR_R(instruction ins,
  emitCurIGsize += sz;
 }
 
-void emitter::emitIns_R_R_C(
- instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, CORINFO_FIELD_HANDLE fldHnd, int offs)
+void emitter::emitIns_R_R_C(instruction ins,
+ emitAttr attr,
+ regNumber reg1,
+ regNumber reg2,
+ CORINFO_FIELD_HANDLE fldHnd,
+ int offs,
+ insOpts instOptions)
 {
  assert(IsAvx512OrPriorInstruction(ins));
  assert(IsThreeOperandAVXInstruction(ins));
@@ -6797,6 +6825,11 @@ void emitter::emitIns_R_R_C(
  id->idReg1(reg1);
  id->idReg2(reg2);
  id->idAddr()->iiaFieldHnd = fldHnd;
+ if (instOptions == INS_OPTS_EVEX_b)
+ {
+ assert(UseEvexEncoding());
+ id->idSetEvexbContext();
+ }
 
  UNATIVE_OFFSET sz = emitInsSizeCV(id, insCodeRM(ins));
  id->idCodeSize(sz);
@@ -6829,7 +6862,8 @@ void emitter::emitIns_R_R_R(instruction ins, emitAttr attr, regNumber targetReg,
  emitCurIGsize += sz;
 }
 
-void emitter::emitIns_R_R_S(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, int varx, int offs)
+void emitter::emitIns_R_R_S(
+ instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, int varx, int offs, insOpts instOptions)
 {
  assert(IsAvx512OrPriorInstruction(ins));
  assert(IsThreeOperandAVXInstruction(ins));
@@ -6842,6 +6876,11 @@ void emitter::emitIns_R_R_S(instruction ins, emitAttr attr, regNumber reg1, regN
  id->idReg2(reg2);
  id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs);
 
+ if (instOptions == INS_OPTS_EVEX_b)
+ {
+ assert(UseEvexEncoding());
+ id->idSetEvexbContext();
+ }
 #ifdef DEBUG
  id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs;
 #endif
@@ -8126,14 +8165,15 @@ void emitter::emitIns_SIMD_R_R_I(instruction ins, emitAttr attr, regNumber targe
 // indir -- The GenTreeIndir used for the memory address
 //
 void emitter::emitIns_SIMD_R_R_A(
- instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, GenTreeIndir* indir)
+ instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, GenTreeIndir* indir, insOpts instOptions)
 {
  if (UseSimdEncoding())
  {
- emitIns_R_R_A(ins, attr, targetReg, op1Reg, indir);
+ emitIns_R_R_A(ins, attr, targetReg, op1Reg, indir, instOptions);
  }
  else
  {
+ assert(instOptions == INS_OPTS_NONE);
  emitIns_Mov(INS_movaps, attr, targetReg, op1Reg, /* canSkip */ true);
  emitIns_R_A(ins, attr, targetReg, indir);
  }
@@ -8151,15 +8191,21 @@ void emitter::emitIns_SIMD_R_R_A(
 // fldHnd -- The CORINFO_FIELD_HANDLE used for the memory address
 // offs -- The offset added to the memory address from fldHnd
 //
-void emitter::emitIns_SIMD_R_R_C(
- instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, CORINFO_FIELD_HANDLE fldHnd, int offs)
+void emitter::emitIns_SIMD_R_R_C(instruction ins,
+ emitAttr attr,
+ regNumber targetReg,
+ regNumber op1Reg,
+ CORINFO_FIELD_HANDLE fldHnd,
+ int offs,
+ insOpts instOptions)
 {
  if (UseSimdEncoding())
  {
- emitIns_R_R_C(ins, attr, targetReg, op1Reg, fldHnd, offs);
+ emitIns_R_R_C(ins, attr, targetReg, op1Reg, fldHnd, offs, instOptions);
  }
  else
  {
+ assert(instOptions == INS_OPTS_NONE);
  emitIns_Mov(INS_movaps, attr, targetReg, op1Reg, /* canSkip */ true);
  emitIns_R_C(ins, attr, targetReg, fldHnd, offs);
  }
@@ -8214,14 +8260,15 @@ void emitter::emitIns_SIMD_R_R_R(
 // offs -- The offset added to the memory address from varx
 //
 void emitter::emitIns_SIMD_R_R_S(
- instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, int varx, int offs)
+ instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, int varx, int offs, insOpts instOptions)
 {
  if (UseSimdEncoding())
  {
- emitIns_R_R_S(ins, attr, targetReg, op1Reg, varx, offs);
+ emitIns_R_R_S(ins, attr, targetReg, op1Reg, varx, offs, instOptions);
  }
  else
  {
+ assert(instOptions == INS_OPTS_NONE);
  emitIns_Mov(INS_movaps, attr, targetReg, op1Reg, /* canSkip */ true);
  emitIns_R_S(ins, attr, targetReg, varx, offs);
  }
@@ -15709,7 +15756,7 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i)
 // Return Value:
 // size in bytes.
 //
-ssize_t emitter::GetInputSizeInBytes(instrDesc* id)
+ssize_t emitter::GetInputSizeInBytes(instrDesc* id) const
 {
  insFlags inputSize = static_cast<insFlags>((CodeGenInterface::instInfo[id->idIns()] & Input_Mask));