[Arm64] ASIMD By Element Intrinsics

src/coreclr/src/jit/codegenarm64.cpp

@@ -9303,6 +9303,17 @@ void CodeGen::genArm64EmitterUnitTests()
  theEmitter->emitIns_R_R_R_I(INS_smull2, EA_16BYTE, REG_V6, REG_V7, REG_V8, 7, INS_OPTS_8H);
  theEmitter->emitIns_R_R_R_I(INS_smull2, EA_16BYTE, REG_V9, REG_V10, REG_V11, 3, INS_OPTS_4S);
 
+ // umlal vector, by element
+ theEmitter->emitIns_R_R_R_I(INS_umlal, EA_8BYTE, REG_V0, REG_V1, REG_V2, 3, INS_OPTS_4H);
+ theEmitter->emitIns_R_R_R_I(INS_umlal, EA_8BYTE, REG_V3, REG_V4, REG_V5, 1, INS_OPTS_2S);
+
+ // umlal2 vector, by element
+ theEmitter->emitIns_R_R_R_I(INS_umlal2, EA_16BYTE, REG_V6, REG_V7, REG_V8, 7, INS_OPTS_8H);
+ theEmitter->emitIns_R_R_R_I(INS_umlal2, EA_16BYTE, REG_V9, REG_V10, REG_V11, 3, INS_OPTS_4S);
+
+ // umlsl vector, by element
+ theEmitter->emitIns_R_R_R_I(INS_umlsl, EA_8BYTE, REG_V0, REG_V1, REG_V2, 3, INS_OPTS_4H);
+
  // umlsl2 vector, by element
  theEmitter->emitIns_R_R_R_I(INS_umlsl2, EA_16BYTE, REG_V6, REG_V7, REG_V8, 7, INS_OPTS_8H);
  theEmitter->emitIns_R_R_R_I(INS_umlsl2, EA_16BYTE, REG_V9, REG_V10, REG_V11, 3, INS_OPTS_4S);

src/coreclr/src/jit/compiler.h

@@ -3782,7 +3782,7 @@ class Compiler
  GenTree* getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass, bool expectAddr = false);
  GenTree* impNonConstFallback(NamedIntrinsic intrinsic, var_types simdType, var_types baseType);
  GenTree* addRangeCheckIfNeeded(
- NamedIntrinsic intrinsic, GenTree* lastOp, bool mustExpand, int immLowerBound, int immUpperBound);
+ NamedIntrinsic intrinsic, GenTree* immOp, bool mustExpand, int immLowerBound, int immUpperBound);
 
 #ifdef TARGET_XARCH
  GenTree* impBaseIntrinsic(NamedIntrinsic intrinsic,

src/coreclr/src/jit/emitarm64.cpp

@@ -817,7 +817,7 @@ void emitter::emitInsSanityCheck(instrDesc* id)
  assert(isVectorRegister(id->idReg2()));
  assert(isVectorRegister(id->idReg3()));
  elemsize = optGetElemsize(id->idInsOpt());
- assert(isValidVectorIndex(id->idOpSize(), elemsize, emitGetInsSC(id)));
+ assert(isValidVectorIndex(EA_16BYTE, elemsize, emitGetInsSC(id)));
  break;
 
  case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector)
@@ -6247,7 +6247,7 @@ void emitter::emitIns_R_R_R_I(instruction ins,
  assert(isValidArrangement(size, opt));
  elemsize = optGetElemsize(opt);
  assert(isValidVectorElemsizeFloat(elemsize));
- assert(isValidVectorIndex(size, elemsize, imm));
+ assert(isValidVectorIndex(EA_16BYTE, elemsize, imm));
  assert(opt != INS_OPTS_1D); // Reserved encoding
  fmt = IF_DV_3BI;
  }
@@ -6277,7 +6277,7 @@ void emitter::emitIns_R_R_R_I(instruction ins,
  // Only has encodings for H or S elemsize
  assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE));
  // Only has encodings for V0..V15
- if ((elemsize == EA_2BYTE) && (reg3 >= REG_V16))
+ if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0))
  {
  noway_assert(!"Invalid reg3");
  }
@@ -6368,6 +6368,11 @@ void emitter::emitIns_R_R_R_I(instruction ins,
  assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S));
  elemsize = optGetElemsize(opt);
  assert(isValidVectorIndex(EA_16BYTE, elemsize, imm));
+ // Restricted to V0-V15 when element size is H
+ if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0))
+ {
+ assert(!"Invalid reg3");
+ }
  fmt = IF_DV_3HI;
  break;
 
@@ -6384,6 +6389,11 @@ void emitter::emitIns_R_R_R_I(instruction ins,
  assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S));
  elemsize = optGetElemsize(opt);
  assert(isValidVectorIndex(EA_16BYTE, elemsize, imm));
+ // Restricted to V0-V15 when element size is H
+ if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0))
+ {
+ assert(!"Invalid reg3");
+ }
  fmt = IF_DV_3HI;
  break;
 
@@ -10966,7 +10976,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
  code = emitInsCode(ins, fmt);
  imm = emitGetInsSC(id);
  elemsize = optGetElemsize(id->idInsOpt());
- assert(isValidVectorIndex(id->idOpSize(), elemsize, imm));
+ assert(isValidVectorIndex(EA_16BYTE, elemsize, imm));
  code |= insEncodeVectorsize(id->idOpSize()); // Q
  code |= insEncodeFloatElemsize(elemsize); // X
  code |= insEncodeFloatIndex(elemsize, imm); // L H

src/coreclr/src/jit/gentree.cpp

@@ -1461,7 +1461,7 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK)
  if ((op1->AsHWIntrinsic()->gtHWIntrinsicId != op2->AsHWIntrinsic()->gtHWIntrinsicId) ||
  (op1->AsHWIntrinsic()->gtSIMDBaseType != op2->AsHWIntrinsic()->gtSIMDBaseType) ||
  (op1->AsHWIntrinsic()->gtSIMDSize != op2->AsHWIntrinsic()->gtSIMDSize) ||
- (op1->AsHWIntrinsic()->GetOtherBaseType() != op2->AsHWIntrinsic()->GetOtherBaseType()))
+ (op1->AsHWIntrinsic()->GetAuxiliaryType() != op2->AsHWIntrinsic()->GetAuxiliaryType()))
  {
  return false;
  }
@@ -2131,7 +2131,7 @@ unsigned Compiler::gtHashValue(GenTree* tree)
  hash += tree->AsHWIntrinsic()->gtHWIntrinsicId;
  hash += tree->AsHWIntrinsic()->gtSIMDBaseType;
  hash += tree->AsHWIntrinsic()->gtSIMDSize;
- hash += tree->AsHWIntrinsic()->GetOtherBaseType();
+ hash += tree->AsHWIntrinsic()->GetAuxiliaryType();
  break;
 #endif // FEATURE_HW_INTRINSICS
 
@@ -7478,7 +7478,7 @@ GenTree* Compiler::gtCloneExpr(
  GenTreeHWIntrinsic(hwintrinsicOp->TypeGet(), hwintrinsicOp->gtGetOp1(),
  hwintrinsicOp->gtGetOp2IfPresent(), hwintrinsicOp->gtHWIntrinsicId,
  hwintrinsicOp->gtSIMDBaseType, hwintrinsicOp->gtSIMDSize);
- copy->AsHWIntrinsic()->SetOtherBaseType(hwintrinsicOp->GetOtherBaseType());
+ copy->AsHWIntrinsic()->SetAuxiliaryType(hwintrinsicOp->GetAuxiliaryType());
  }
  break;
 #endif
@@ -18661,13 +18661,13 @@ GenTreeHWIntrinsic* Compiler::gtNewScalarHWIntrinsicNode(
 // Returns true for the HW Instrinsic instructions that have MemoryLoad semantics, false otherwise
 bool GenTreeHWIntrinsic::OperIsMemoryLoad() const
 {
-#ifdef TARGET_XARCH
- // Some xarch instructions have MemoryLoad sematics
+#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
  HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(gtHWIntrinsicId);
  if (category == HW_Category_MemoryLoad)
  {
  return true;
  }
+#ifdef TARGET_XARCH
  else if (HWIntrinsicInfo::MaybeMemoryLoad(gtHWIntrinsicId))
  {
  // Some intrinsics (without HW_Category_MemoryLoad) also have MemoryLoad semantics
@@ -18697,19 +18697,20 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad() const
  }
  }
 #endif // TARGET_XARCH
+#endif // TARGET_XARCH || TARGET_ARM64
  return false;
 }
 
 // Returns true for the HW Instrinsic instructions that have MemoryStore semantics, false otherwise
 bool GenTreeHWIntrinsic::OperIsMemoryStore() const
 {
-#ifdef TARGET_XARCH
- // Some xarch instructions have MemoryStore sematics
+#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
  HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(gtHWIntrinsicId);
  if (category == HW_Category_MemoryStore)
  {
  return true;
  }
+#ifdef TARGET_XARCH
  else if (HWIntrinsicInfo::MaybeMemoryStore(gtHWIntrinsicId) &&
  (category == HW_Category_IMM || category == HW_Category_Scalar))
  {
@@ -18732,13 +18733,14 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore() const
  }
  }
 #endif // TARGET_XARCH
+#endif // TARGET_XARCH || TARGET_ARM64
  return false;
 }
 
 // Returns true for the HW Instrinsic instructions that have MemoryLoad semantics, false otherwise
 bool GenTreeHWIntrinsic::OperIsMemoryLoadOrStore() const
 {
-#ifdef TARGET_XARCH
+#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
  return OperIsMemoryLoad() || OperIsMemoryStore();
 #else
  return false;

src/coreclr/src/jit/gentree.h

@@ -4728,8 +4728,8 @@ struct GenTreeJitIntrinsic : public GenTreeOp
  ClassLayout* m_layout;
 
  union {
- var_types  gtOtherBaseType; // For AVX2 Gather* intrinsics
- regNumberSmall gtOtherReg; // For intrinsics that return 2 registers
+ var_types gtAuxiliaryType; // For intrinsics than need another type (e.g. Avx2.Gather* or SIMD (by element))
+ regNumberSmall gtOtherReg; // For intrinsics that return 2 registers
  };
 
 public:
@@ -4766,14 +4766,14 @@ struct GenTreeJitIntrinsic : public GenTreeOp
  assert(gtOtherReg == reg);
  }
 
- var_types GetOtherBaseType() const
+ var_types GetAuxiliaryType() const
  {
- return gtOtherBaseType;
+ return gtAuxiliaryType;
  }
 
- void SetOtherBaseType(var_types type)
+ void SetAuxiliaryType(var_types type)
  {
-gtOtherBaseType = type;
+gtAuxiliaryType = type;
  }
 
  GenTreeJitIntrinsic(genTreeOps oper, var_types type, GenTree* op1, GenTree* op2, var_types baseType, unsigned size)