JIT: Accelerate more casts on x86

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST:

src/coreclr/jit/compiler.h

@@ -6249,6 +6249,7 @@ class Compiler
  void fgConvertBBToThrowBB(BasicBlock* block);
 
  bool fgCastNeeded(GenTree* tree, var_types toType);
+ bool fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow = false);
 
  void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
  void fgLoopCallMark();

src/coreclr/jit/decomposelongs.cpp

@@ -138,6 +138,8 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
+ // On x86, long->floating casts are implemented in DecomposeCast.
+ // Those nodes, plus any nodes that produce a long, will be examined.
  if (!tree->TypeIs(TYP_LONG) &&
  !(tree->OperIs(GT_CAST) && varTypeIsLong(tree->AsCast()->CastOp()) && varTypeIsFloating(tree)))
 #else
@@ -159,6 +161,9 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree)
  // HWIntrinsics can consume/produce a long directly, provided its source/target is memory.
  // Here we do a conservative check for specific cases where it is certain the load/store
  // can be contained. In those cases, we can skip decomposition.
+ //
+ // We also look for longs consumed directly by a long->floating cast. These can skip
+ // decomposition because the cast is implemented using HWIntrinsics.
 
  GenTree* user = use.User();
 
@@ -582,44 +587,187 @@ GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use)
  }
 
 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_X86)
- if (varTypeIsFloating(dstType))
+ if (varTypeIsFloating(srcType) || varTypeIsFloating(dstType))
  {
  // We will reach this path only if morph did not convert the cast to a helper call,
  // meaning we can perform the cast using SIMD instructions.
- // The sequence this creates is simply:
- // AVX512DQ.VL.ConvertToVector128Single(Vector128.CreateScalarUnsafe(LONG)).ToScalar()
-
- NamedIntrinsic intrinsicId = NI_Illegal;
- GenTree* srcOp = cast->CastOp();
- var_types dstType = cast->CastToType();
- CorInfoType baseFloatingType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
- CorInfoType baseIntegralType = cast->IsUnsigned() ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
 
  assert(!cast->gtOverflow());
  assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512));
 
- intrinsicId = (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+ GenTree* srcOp = cast->CastOp();
+ GenTree* castResult = nullptr;
+ LIR::Range castRange = LIR::EmptyRange();
+ CorInfoType srcBaseType = CORINFO_TYPE_UNDEF;
+ CorInfoType dstBaseType = CORINFO_TYPE_UNDEF;
 
- GenTree* createScalar = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, baseIntegralType, 16);
- GenTree* convert =
- m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, createScalar, intrinsicId, baseIntegralType, 16);
- GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(dstType, convert, baseFloatingType, 16);
+ if (varTypeIsFloating(srcType))
+ {
+ srcBaseType = (srcType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ dstBaseType = (dstType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ }
+ else
+ {
+ srcBaseType = (srcType == TYP_ULONG) ? CORINFO_TYPE_ULONG : CORINFO_TYPE_LONG;
+ dstBaseType = (dstType == TYP_FLOAT) ? CORINFO_TYPE_FLOAT : CORINFO_TYPE_DOUBLE;
+ }
 
- Range().InsertAfter(cast, createScalar, convert, toScalar);
- Range().Remove(cast);
+ // This creates the equivalent of the following C# code:
+ // var srcVec = Vector128.CreateScalarUnsafe(castOp);
+
+ GenTree* srcVector = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, srcOp, srcBaseType, 16);
+ castRange.InsertAtEnd(srcVector);
 
- if (createScalar->IsCnsVec())
+ if (srcVector->IsCnsVec())
  {
  Range().Remove(srcOp);
  }
 
+ if (varTypeIsFloating(dstType))
+ {
+ // long->floating casts don't require any kind of fixup. We simply use the vector
+ // form of the instructions, because the scalar form is not supported on 32-bit.
+
+ NamedIntrinsic intrinsicId =
+ (dstType == TYP_FLOAT) ? NI_AVX512_ConvertToVector128Single : NI_AVX512_ConvertToVector128Double;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX10v2))
+ {
+ // Likewise, the AVX10.2 saturating floating->long instructions give the correct result,
+ // but we have to use the vector form.
+
+ NamedIntrinsic intrinsicId = (dstType == TYP_ULONG)
+ ? NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation
+ : NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
+
+ castResult = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, intrinsicId, srcBaseType, 16);
+ }
+ else if (dstType == TYP_ULONG)
+ {
+ // AVX-512 unsigned conversion instructions correctly saturate for positive overflow, so
+ // we only need to fix up negative or NaN values before conversion.
+ //
+ // maxs[sd] will take the value from the second operand if the first operand's value is
+ // NaN, which allows us to fix up both negative and NaN values with a single instruction.
+ //
+ // This creates the equivalent of the following C# code:
+ // var fixupVal = Sse.MaxScalar(srcVec, Vector128<T>.Zero);
+ // castResult = Avx512DQ.VL.ConvertToVector128UInt64WithTruncation(fixupVal);
+
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ GenTree* fixupVal = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcVector, zero, NI_X86Base_MaxScalar,
+ srcBaseType, 16);
+
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(fixupVal);
+
+ castResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, fixupVal,
+ NI_AVX512_ConvertToVector128UInt64WithTruncation, srcBaseType, 16);
+ }
+ else
+ {
+ assert(dstType == TYP_LONG);
+
+ // We will use the input value multiple times, so we replace it with a lclVar.
+ LIR::Use srcUse;
+ LIR::Use::MakeDummyUse(castRange, srcVector, &srcUse);
+ srcUse.ReplaceWithLclVar(m_compiler);
+ srcVector = srcUse.Def();
+
+ // We fix up NaN values by masking in zero during conversion. Negative saturation is handled
+ // correctly by the conversion instructions. Positive saturation is handled after conversion,
+ // because MaxValue is not precisely representable in the floating format.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var nanMask = Avx.CompareScalar(srcVec, srcVec, FloatComparisonMode.OrderedNonSignaling);
+ // var convert = Avx512DQ.VL.ConvertToVector128Int64WithTruncation(srcVec);
+ // convertResult = Vector128.ConditionalSelect(nanMask, convert, Vector128<long>.Zero);
+
+ GenTree* srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMode =
+ m_compiler->gtNewIconNode(static_cast<int32_t>(FloatComparisonMode::OrderedNonSignaling));
+ GenTree* nanMask = m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcVector, srcClone, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(nanMask);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* convertResult =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, srcClone,
+ NI_AVX512_ConvertToVector128Int64WithTruncation, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(convertResult);
+
+ nanMask = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, nanMask, dstBaseType, 16);
+ GenTree* zero = m_compiler->gtNewZeroConNode(TYP_SIMD16);
+ convertResult = m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, nanMask, convertResult, zero, dstBaseType, 16);
+
+ castRange.InsertAtEnd(nanMask);
+ castRange.InsertAtEnd(zero);
+ castRange.InsertAtEnd(convertResult);
+
+ // Now we handle saturation of the result for positive overflow.
+ //
+ // This creates roughly the equivalent of the following C# code:
+ // var compareMode = FloatComparisonMode.OrderedGreaterThanOrEqualNonSignaling;
+ // var maxFloatingValue = Vector128.Create(9223372036854775808.0);
+ // var compareMax = Avx.CompareScalar(srcVec, maxFloatingValue, compareMode);
+ // var maxLong = Vector128<long>.AllOnes >>> 1;
+ // castResult = Vector128.ConditionalSelect(compareMax, maxLong, convertResult);
+
+ compareMode = m_compiler->gtNewIconNode(
+ static_cast<int32_t>(FloatComparisonMode::OrderedGreaterThanOrEqualNonSignaling));
+
+ GenTreeVecCon* maxFloatingValue = m_compiler->gtNewVconNode(TYP_SIMD16);
+ maxFloatingValue->EvaluateBroadcastInPlace(srcType, 9223372036854775808.0);
+
+ srcClone = m_compiler->gtClone(srcVector);
+ GenTree* compareMax =
+ m_compiler->gtNewSimdHWIntrinsicNode(TYP_MASK, srcClone, maxFloatingValue, compareMode,
+ NI_AVX512_CompareScalarMask, srcBaseType, 16);
+
+ castRange.InsertAtEnd(srcClone);
+ castRange.InsertAtEnd(maxFloatingValue);
+ castRange.InsertAtEnd(compareMode);
+ castRange.InsertAtEnd(compareMax);
+
+ GenTree* allOnes = m_compiler->gtNewAllBitsSetConNode(TYP_SIMD16);
+ GenTree* one = m_compiler->gtNewIconNode(1);
+ GenTree* maxLong = m_compiler->gtNewSimdBinOpNode(GT_RSZ, TYP_SIMD16, allOnes, one, dstBaseType, 16);
+
+ castRange.InsertAtEnd(allOnes);
+ castRange.InsertAtEnd(one);
+ castRange.InsertAtEnd(maxLong);
+
+ compareMax = m_compiler->gtNewSimdCvtMaskToVectorNode(TYP_SIMD16, compareMax, dstBaseType, 16);
+ castResult =
+ m_compiler->gtNewSimdCndSelNode(TYP_SIMD16, compareMax, maxLong, convertResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(compareMax);
+ }
+
+ // Because the results are in a SIMD register, we need to ToScalar() them out.
+ GenTree* toScalar = m_compiler->gtNewSimdToScalarNode(genActualType(dstType), castResult, dstBaseType, 16);
+
+ castRange.InsertAtEnd(castResult);
+ castRange.InsertAtEnd(toScalar);
+
+ Range().InsertAfter(cast, std::move(castRange));
+ Range().Remove(cast);
+
  if (use.IsDummyUse())
  {
  toScalar->SetUnusedValue();
  }
  use.ReplaceWith(toScalar);
 
- return toScalar->gtNext;
+ return toScalar;
  }
 #endif // FEATURE_HW_INTRINSICS && TARGET_X86
 

src/coreclr/jit/flowgraph.cpp

@@ -1270,6 +1270,43 @@ bool Compiler::fgCastNeeded(GenTree* tree, var_types toType)
  return true;
 }
 
+//-------------------------------------------------------------------------------------
+// fgCastRequiresHelper: Check whether a given cast must be converted to a helper call.
+//
+// Arguments:
+// fromType - The source type of the cast.
+// toType - The target type of the cast.
+// overflow - True if the cast has the GTF_OVERFLOW flag set.
+//
+// Return Value:
+// True if the cast requires a helper call, otherwise false.
+//
+bool Compiler::fgCastRequiresHelper(var_types fromType, var_types toType, bool overflow /* false */)
+{
+ if (varTypeIsFloating(fromType) && overflow)
+ {
+ assert(varTypeIsIntegral(toType));
+ return true;
+ }
+
+#if !defined(TARGET_64BIT)
+ if ((varTypeIsFloating(fromType) && varTypeIsLong(toType)) ||
+ (varTypeIsLong(fromType) && varTypeIsFloating(toType)))
+ {
+#if defined(TARGET_X86)
+ if (compOpportunisticallyDependsOn(InstructionSet_AVX512))
+ {
+ return false;
+ }
+#endif // TARGET_X86
+
+ return true;
+ }
+#endif // !TARGET_64BIT
+
+ return false;
+}
+
 GenTree* Compiler::fgGetCritSectOfStaticMethod()
 {
  noway_assert(!compIsForInlining());

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -1230,6 +1230,7 @@ HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask,
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, {INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
+HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd}, HW_Category_IMM, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd}, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask)
 HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg)
 HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, {INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd}, HW_Category_MemoryStore, HW_Flag_NoFlag)

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -494,6 +494,7 @@ int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id)
  case NI_AVX_CompareScalar:
  case NI_AVX512_Compare:
  case NI_AVX512_CompareMask:
+ case NI_AVX512_CompareScalarMask:
  case NI_AVX10v2_MinMaxScalar:
  case NI_AVX10v2_MinMax:
  {

src/coreclr/jit/importer.cpp

@@ -8258,28 +8258,8 @@ void Compiler::impImportBlockCode(BasicBlock* block)
  goto _CONV;
 
  _CONV:
- // only converts from FLOAT or DOUBLE to an integer type
- // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls
-
- if (varTypeIsFloating(lclTyp))
- {
- callNode = varTypeIsLong(impStackTop().val) ||
- uns // uint->dbl gets turned into uint->long->dbl
-#ifdef TARGET_64BIT
- // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK?
- // TYP_BYREF could be used as TYP_I_IMPL which is long.
- // TODO-CQ: remove this when we lower casts long/ulong --> float/double
- // and generate SSE2 code instead of going through helper calls.
- || impStackTop().val->TypeIs(TYP_BYREF)
-#endif
- ;
- }
- else
- {
- callNode = varTypeIsFloating(impStackTop().val->TypeGet());
- }
-
- op1 = impPopStack().val;
+ op1 = impPopStack().val;
+ callNode = fgCastRequiresHelper(op1->TypeGet(), lclTyp, ovfl);
 
  impBashVarAddrsToI(op1);
 

src/coreclr/jit/lower.cpp

@@ -501,7 +501,7 @@ GenTree* Lowering::LowerNode(GenTree* node)
  }
 
  LowerCast(node);
-break;
+return nextNode;
  }
 
  case GT_BITCAST: