From 7b9fcaf39ec943173a5fb059a38b51f05b73e9f5 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Fri, 24 Jul 2026 10:05:34 +0000 Subject: [PATCH 1/7] arm64: Reuse SVE mask constants in LSRA --- src/coreclr/jit/codegencommon.cpp | 3 +- src/coreclr/jit/gentree.h | 7 +- src/coreclr/jit/hwintrinsic.h | 12 + src/coreclr/jit/hwintrinsicarm64.cpp | 6 + src/coreclr/jit/lsra.cpp | 61 ++- src/coreclr/jit/lsra.h | 27 +- src/coreclr/jit/lsraarm64.cpp | 162 ++++++++ src/coreclr/jit/lsrabuild.cpp | 49 ++- src/tests/JIT/opt/SVE/ConstantMaskReuse.cs | 351 ++++++++++++++++++ .../JIT/opt/SVE/ConstantMaskReuse.csproj | 12 + 10 files changed, 677 insertions(+), 13 deletions(-) create mode 100644 src/tests/JIT/opt/SVE/ConstantMaskReuse.cs create mode 100644 src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7f3cdc29802214..980e150bc00602 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8669,9 +8669,8 @@ void CodeGen::genCodeForReuseVal(GenTree* treeNode) { assert(treeNode->IsReuseRegVal()); - // For now, this is only used for constant nodes. #if defined(FEATURE_MASKED_HW_INTRINSICS) - assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK)); + assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK, GT_HWINTRINSIC)); #elif defined(FEATURE_SIMD) assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC)); #else diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index da8b1a44a63e95..29ad165f4d1767 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -2546,19 +2546,18 @@ struct GenTree bool IsReuseRegVal() const { - // This can be extended to non-constant nodes, but not to local or indir nodes. - return OperIsConst() && ((gtFlags & GTF_REUSE_REG_VAL) != 0); + return (OperIsConst() || OperIsHWIntrinsic()) && ((gtFlags & GTF_REUSE_REG_VAL) != 0); } void SetReuseRegVal() { - assert(OperIsConst()); + assert(OperIsConst() || OperIsHWIntrinsic()); gtFlags |= GTF_REUSE_REG_VAL; } void ResetReuseRegVal() { - assert(OperIsConst()); + assert(OperIsConst() || OperIsHWIntrinsic()); gtFlags &= ~GTF_REUSE_REG_VAL; } diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 7fb17d29509b54..2db1338c259955 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1028,6 +1028,18 @@ struct HWIntrinsicInfo return (flags & HW_Flag_Scalable) != 0; } +#ifdef FEATURE_MASKED_HW_INTRINSICS + static bool IsSveCreateTrueMask(NamedIntrinsic id) + { + static_assert(AreContiguous(NI_Sve_CreateTrueMaskByte, NI_Sve_CreateTrueMaskDouble, NI_Sve_CreateTrueMaskInt16, + NI_Sve_CreateTrueMaskInt32, NI_Sve_CreateTrueMaskInt64, NI_Sve_CreateTrueMaskSByte, + NI_Sve_CreateTrueMaskSingle, NI_Sve_CreateTrueMaskUInt16, + NI_Sve_CreateTrueMaskUInt32, NI_Sve_CreateTrueMaskUInt64)); + return (id >= NI_Sve_CreateTrueMaskByte) && (id <= NI_Sve_CreateTrueMaskUInt64); + } + +#endif // FEATURE_MASKED_HW_INTRINSICS + static bool IsLowMaskedOperation(NamedIntrinsic id) { const HWIntrinsicFlag flags = lookupFlags(id); diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index e5de5e0e574c0b..6d598980c77188 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1035,6 +1035,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, int64_t pattern = op1->AsIntConCommon()->IntegralValue(); simdmask_t simdVal; + if (pattern == SveMaskPatternLargestPowerOf2) + { + pattern = SveMaskPatternAll; + op1 = gtNewIconNode(pattern); + } + if (EvaluateSimdPatternToMask(simdBaseType, &simdVal, (SveMaskPattern)pattern)) { GenTreeMskCon* mskCon = gtNewMskConNode(retType); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b9fcf5c6e459dd..3d605e02bf5e60 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2717,6 +2717,11 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo GenTree* otherTreeNode = physRegRecord->assignedInterval->firstRefPosition->treeNode; noway_assert(otherTreeNode != nullptr); + if (refPosition->reuseConstantValue && (physRegRecord->assignedInterval == interval)) + { + return true; + } + if (refPosition->treeNode->OperGet() != otherTreeNode->OperGet()) { return false; @@ -2783,6 +2788,32 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo return false; } +//------------------------------------------------------------------------ +// setConstantReuse: Mark a repeated constant definition as reusable when its +// previous value survived allocation. +// +// Arguments: +// refPosition - The repeated constant definition +// previousReg - The register containing its previous value, if any +// assignedReg - The register selected for the definition +// +void LinearScan::setConstantReuse(RefPosition* refPosition, regNumber previousReg, regNumber assignedReg) +{ + if (!refPosition->reuseConstantValue) + { + return; + } + + if (assignedReg == previousReg) + { + refPosition->treeNode->SetReuseRegVal(); + } + else + { + refPosition->treeNode->ResetReuseRegVal(); + } +} + //------------------------------------------------------------------------ // allocateRegMinimal: Find a register that satisfies the requirements for refPosition, // taking into account the preferences for the given Interval, @@ -4556,9 +4587,9 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // Only focus on actual registers present deadCandidates &= actualRegistersMask; - handleDeadCandidates(deadCandidates.getLow(), REG_LOW_BASE, inVarToRegMap); + handleDeadCandidates(deadCandidates.getLow(), REG_LOW_BASE, inVarToRegMap, currentBlock); #ifdef HAS_MORE_THAN_64_REGISTERS - handleDeadCandidates(deadCandidates.getHigh(), REG_HIGH_BASE, inVarToRegMap); + handleDeadCandidates(deadCandidates.getHigh(), REG_HIGH_BASE, inVarToRegMap, currentBlock); #endif // HAS_MORE_THAN_64_REGISTERS #endif // TARGET_ARM } @@ -4570,11 +4601,15 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // deadCandidates - mask of registers. // regBase - base register number. // inVarToRegMap - variable to register map. +// currentBlock - block being entered. // // Return Value: // None // -void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBase, VarToRegMap inVarToRegMap) +void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, + int regBase, + VarToRegMap inVarToRegMap, + BasicBlock* currentBlock) { while (deadCandidates != RBM_NONE) { @@ -4588,6 +4623,16 @@ void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBa { assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); + RefPosition* nextRefPosition = assignedInterval->getNextRefPosition(); + if (assignedInterval->isConstant && (nextRefPosition != nullptr) && nextRefPosition->reuseConstantValue && + (nextRefPosition->bbNum == currentBlock->bbNum)) + { + // Keep the inactive constant associated with this available register so + // the definition in the successor can reuse it if it remains undisturbed. + setConstantReg(reg, assignedInterval->registerType); + continue; + } + if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) { assignedInterval->isActive = false; @@ -5094,7 +5139,8 @@ void LinearScan::allocateRegistersMinimal() currentInterval = currentRefPosition.getInterval(); assert(currentInterval != nullptr); assert(!currentInterval->isLocalVar); - assignedRegister = currentInterval->physReg; + assignedRegister = currentInterval->physReg; + regNumber reuseConstantReg = currentRefPosition.reuseConstantValue ? assignedRegister : REG_NA; // Identify the special cases where we decide up-front not to allocate bool allocate = true; @@ -5324,6 +5370,8 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { + setConstantReuse(¤tRefPosition, reuseConstantReg, assignedRegister); + assignedRegBit = genSingleTypeRegMask(assignedRegister); SingleTypeRegSet regMask = getSingleTypeRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation.AddRegsetForType(regMask, currentInterval->registerType); @@ -5822,7 +5870,8 @@ void LinearScan::allocateRegisters() assert(currentRefPosition.isIntervalRef()); currentInterval = currentRefPosition.getInterval(); assert(currentInterval != nullptr); - assignedRegister = currentInterval->physReg; + assignedRegister = currentInterval->physReg; + regNumber reuseConstantReg = currentRefPosition.reuseConstantValue ? assignedRegister : REG_NA; // Identify the special cases where we decide up-front not to allocate bool allocate = true; @@ -6556,6 +6605,8 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { + setConstantReuse(¤tRefPosition, reuseConstantReg, assignedRegister); + assignedRegBit = genSingleTypeRegMask(assignedRegister); SingleTypeRegSet regMask = getSingleTypeRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation.AddRegsetForType(regMask, currentInterval->registerType); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 3541e1a2309349..1816344ea7fc9a 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -999,7 +999,10 @@ class LinearScan : public RegAllocInterface // Record variable locations at start/end of block void processBlockStartLocations(BasicBlock* current); - FORCEINLINE void handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBase, VarToRegMap inVarToRegMap); + FORCEINLINE void handleDeadCandidates(SingleTypeRegSet deadCandidates, + int regBase, + VarToRegMap inVarToRegMap, + BasicBlock* currentBlock); void processBlockEndLocations(BasicBlock* current); void resetAllRegistersState(); @@ -1168,6 +1171,14 @@ class LinearScan : public RegAllocInterface regNumber assignCopyRegMinimal(RefPosition* refPosition); bool isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPosition); + void setConstantReuse(RefPosition* refPosition, regNumber previousReg, regNumber assignedReg); +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + bool tryGetReusableSveMaskInterval(GenTree* tree, Interval** interval); + void clearReusableSveMaskIntervals() + { + reusableSveMaskIntervals = nullptr; + } +#endif bool isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord); void checkAndAssignInterval(RegRecord* regRec, Interval* interval); void assignPhysReg(RegRecord* regRec, Interval* interval); @@ -2134,6 +2145,17 @@ class LinearScan : public RegAllocInterface unsigned availableRegCount; regNumber* regIndices; +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + struct SveMaskIntervalEntry + { + GenTree* tree; + Interval* interval; + SveMaskIntervalEntry* next; + }; + + SveMaskIntervalEntry* reusableSveMaskIntervals = nullptr; +#endif + FORCEINLINE unsigned get_AVAILABLE_REG_COUNT() const { return this->availableRegCount; @@ -2572,6 +2594,8 @@ class RefPosition unsigned char isPhysRegRef : 1; // true if 'referent' points of a RegRecord, false if it points to an Interval unsigned char isFixedRegRef : 1; unsigned char isLocalDefUse : 1; + // The definition is equivalent to an earlier definition in this interval. + unsigned char reuseConstantValue : 1; // delayRegFree indicates that the register should not be freed right away, but instead wait // until the next Location after it would normally be freed. This is used for the case of @@ -2637,6 +2661,7 @@ class RefPosition , isPhysRegRef(false) , isFixedRegRef(false) , isLocalDefUse(false) + , reuseConstantValue(false) , delayRegFree(false) , outOfOrder(false) #ifdef DEBUG diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index a59ed0b45c02b3..6da8b657882860 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1394,6 +1394,168 @@ int LinearScan::BuildNode(GenTree* tree) #include "hwintrinsic.h" +#if defined(FEATURE_MASKED_HW_INTRINSICS) +constexpr int PFalseConstantPattern = -1; + +struct SveMaskConstant +{ + insOpts opt; + int pattern; +}; + +//------------------------------------------------------------------------ +// TryGetSvePTrueOpt: Get the instruction option for an SVE ptrue base type. +// +// Arguments: +// baseType - The SVE element type. +// opt - [out] The corresponding instruction option. +// +// Return Value: +// True if baseType is supported; otherwise false. +// +static bool TryGetSvePTrueOpt(var_types baseType, insOpts* opt) +{ + switch (baseType) + { + case TYP_BYTE: + case TYP_UBYTE: + *opt = INS_OPTS_SCALABLE_B; + return true; + case TYP_SHORT: + case TYP_USHORT: + *opt = INS_OPTS_SCALABLE_H; + return true; + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + *opt = INS_OPTS_SCALABLE_S; + return true; + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + *opt = INS_OPTS_SCALABLE_D; + return true; + default: + return false; + } +} + +//------------------------------------------------------------------------ +// TryGetSveMaskConstant: Get the instruction option and pattern represented by an SVE mask node. +// +// Arguments: +// node - The mask node. +// value - [out] The mask constant description. +// +// Return Value: +// True if node represents an SVE mask constant; otherwise false. +// +static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) +{ + if (node->OperIs(GT_CNS_MSK)) + { + GenTreeMskCon* mask = node->AsMskCon(); + if (mask->IsZero()) + { + value->opt = INS_OPTS_SCALABLE_B; + value->pattern = PFalseConstantPattern; + return true; + } + + const var_types types[] = {TYP_BYTE, TYP_SHORT, TYP_INT, TYP_LONG}; + for (var_types type : types) + { + SveMaskPattern pattern = EvaluateSimdMaskToPattern(type, mask->gtSimdMaskVal); + if (pattern != SveMaskPatternNone) + { + bool found = TryGetSvePTrueOpt(type, &value->opt); + value->pattern = static_cast(pattern); + assert(found); + return true; + } + } + return false; + } + + if (!node->OperIsHWIntrinsic()) + { + return false; + } + + GenTreeHWIntrinsic* intrinsic = node->AsHWIntrinsic(); + NamedIntrinsic id = intrinsic->GetHWIntrinsicId(); + if (id == NI_Sve_ConversionTrueMask) + { + value->pattern = static_cast(SveMaskPatternAll); + return TryGetSvePTrueOpt(intrinsic->GetSimdBaseType(), &value->opt); + } + + if (!HWIntrinsicInfo::IsSveCreateTrueMask(id)) + { + return false; + } + + GenTree* pattern = intrinsic->Op(1); + if ((pattern == nullptr) || !pattern->IsCnsIntOrI() || + !TryGetSvePTrueOpt(intrinsic->GetSimdBaseType(), &value->opt)) + { + return false; + } + + value->pattern = static_cast(pattern->AsIntConCommon()->IntegralValue()); + return true; +} + +//------------------------------------------------------------------------ +// tryGetReusableSveMaskInterval: Find or record a reusable interval for an SVE mask constant. +// +// Arguments: +// tree - The SVE mask constant node. +// interval - [out] The reusable interval, or nullptr for a new reuse group. +// +// Return Value: +// True if tree represents an SVE mask constant; otherwise false. +// +bool LinearScan::tryGetReusableSveMaskInterval(GenTree* tree, Interval** interval) +{ + if (!m_compiler->opts.OptimizationEnabled()) + { + return false; + } + + SveMaskConstant value; + if (!TryGetSveMaskConstant(tree, &value)) + { + return false; + } + + for (SveMaskIntervalEntry* entry = reusableSveMaskIntervals; entry != nullptr; entry = entry->next) + { + SveMaskConstant entryValue; + if (TryGetSveMaskConstant(entry->tree, &entryValue) && (value.opt == entryValue.opt) && + (value.pattern == entryValue.pattern)) + { + *interval = entry->interval; + JITDUMP("SVE mask constant [%06u] reuses interval %u from [%06u]\n", Compiler::dspTreeID(tree), + entry->interval->intervalIndex, Compiler::dspTreeID(entry->tree)); + return true; + } + } + + // Keep one interval for equivalent masks. This extends the lifetime of the + // materialized predicate and lets later definitions reuse its register. + SveMaskIntervalEntry* entry = new (m_compiler, CMK_LSRA) SveMaskIntervalEntry; + entry->tree = tree; + entry->interval = nullptr; + entry->next = reusableSveMaskIntervals; + reusableSveMaskIntervals = entry; + *interval = nullptr; + JITDUMP("SVE mask constant [%06u] starts a reuse group\n", Compiler::dspTreeID(tree)); + return true; +} + +#endif // FEATURE_MASKED_HW_INTRINSICS + //------------------------------------------------------------------------ // BuildHWIntrinsic: Set the NodeInfo for a GT_HWINTRINSIC tree. // diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 4571e903b4215f..1c46b1f8d5d4a6 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -469,6 +469,17 @@ void LinearScan::associateRefPosWithInterval(RefPosition* rp) checkConflictingDefUse(rp); rp->lastUse = true; } + + if (theInterval->isConstant && RefTypeIsDef(rp->refType)) + { + // A later definition of the same constant shares this interval. + // Keep its preceding value live through that definition. + RefPosition* const prevRP = theInterval->recentRefPosition; + if ((prevRP != nullptr) && (prevRP->bbNum == rp->bbNum) && RefTypeIsUse(prevRP->refType)) + { + prevRP->lastUse = false; + } + } } RefPosition* prevRP = theReferent->recentRefPosition; @@ -1705,6 +1716,12 @@ int LinearScan::ComputeAvailableSrcCount(GenTree* node) // void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc) { +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (tree->OperIs(GT_CALL)) + { + clearReusableSveMaskIntervals(); + } +#endif #ifdef DEBUG if (VERBOSE) { @@ -2451,6 +2468,13 @@ void LinearScan::buildIntervals() } LIR::Range& blockRange = LIR::AsRange(block); +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if ((prevBlock == nullptr) || (block->GetUniquePred(m_compiler) != prevBlock) || + blockInfo[block->bbNum].hasEHBoundaryIn || blockInfo[prevBlock->bbNum].hasEHBoundaryOut) + { + clearReusableSveMaskIntervals(); + } +#endif for (GenTree* node : blockRange) { // We increment the location of each tree node by 2 so that the node definition, if any, @@ -3002,7 +3026,29 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, SingleTypeRegSet dstCandidates, needToKillFloatRegs = true; } - Interval* interval = newInterval(type); + Interval* interval = nullptr; +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + bool isSveMaskConstant = tryGetReusableSveMaskInterval(tree, &interval); +#else + bool isSveMaskConstant = false; +#endif + bool reuseSveMaskConstant = interval != nullptr; + if (interval == nullptr) + { + interval = newInterval(type); +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (isSveMaskConstant) + { + assert(reusableSveMaskIntervals->interval == nullptr); + reusableSveMaskIntervals->interval = interval; + } +#endif + } + + if (isSveMaskConstant) + { + interval->isConstant = true; + } if (tree->GetRegNum() != REG_NA) { if (!tree->IsMultiRegNode() || (multiRegIdx == 0)) @@ -3033,6 +3079,7 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, SingleTypeRegSet dstCandidates, } RefPosition* defRefPosition = newRefPosition(interval, currentLoc + 1, RefTypeDef, tree, dstCandidates, multiRegIdx); + defRefPosition->reuseConstantValue = reuseSveMaskConstant; if (tree->IsUnusedValue()) { defRefPosition->isLocalDefUse = true; diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs new file mode 100644 index 00000000000000..794efbbff5c51d --- /dev/null +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs @@ -0,0 +1,351 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +public class ConstantMaskReuse +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Consume(T value) + { + } + + [Fact] + public static void TestEntryPoint() + { + if (!Sve.IsSupported) + { + return; + } + + Vector a = Vector.Create(11); + Vector b = Vector.Create(22); + Vector c = Vector.Create(33); + int[] values = new int[32]; + Vector mask1 = Vector128.CreateScalar(0x1UL).AsVector(); + Vector mask2 = Vector128.CreateScalar(0x2UL).AsVector(); + + Consume(PTrueSingleCompareMask(a, b)); + Consume(PTrueSingleCreateTrueMask(a, b)); + Consume(PTrueSingleCreateTrueMaskPattern(a, b)); + Consume(PFalseSingleCreateFalseMask(a, b)); + Consume(PTrueSingleAllBitsMask(a, b)); + Consume(PTrueSingleEmbeddedMask(a, b)); + Consume(PTrueSingleConversionTrueMask(mask1)); + + Consume(PTrueMultipleCompareMask(a, b)); + Consume(PTrueMultipleCreateTrueMask(a, b, c)); + Consume(PTrueMultipleCreateTrueMaskPattern(a, b, c)); + Consume(PFalseMultipleCreateFalseMask(a, b, c)); + Consume(PFalseMultipleLoadMasks(values)); + Consume(PTrueMultipleAllBitsMask(a, b, c)); + Consume(PTrueMultipleEmbeddedMask(a, b, c)); + Consume(PTrueMultipleMixedSources(a, b, c)); + Consume(PTrueMultipleConversionTrueMask(mask1, mask2)); + Consume(PTrueSeparatedByCall(a, b)); + Consume(PTrueSeparateBlocks(a, Environment.TickCount != 0)); + Consume(PTrueUniquePredecessor(a, Environment.TickCount != 0)); + Consume(PTrueJoin(a, Environment.TickCount != 0)); + + if (Sve2.IsSupported) + { + Vector d = Vector.Create(42.0); + Vector e = Vector.Create((byte)42); + + Consume(PTrueMultipleSve2Log2Negate(d)); + Consume(PTrueMultipleSve2ZeroExtend8(e)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector Identity(Vector value) => value; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSeparatedByCall(Vector a, Vector b) + { + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64-NOT: ptrue {{p[0-9]+}}.s + Vector result1 = Sve.Abs(a); + Consume(result1); + Vector result2 = Sve.Negate(Identity(b)); + Consume(result2); + return result1; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSeparateBlocks(Vector value, bool condition) + { + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64-NOT: ptrue {{p[0-9]+}}.s + if (condition) + { + return Sve.Abs(value); + } + + return Sve.Negate(value); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueUniquePredecessor(Vector value, bool condition) + { + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64-NOT: ptrue {{p[0-9]+}}.s + Vector result = Sve.Abs(value); + if (condition) + { + return result; + } + + return Sve.Negate(result); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueJoin(Vector value, bool condition) + { + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64-NOT: ptrue {{p[0-9]+}}.s + Vector result = condition ? Sve.Abs(value) : Sve.Negate(value); + return Sve.Add(result, value); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleCompareMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + return Sve.CompareGreaterThan(a, b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleCreateTrueMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleCreateTrueMaskPattern(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PFalseSingleCreateFalseMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: pfalse {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateFalseMaskInt32()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleAllBitsMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + return Sve.ConditionalSelect(Vector.AllBitsSet, Sve.AbsoluteDifference(a, b), a); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleEmbeddedMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + return Sve.AbsoluteDifference(a, b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong PTrueSingleConversionTrueMask(Vector mask) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: cmpne {{p[0-9]+}}.d, {{p[0-9]+}}/z, {{z[0-9]+}}.d, #0 + //ARM64-FULL-LINE-NEXT: mov {{x[0-9]+}}, xzr + //ARM64-FULL-LINE-NEXT: sqdecp {{x[0-9]+}}, {{p[0-9]+}}.d + return Sve.SaturatingDecrementByActiveElementCount(0UL, mask); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleCompareMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CompareGreaterThan(a, b); + Vector mask2 = Sve.CompareLessThan(a, b); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, a); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleCreateTrueMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateTrueMaskInt32()); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleCreateTrueMaskPattern(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PFalseMultipleCreateFalseMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: pfalse {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateFalseMaskInt32()); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateFalseMaskInt32()); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector PFalseMultipleLoadMasks(int[] values) + { + //ARM64-FULL-LINE: pfalse {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: wrffr {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: ldnf1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [{{x[0-9]+}}] + //ARM64-FULL-LINE-NEXT: add {{x[0-9]+}}, {{x[0-9]+}}, #4 + //ARM64-FULL-LINE-NEXT: ldnf1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [{{x[0-9]+}}] + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + fixed (int* ptr = values) + { + Vector result1 = Sve.LoadVectorNonFaulting(Sve.CreateFalseMaskInt32(), ptr); + Vector result2 = Sve.LoadVectorNonFaulting(Vector.Zero, ptr + 1); + return Sve.Add(result1, result2); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleAllBitsMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector result1 = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.AbsoluteDifference(a, b), a); + Vector result2 = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.Abs(c), c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleEmbeddedMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector result1 = Sve.AbsoluteDifference(a, b); + Vector result2 = Sve.Abs(c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleMixedSources(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); + Vector result1 = Sve.ConditionalSelect(mask, Sve.AbsoluteDifference(a, b), a); + Vector result2 = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.Abs(c), c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong PTrueMultipleConversionTrueMask(Vector mask1, Vector mask2) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: cmpne {{p[0-9]+}}.d, {{p[0-9]+}}/z, {{z[0-9]+}}.d, #0 + //ARM64-FULL-LINE-NEXT: mov {{x[0-9]+}}, xzr + //ARM64-FULL-LINE-NEXT: sqdecp {{x[0-9]+}}, {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: cmpne {{p[0-9]+}}.d, {{p[0-9]+}}/z, {{z[0-9]+}}.d, #0 + //ARM64-FULL-LINE-NEXT: mov {{x[0-9]+}}, xzr + //ARM64-FULL-LINE-NEXT: sqdecp {{x[0-9]+}}, {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}} + ulong result1 = Sve.SaturatingDecrementByActiveElementCount(0UL, mask1); + ulong result2 = Sve.SaturatingDecrementByActiveElementCount(0UL, mask2); + return result1 + result2; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleSve2Log2Negate(Vector value) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: flogb {{z[0-9]+}}.d, {{p[0-9]+}}/m, {{z[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: neg {{z[0-9]+}}.d, {{p[0-9]+}}/m, {{z[0-9]+}}.d + Vector exponent = Sve2.Log2(value); + Vector scale = Sve.Negate(exponent); + return Sve.Scale(value, scale); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleSve2ZeroExtend8(Vector value) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: uxtb {{z[0-9]+}}.h, {{p[0-9]+}}/m, {{z[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: uxtb {{z[0-9]+}}.h, {{p[0-9]+}}/m, {{z[0-9]+}}.h + Vector result1 = Sve.ZeroExtend8((Vector)value); + Vector result2 = Sve.ZeroExtend8((Vector)Sve.Add(value, value)); + return Sve.Add(result1, result2); + } +} diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj new file mode 100644 index 00000000000000..4c249bcb771a5f --- /dev/null +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + From 43fecb11da09d7eb0ca5d143ad39c14577c7748f Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 28 Jul 2026 09:32:15 +0000 Subject: [PATCH 2/7] Reuse SVE mask constants in optimized cod --- src/coreclr/jit/hwintrinsicarm64.cpp | 6 - src/coreclr/jit/lsra.cpp | 136 +++++++++--------- src/coreclr/jit/lsra.h | 27 +--- src/coreclr/jit/lsraarm64.cpp | 81 +++++------ src/coreclr/jit/lsrabuild.cpp | 49 +------ src/tests/JIT/opt/SVE/ConstantMaskReuse.cs | 43 +++++- .../JIT/opt/SVE/ConstantMaskReuse.csproj | 4 +- 7 files changed, 156 insertions(+), 190 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 6d598980c77188..e5de5e0e574c0b 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1035,12 +1035,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, int64_t pattern = op1->AsIntConCommon()->IntegralValue(); simdmask_t simdVal; - if (pattern == SveMaskPatternLargestPowerOf2) - { - pattern = SveMaskPatternAll; - op1 = gtNewIconNode(pattern); - } - if (EvaluateSimdPatternToMask(simdBaseType, &simdVal, (SveMaskPattern)pattern)) { GenTreeMskCon* mskCon = gtNewMskConNode(retType); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 3d605e02bf5e60..16d8f9423ef7fe 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2717,10 +2717,12 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo GenTree* otherTreeNode = physRegRecord->assignedInterval->firstRefPosition->treeNode; noway_assert(otherTreeNode != nullptr); - if (refPosition->reuseConstantValue && (physRegRecord->assignedInterval == interval)) +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (areMatchingSveMaskConstants(refPosition->treeNode, otherTreeNode)) { return true; } +#endif if (refPosition->treeNode->OperGet() != otherTreeNode->OperGet()) { @@ -2788,32 +2790,6 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo return false; } -//------------------------------------------------------------------------ -// setConstantReuse: Mark a repeated constant definition as reusable when its -// previous value survived allocation. -// -// Arguments: -// refPosition - The repeated constant definition -// previousReg - The register containing its previous value, if any -// assignedReg - The register selected for the definition -// -void LinearScan::setConstantReuse(RefPosition* refPosition, regNumber previousReg, regNumber assignedReg) -{ - if (!refPosition->reuseConstantValue) - { - return; - } - - if (assignedReg == previousReg) - { - refPosition->treeNode->SetReuseRegVal(); - } - else - { - refPosition->treeNode->ResetReuseRegVal(); - } -} - //------------------------------------------------------------------------ // allocateRegMinimal: Find a register that satisfies the requirements for refPosition, // taking into account the preferences for the given Interval, @@ -3976,6 +3952,15 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) assert(currentBlock != nullptr); markBlockVisited(currentBlock); + BasicBlock* nextBlock = getNextBlock(); + bool preserveMaskConstants = false; +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + preserveMaskConstants = m_compiler->opts.OptimizationEnabled() && (nextBlock != nullptr) && + (nextBlock->GetUniquePred(m_compiler) == currentBlock) && + !blockInfo[nextBlock->bbNum].hasEHBoundaryIn && + !blockInfo[currentBlock->bbNum].hasEHBoundaryOut; +#endif + if (localVarsEnregistered) { processBlockEndLocations(currentBlock); @@ -3984,15 +3969,14 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) // When the last block in the method has successors, there will be a final "RefTypeBB" to // ensure that we get the varToRegMap set appropriately, but in that case we don't need // to worry about "nextBlock". - BasicBlock* nextBlock = getNextBlock(); if (nextBlock != nullptr) { - processBlockStartLocations(nextBlock); + processBlockStartLocations(nextBlock, preserveMaskConstants); } } else { - resetAllRegistersState(); + resetAllRegistersState(preserveMaskConstants); } } @@ -4241,10 +4225,13 @@ void LinearScan::unassignIntervalBlockStart(RegRecord* regRecord, VarToRegMap in } //------------------------------------------------------------------------ -// resetAllRegistersState: Resets the next interval ref, spill cost and clears -// the constant registers. +// resetAllRegistersState: Resets the next interval ref and spill cost, and optionally +// preserves mask constants. +// +// Arguments: +// preserveMaskConstants - whether mask constants survive from the preceding block // -void LinearScan::resetAllRegistersState() +void LinearScan::resetAllRegistersState(bool preserveMaskConstants) { assert(!enregisterLocalVars); // Just clear any constant registers and return. @@ -4254,21 +4241,29 @@ void LinearScan::resetAllRegistersState() int regIndex = REG_FIRST; for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; NEXT_REGISTER(reg, regIndex)) { - RegRecord* physRegRecord = getRegisterRecord(reg); + RegRecord* physRegRecord = getRegisterRecord(reg); + Interval* assignedInterval = physRegRecord->assignedInterval; #ifdef DEBUG - Interval* assignedInterval = physRegRecord->assignedInterval; assert(assignedInterval == nullptr || assignedInterval->isConstant); #endif - physRegRecord->assignedInterval = nullptr; + if (preserveMaskConstants && (assignedInterval != nullptr) && varTypeIsMask(assignedInterval->registerType)) + { + setConstantReg(reg, assignedInterval->registerType); + } + else + { + physRegRecord->assignedInterval = nullptr; + } } } //------------------------------------------------------------------------ -// processBlockStartLocations: Update var locations on entry to 'currentBlock' and clear constant -// registers. +// processBlockStartLocations: Update var locations on entry to 'currentBlock' and update +// the constant register state. // // Arguments: -// currentBlock - the BasicBlock we are about to allocate registers for +// currentBlock - the BasicBlock we are about to allocate registers for +// preserveMaskConstants - whether mask constants survive from the preceding block // // Return Value: // None @@ -4280,7 +4275,7 @@ void LinearScan::resetAllRegistersState() // modify the inVarToRegMap in cases where a lclVar was spilled after the block had been // completed. // -void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) +void LinearScan::processBlockStartLocations(BasicBlock* currentBlock, bool preserveMaskConstants) { // We should only call this method if we have register candidates. @@ -4587,9 +4582,9 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // Only focus on actual registers present deadCandidates &= actualRegistersMask; - handleDeadCandidates(deadCandidates.getLow(), REG_LOW_BASE, inVarToRegMap, currentBlock); + handleDeadCandidates(deadCandidates.getLow(), REG_LOW_BASE, inVarToRegMap, preserveMaskConstants); #ifdef HAS_MORE_THAN_64_REGISTERS - handleDeadCandidates(deadCandidates.getHigh(), REG_HIGH_BASE, inVarToRegMap, currentBlock); + handleDeadCandidates(deadCandidates.getHigh(), REG_HIGH_BASE, inVarToRegMap, preserveMaskConstants); #endif // HAS_MORE_THAN_64_REGISTERS #endif // TARGET_ARM } @@ -4601,7 +4596,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // deadCandidates - mask of registers. // regBase - base register number. // inVarToRegMap - variable to register map. -// currentBlock - block being entered. +// preserveMaskConstants - whether mask constants survive from the preceding block. // // Return Value: // None @@ -4609,7 +4604,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBase, VarToRegMap inVarToRegMap, - BasicBlock* currentBlock) + bool preserveMaskConstants) { while (deadCandidates != RBM_NONE) { @@ -4623,12 +4618,10 @@ void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, { assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); - RefPosition* nextRefPosition = assignedInterval->getNextRefPosition(); - if (assignedInterval->isConstant && (nextRefPosition != nullptr) && nextRefPosition->reuseConstantValue && - (nextRefPosition->bbNum == currentBlock->bbNum)) + if (preserveMaskConstants && assignedInterval->isConstant && varTypeIsMask(assignedInterval->registerType)) { - // Keep the inactive constant associated with this available register so - // the definition in the successor can reuse it if it remains undisturbed. + // Keep the mask constant associated with this available register so a + // matching definition in the successor can reuse it. setConstantReg(reg, assignedInterval->registerType); continue; } @@ -5139,8 +5132,7 @@ void LinearScan::allocateRegistersMinimal() currentInterval = currentRefPosition.getInterval(); assert(currentInterval != nullptr); assert(!currentInterval->isLocalVar); - assignedRegister = currentInterval->physReg; - regNumber reuseConstantReg = currentRefPosition.reuseConstantValue ? assignedRegister : REG_NA; + assignedRegister = currentInterval->physReg; // Identify the special cases where we decide up-front not to allocate bool allocate = true; @@ -5370,8 +5362,6 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { - setConstantReuse(¤tRefPosition, reuseConstantReg, assignedRegister); - assignedRegBit = genSingleTypeRegMask(assignedRegister); SingleTypeRegSet regMask = getSingleTypeRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation.AddRegsetForType(regMask, currentInterval->registerType); @@ -5870,8 +5860,7 @@ void LinearScan::allocateRegisters() assert(currentRefPosition.isIntervalRef()); currentInterval = currentRefPosition.getInterval(); assert(currentInterval != nullptr); - assignedRegister = currentInterval->physReg; - regNumber reuseConstantReg = currentRefPosition.reuseConstantValue ? assignedRegister : REG_NA; + assignedRegister = currentInterval->physReg; // Identify the special cases where we decide up-front not to allocate bool allocate = true; @@ -6605,8 +6594,6 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { - setConstantReuse(¤tRefPosition, reuseConstantReg, assignedRegister); - assignedRegBit = genSingleTypeRegMask(assignedRegister); SingleTypeRegSet regMask = getSingleTypeRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation.AddRegsetForType(regMask, currentInterval->registerType); @@ -7971,7 +7958,7 @@ void LinearScan::resolveRegisters() curBBStartLocation = currentRefPosition->nodeLocation; if (block != m_compiler->fgFirstBB) { - processBlockStartLocations(block); + processBlockStartLocations(block, false); } // Handle the DummyDefs, updating the incoming var location. @@ -13789,6 +13776,31 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* freeCandidates = linearScan->getFreeCandidates(candidates, regType); } + if (freeCandidates != RBM_NONE) + { + // Set the 'matchingConstants' set. + if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) + { + matchingConstants = linearScan->getMatchingConstants(candidates, currentInterval, refPosition); + } + +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (linearScan->m_compiler->opts.OptimizationEnabled() && varTypeIsMask(regType)) + { + // Avoid overwriting an available mask constant when another free register exists. A later + // definition can then reuse the constant; matching constants remain preferred candidates. + SingleTypeRegSet constantsToPreserve = + linearScan->m_RegistersWithConstants.GetRegSetForType(regType) & ~matchingConstants & ~fixedRegMask; + SingleTypeRegSet remainingFreeCandidates = freeCandidates & ~constantsToPreserve; + if (remainingFreeCandidates != RBM_NONE) + { + candidates &= ~constantsToPreserve; + freeCandidates = remainingFreeCandidates; + } + } +#endif + } + // If no free candidates, then double check if refPosition is an actual ref. if (freeCandidates == RBM_NONE) { @@ -13799,14 +13811,6 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* return RBM_NONE; } } - else - { - // Set the 'matchingConstants' set. - if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) - { - matchingConstants = linearScan->getMatchingConstants(candidates, currentInterval, refPosition); - } - } #define IF_FOUND_GOTO_DONE \ if (found) \ diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 1816344ea7fc9a..38af3cc15b4fd5 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -997,14 +997,14 @@ class LinearScan : public RegAllocInterface void processBlockEndAllocation(BasicBlock* current); // Record variable locations at start/end of block - void processBlockStartLocations(BasicBlock* current); + void processBlockStartLocations(BasicBlock* current, bool preserveMaskConstants); FORCEINLINE void handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBase, VarToRegMap inVarToRegMap, - BasicBlock* currentBlock); + bool preserveMaskConstants); void processBlockEndLocations(BasicBlock* current); - void resetAllRegistersState(); + void resetAllRegistersState(bool preserveMaskConstants); #ifdef TARGET_ARM bool isSecondHalfReg(RegRecord* regRec, Interval* interval); @@ -1171,13 +1171,8 @@ class LinearScan : public RegAllocInterface regNumber assignCopyRegMinimal(RefPosition* refPosition); bool isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPosition); - void setConstantReuse(RefPosition* refPosition, regNumber previousReg, regNumber assignedReg); #if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - bool tryGetReusableSveMaskInterval(GenTree* tree, Interval** interval); - void clearReusableSveMaskIntervals() - { - reusableSveMaskIntervals = nullptr; - } + bool areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2); #endif bool isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord); void checkAndAssignInterval(RegRecord* regRec, Interval* interval); @@ -2145,17 +2140,6 @@ class LinearScan : public RegAllocInterface unsigned availableRegCount; regNumber* regIndices; -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - struct SveMaskIntervalEntry - { - GenTree* tree; - Interval* interval; - SveMaskIntervalEntry* next; - }; - - SveMaskIntervalEntry* reusableSveMaskIntervals = nullptr; -#endif - FORCEINLINE unsigned get_AVAILABLE_REG_COUNT() const { return this->availableRegCount; @@ -2594,8 +2578,6 @@ class RefPosition unsigned char isPhysRegRef : 1; // true if 'referent' points of a RegRecord, false if it points to an Interval unsigned char isFixedRegRef : 1; unsigned char isLocalDefUse : 1; - // The definition is equivalent to an earlier definition in this interval. - unsigned char reuseConstantValue : 1; // delayRegFree indicates that the register should not be freed right away, but instead wait // until the next Location after it would normally be freed. This is used for the case of @@ -2661,7 +2643,6 @@ class RefPosition , isPhysRegRef(false) , isFixedRegRef(false) , isLocalDefUse(false) - , reuseConstantValue(false) , delayRegFree(false) , outOfOrder(false) #ifdef DEBUG diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 6da8b657882860..4157bb0e04a5b2 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1403,6 +1403,25 @@ struct SveMaskConstant int pattern; }; +//------------------------------------------------------------------------ +// NormalizeSveMaskPattern: Canonicalize equivalent SVE mask patterns. +// +// Arguments: +// pattern - The SVE mask pattern. +// +// Return Value: +// The canonical pattern. +// +static int NormalizeSveMaskPattern(int pattern) +{ + if (pattern == SveMaskPatternLargestPowerOf2) + { + return SveMaskPatternAll; + } + + return pattern; +} + //------------------------------------------------------------------------ // TryGetSvePTrueOpt: Get the instruction option for an SVE ptrue base type. // @@ -1469,7 +1488,7 @@ static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) if (pattern != SveMaskPatternNone) { bool found = TryGetSvePTrueOpt(type, &value->opt); - value->pattern = static_cast(pattern); + value->pattern = NormalizeSveMaskPattern(static_cast(pattern)); assert(found); return true; } @@ -1502,56 +1521,26 @@ static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) return false; } - value->pattern = static_cast(pattern->AsIntConCommon()->IntegralValue()); + value->pattern = NormalizeSveMaskPattern(static_cast(pattern->AsIntConCommon()->IntegralValue())); return true; } //------------------------------------------------------------------------ -// tryGetReusableSveMaskInterval: Find or record a reusable interval for an SVE mask constant. +// areMatchingSveMaskConstants: Check whether two nodes materialize the same SVE mask constant. // // Arguments: -// tree - The SVE mask constant node. -// interval - [out] The reusable interval, or nullptr for a new reuse group. +// tree1 - The first node. +// tree2 - The second node. // // Return Value: -// True if tree represents an SVE mask constant; otherwise false. +// True if both nodes represent the same SVE mask constant; otherwise false. // -bool LinearScan::tryGetReusableSveMaskInterval(GenTree* tree, Interval** interval) +bool LinearScan::areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2) { - if (!m_compiler->opts.OptimizationEnabled()) - { - return false; - } - - SveMaskConstant value; - if (!TryGetSveMaskConstant(tree, &value)) - { - return false; - } - - for (SveMaskIntervalEntry* entry = reusableSveMaskIntervals; entry != nullptr; entry = entry->next) - { - SveMaskConstant entryValue; - if (TryGetSveMaskConstant(entry->tree, &entryValue) && (value.opt == entryValue.opt) && - (value.pattern == entryValue.pattern)) - { - *interval = entry->interval; - JITDUMP("SVE mask constant [%06u] reuses interval %u from [%06u]\n", Compiler::dspTreeID(tree), - entry->interval->intervalIndex, Compiler::dspTreeID(entry->tree)); - return true; - } - } - - // Keep one interval for equivalent masks. This extends the lifetime of the - // materialized predicate and lets later definitions reuse its register. - SveMaskIntervalEntry* entry = new (m_compiler, CMK_LSRA) SveMaskIntervalEntry; - entry->tree = tree; - entry->interval = nullptr; - entry->next = reusableSveMaskIntervals; - reusableSveMaskIntervals = entry; - *interval = nullptr; - JITDUMP("SVE mask constant [%06u] starts a reuse group\n", Compiler::dspTreeID(tree)); - return true; + SveMaskConstant value1; + SveMaskConstant value2; + return TryGetSveMaskConstant(tree1, &value1) && TryGetSveMaskConstant(tree2, &value2) && + (value1.opt == value2.opt) && (value1.pattern == value2.pattern); } #endif // FEATURE_MASKED_HW_INTRINSICS @@ -1726,7 +1715,15 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou } else if ((dstCount == 1) || (dstCount == 2)) { - BuildDef(intrinsicTree); + RefPosition* def = BuildDef(intrinsicTree); + +#if defined(FEATURE_MASKED_HW_INTRINSICS) + SveMaskConstant value; + if (m_compiler->opts.OptimizationEnabled() && TryGetSveMaskConstant(intrinsicTree, &value)) + { + def->getInterval()->isConstant = true; + } +#endif if (dstCount == 2) { diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 1c46b1f8d5d4a6..4571e903b4215f 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -469,17 +469,6 @@ void LinearScan::associateRefPosWithInterval(RefPosition* rp) checkConflictingDefUse(rp); rp->lastUse = true; } - - if (theInterval->isConstant && RefTypeIsDef(rp->refType)) - { - // A later definition of the same constant shares this interval. - // Keep its preceding value live through that definition. - RefPosition* const prevRP = theInterval->recentRefPosition; - if ((prevRP != nullptr) && (prevRP->bbNum == rp->bbNum) && RefTypeIsUse(prevRP->refType)) - { - prevRP->lastUse = false; - } - } } RefPosition* prevRP = theReferent->recentRefPosition; @@ -1716,12 +1705,6 @@ int LinearScan::ComputeAvailableSrcCount(GenTree* node) // void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc) { -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - if (tree->OperIs(GT_CALL)) - { - clearReusableSveMaskIntervals(); - } -#endif #ifdef DEBUG if (VERBOSE) { @@ -2468,13 +2451,6 @@ void LinearScan::buildIntervals() } LIR::Range& blockRange = LIR::AsRange(block); -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - if ((prevBlock == nullptr) || (block->GetUniquePred(m_compiler) != prevBlock) || - blockInfo[block->bbNum].hasEHBoundaryIn || blockInfo[prevBlock->bbNum].hasEHBoundaryOut) - { - clearReusableSveMaskIntervals(); - } -#endif for (GenTree* node : blockRange) { // We increment the location of each tree node by 2 so that the node definition, if any, @@ -3026,29 +3002,7 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, SingleTypeRegSet dstCandidates, needToKillFloatRegs = true; } - Interval* interval = nullptr; -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - bool isSveMaskConstant = tryGetReusableSveMaskInterval(tree, &interval); -#else - bool isSveMaskConstant = false; -#endif - bool reuseSveMaskConstant = interval != nullptr; - if (interval == nullptr) - { - interval = newInterval(type); -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - if (isSveMaskConstant) - { - assert(reusableSveMaskIntervals->interval == nullptr); - reusableSveMaskIntervals->interval = interval; - } -#endif - } - - if (isSveMaskConstant) - { - interval->isConstant = true; - } + Interval* interval = newInterval(type); if (tree->GetRegNum() != REG_NA) { if (!tree->IsMultiRegNode() || (multiRegIdx == 0)) @@ -3079,7 +3033,6 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, SingleTypeRegSet dstCandidates, } RefPosition* defRefPosition = newRefPosition(interval, currentLoc + 1, RefTypeDef, tree, dstCandidates, multiRegIdx); - defRefPosition->reuseConstantValue = reuseSveMaskConstant; if (tree->IsUnusedValue()) { defRefPosition->isLocalDefUse = true; diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs index 794efbbff5c51d..4c5fb4c805fcdf 100644 --- a/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs @@ -46,6 +46,8 @@ public static void TestEntryPoint() Consume(PTrueMultipleAllBitsMask(a, b, c)); Consume(PTrueMultipleEmbeddedMask(a, b, c)); Consume(PTrueMultipleMixedSources(a, b, c)); + Consume(PTrueDifferentPatterns(a, b, c)); + Consume(PTrueDifferentElementSizes(a)); Consume(PTrueMultipleConversionTrueMask(mask1, mask2)); Consume(PTrueSeparatedByCall(a, b)); Consume(PTrueSeparateBlocks(a, Environment.TickCount != 0)); @@ -121,7 +123,7 @@ private static Vector PTrueSingleCompareMask(Vector a, Vector b) { //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s - //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 return Sve.CompareGreaterThan(a, b); } @@ -131,7 +133,7 @@ private static Vector PTrueSingleCreateTrueMask(Vector a, Vector //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b - //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); } @@ -142,7 +144,7 @@ private static Vector PTrueSingleCreateTrueMaskPattern(Vector a, Vecto //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b - //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #1 + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); } @@ -153,7 +155,7 @@ private static Vector PFalseSingleCreateFalseMask(Vector a, Vector PTrueMultipleMixedSources(Vector a, Vector return Sve.Add(result1, result2); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueDifferentPatterns(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.s, vl2 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount2)); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueDifferentElementSizes(Vector value) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: neg {{z[0-9]+}}.h, {{p[0-9]+}}/m, {{z[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector result1 = Sve.Abs(value); + Vector result2 = Sve.Negate((Vector)value); + return Sve.Add(result1, (Vector)result2); + } + [MethodImpl(MethodImplOptions.NoInlining)] private static ulong PTrueMultipleConversionTrueMask(Vector mask1, Vector mask2) { diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj index 4c249bcb771a5f..2a23f58397625f 100644 --- a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj @@ -5,7 +5,9 @@ True - + + true + From 3ce1a1ab4facfec71feadd9e1aaecfb0dc97229e Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 28 Jul 2026 12:44:14 +0000 Subject: [PATCH 3/7] Restrict SVE mask reuse to scalable mask constants --- src/coreclr/jit/codegencommon.cpp | 2 +- src/coreclr/jit/gentree.h | 6 +- src/coreclr/jit/hwintrinsic.h | 12 -- src/coreclr/jit/lsra.cpp | 18 +- src/coreclr/jit/lsra.h | 3 - src/coreclr/jit/lsraarm64.cpp | 161 +----------------- .../JIT/opt/SVE/ConstantMaskReuse.csproj | 1 + 7 files changed, 12 insertions(+), 191 deletions(-) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 980e150bc00602..5f52dd38575843 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8670,7 +8670,7 @@ void CodeGen::genCodeForReuseVal(GenTree* treeNode) assert(treeNode->IsReuseRegVal()); #if defined(FEATURE_MASKED_HW_INTRINSICS) - assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK, GT_HWINTRINSIC)); + assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK)); #elif defined(FEATURE_SIMD) assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC)); #else diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 29ad165f4d1767..7a99ae22253778 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -2546,18 +2546,18 @@ struct GenTree bool IsReuseRegVal() const { - return (OperIsConst() || OperIsHWIntrinsic()) && ((gtFlags & GTF_REUSE_REG_VAL) != 0); + return OperIsConst() && ((gtFlags & GTF_REUSE_REG_VAL) != 0); } void SetReuseRegVal() { - assert(OperIsConst() || OperIsHWIntrinsic()); + assert(OperIsConst()); gtFlags |= GTF_REUSE_REG_VAL; } void ResetReuseRegVal() { - assert(OperIsConst() || OperIsHWIntrinsic()); + assert(OperIsConst()); gtFlags &= ~GTF_REUSE_REG_VAL; } diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 2db1338c259955..7fb17d29509b54 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1028,18 +1028,6 @@ struct HWIntrinsicInfo return (flags & HW_Flag_Scalable) != 0; } -#ifdef FEATURE_MASKED_HW_INTRINSICS - static bool IsSveCreateTrueMask(NamedIntrinsic id) - { - static_assert(AreContiguous(NI_Sve_CreateTrueMaskByte, NI_Sve_CreateTrueMaskDouble, NI_Sve_CreateTrueMaskInt16, - NI_Sve_CreateTrueMaskInt32, NI_Sve_CreateTrueMaskInt64, NI_Sve_CreateTrueMaskSByte, - NI_Sve_CreateTrueMaskSingle, NI_Sve_CreateTrueMaskUInt16, - NI_Sve_CreateTrueMaskUInt32, NI_Sve_CreateTrueMaskUInt64)); - return (id >= NI_Sve_CreateTrueMaskByte) && (id <= NI_Sve_CreateTrueMaskUInt64); - } - -#endif // FEATURE_MASKED_HW_INTRINSICS - static bool IsLowMaskedOperation(NamedIntrinsic id) { const HWIntrinsicFlag flags = lookupFlags(id); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 16d8f9423ef7fe..b9ff24aceea09c 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2717,13 +2717,6 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo GenTree* otherTreeNode = physRegRecord->assignedInterval->firstRefPosition->treeNode; noway_assert(otherTreeNode != nullptr); -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - if (areMatchingSveMaskConstants(refPosition->treeNode, otherTreeNode)) - { - return true; - } -#endif - if (refPosition->treeNode->OperGet() != otherTreeNode->OperGet()) { return false; @@ -3954,9 +3947,9 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) BasicBlock* nextBlock = getNextBlock(); bool preserveMaskConstants = false; -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - preserveMaskConstants = m_compiler->opts.OptimizationEnabled() && (nextBlock != nullptr) && - (nextBlock->GetUniquePred(m_compiler) == currentBlock) && +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) && defined(DEBUG) + preserveMaskConstants = JitConfig.JitUseScalableVectorT() && m_compiler->opts.OptimizationEnabled() && + (nextBlock != nullptr) && (nextBlock->GetUniquePred(m_compiler) == currentBlock) && !blockInfo[nextBlock->bbNum].hasEHBoundaryIn && !blockInfo[currentBlock->bbNum].hasEHBoundaryOut; #endif @@ -13784,8 +13777,9 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* matchingConstants = linearScan->getMatchingConstants(candidates, currentInterval, refPosition); } -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - if (linearScan->m_compiler->opts.OptimizationEnabled() && varTypeIsMask(regType)) +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) && defined(DEBUG) + if (JitConfig.JitUseScalableVectorT() && linearScan->m_compiler->opts.OptimizationEnabled() && + varTypeIsMask(regType)) { // Avoid overwriting an available mask constant when another free register exists. A later // definition can then reuse the constant; matching constants remain preferred candidates. diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 38af3cc15b4fd5..733a5fdcb058ee 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1171,9 +1171,6 @@ class LinearScan : public RegAllocInterface regNumber assignCopyRegMinimal(RefPosition* refPosition); bool isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPosition); -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - bool areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2); -#endif bool isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord); void checkAndAssignInterval(RegRecord* regRec, Interval* interval); void assignPhysReg(RegRecord* regRec, Interval* interval); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 4157bb0e04a5b2..a59ed0b45c02b3 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1394,157 +1394,6 @@ int LinearScan::BuildNode(GenTree* tree) #include "hwintrinsic.h" -#if defined(FEATURE_MASKED_HW_INTRINSICS) -constexpr int PFalseConstantPattern = -1; - -struct SveMaskConstant -{ - insOpts opt; - int pattern; -}; - -//------------------------------------------------------------------------ -// NormalizeSveMaskPattern: Canonicalize equivalent SVE mask patterns. -// -// Arguments: -// pattern - The SVE mask pattern. -// -// Return Value: -// The canonical pattern. -// -static int NormalizeSveMaskPattern(int pattern) -{ - if (pattern == SveMaskPatternLargestPowerOf2) - { - return SveMaskPatternAll; - } - - return pattern; -} - -//------------------------------------------------------------------------ -// TryGetSvePTrueOpt: Get the instruction option for an SVE ptrue base type. -// -// Arguments: -// baseType - The SVE element type. -// opt - [out] The corresponding instruction option. -// -// Return Value: -// True if baseType is supported; otherwise false. -// -static bool TryGetSvePTrueOpt(var_types baseType, insOpts* opt) -{ - switch (baseType) - { - case TYP_BYTE: - case TYP_UBYTE: - *opt = INS_OPTS_SCALABLE_B; - return true; - case TYP_SHORT: - case TYP_USHORT: - *opt = INS_OPTS_SCALABLE_H; - return true; - case TYP_INT: - case TYP_UINT: - case TYP_FLOAT: - *opt = INS_OPTS_SCALABLE_S; - return true; - case TYP_LONG: - case TYP_ULONG: - case TYP_DOUBLE: - *opt = INS_OPTS_SCALABLE_D; - return true; - default: - return false; - } -} - -//------------------------------------------------------------------------ -// TryGetSveMaskConstant: Get the instruction option and pattern represented by an SVE mask node. -// -// Arguments: -// node - The mask node. -// value - [out] The mask constant description. -// -// Return Value: -// True if node represents an SVE mask constant; otherwise false. -// -static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) -{ - if (node->OperIs(GT_CNS_MSK)) - { - GenTreeMskCon* mask = node->AsMskCon(); - if (mask->IsZero()) - { - value->opt = INS_OPTS_SCALABLE_B; - value->pattern = PFalseConstantPattern; - return true; - } - - const var_types types[] = {TYP_BYTE, TYP_SHORT, TYP_INT, TYP_LONG}; - for (var_types type : types) - { - SveMaskPattern pattern = EvaluateSimdMaskToPattern(type, mask->gtSimdMaskVal); - if (pattern != SveMaskPatternNone) - { - bool found = TryGetSvePTrueOpt(type, &value->opt); - value->pattern = NormalizeSveMaskPattern(static_cast(pattern)); - assert(found); - return true; - } - } - return false; - } - - if (!node->OperIsHWIntrinsic()) - { - return false; - } - - GenTreeHWIntrinsic* intrinsic = node->AsHWIntrinsic(); - NamedIntrinsic id = intrinsic->GetHWIntrinsicId(); - if (id == NI_Sve_ConversionTrueMask) - { - value->pattern = static_cast(SveMaskPatternAll); - return TryGetSvePTrueOpt(intrinsic->GetSimdBaseType(), &value->opt); - } - - if (!HWIntrinsicInfo::IsSveCreateTrueMask(id)) - { - return false; - } - - GenTree* pattern = intrinsic->Op(1); - if ((pattern == nullptr) || !pattern->IsCnsIntOrI() || - !TryGetSvePTrueOpt(intrinsic->GetSimdBaseType(), &value->opt)) - { - return false; - } - - value->pattern = NormalizeSveMaskPattern(static_cast(pattern->AsIntConCommon()->IntegralValue())); - return true; -} - -//------------------------------------------------------------------------ -// areMatchingSveMaskConstants: Check whether two nodes materialize the same SVE mask constant. -// -// Arguments: -// tree1 - The first node. -// tree2 - The second node. -// -// Return Value: -// True if both nodes represent the same SVE mask constant; otherwise false. -// -bool LinearScan::areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2) -{ - SveMaskConstant value1; - SveMaskConstant value2; - return TryGetSveMaskConstant(tree1, &value1) && TryGetSveMaskConstant(tree2, &value2) && - (value1.opt == value2.opt) && (value1.pattern == value2.pattern); -} - -#endif // FEATURE_MASKED_HW_INTRINSICS - //------------------------------------------------------------------------ // BuildHWIntrinsic: Set the NodeInfo for a GT_HWINTRINSIC tree. // @@ -1715,15 +1564,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou } else if ((dstCount == 1) || (dstCount == 2)) { - RefPosition* def = BuildDef(intrinsicTree); - -#if defined(FEATURE_MASKED_HW_INTRINSICS) - SveMaskConstant value; - if (m_compiler->opts.OptimizationEnabled() && TryGetSveMaskConstant(intrinsicTree, &value)) - { - def->getInterval()->isConstant = true; - } -#endif + BuildDef(intrinsicTree); if (dstCount == 2) { diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj index 2a23f58397625f..8202afed636e8f 100644 --- a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj @@ -10,5 +10,6 @@ + From 8b3f1540eb83be13cf6ab84fa8fcadc22b263a8d Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 3 Aug 2026 08:19:09 +0000 Subject: [PATCH 4/7] Reuse SVE mask constants through existing LSRA machinery --- src/coreclr/jit/lowerarmarch.cpp | 21 +++ src/coreclr/jit/lsra.cpp | 18 ++- src/coreclr/jit/lsra.h | 3 + src/coreclr/jit/lsraarm64.cpp | 133 ++++++++++++++++++ .../JIT/opt/SVE/ConstantMaskReuse.csproj | 1 - 5 files changed, 169 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d3955c7861ae1d..e493b7c112416b 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1473,6 +1473,27 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); +#if defined(FEATURE_MASKED_HW_INTRINSICS) + if (intrinsicId == NI_Sve_ConversionTrueMask) + { + GenTree* trueMask = m_compiler->gtNewSimdTrueMaskNode(node->GetSimdBaseType()); + BlockRange().InsertBefore(node, trueMask); + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + use.ReplaceWith(trueMask); + } + else + { + trueMask->SetUnusedValue(); + } + + BlockRange().Remove(node); + return LowerNode(trueMask); + } +#endif // FEATURE_MASKED_HW_INTRINSICS + bool isScalar = false; genTreeOps oper = node->GetOperForHWIntrinsicId(&isScalar); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b9ff24aceea09c..16d8f9423ef7fe 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2717,6 +2717,13 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo GenTree* otherTreeNode = physRegRecord->assignedInterval->firstRefPosition->treeNode; noway_assert(otherTreeNode != nullptr); +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (areMatchingSveMaskConstants(refPosition->treeNode, otherTreeNode)) + { + return true; + } +#endif + if (refPosition->treeNode->OperGet() != otherTreeNode->OperGet()) { return false; @@ -3947,9 +3954,9 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) BasicBlock* nextBlock = getNextBlock(); bool preserveMaskConstants = false; -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) && defined(DEBUG) - preserveMaskConstants = JitConfig.JitUseScalableVectorT() && m_compiler->opts.OptimizationEnabled() && - (nextBlock != nullptr) && (nextBlock->GetUniquePred(m_compiler) == currentBlock) && +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + preserveMaskConstants = m_compiler->opts.OptimizationEnabled() && (nextBlock != nullptr) && + (nextBlock->GetUniquePred(m_compiler) == currentBlock) && !blockInfo[nextBlock->bbNum].hasEHBoundaryIn && !blockInfo[currentBlock->bbNum].hasEHBoundaryOut; #endif @@ -13777,9 +13784,8 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* matchingConstants = linearScan->getMatchingConstants(candidates, currentInterval, refPosition); } -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) && defined(DEBUG) - if (JitConfig.JitUseScalableVectorT() && linearScan->m_compiler->opts.OptimizationEnabled() && - varTypeIsMask(regType)) +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (linearScan->m_compiler->opts.OptimizationEnabled() && varTypeIsMask(regType)) { // Avoid overwriting an available mask constant when another free register exists. A later // definition can then reuse the constant; matching constants remain preferred candidates. diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 733a5fdcb058ee..38af3cc15b4fd5 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1171,6 +1171,9 @@ class LinearScan : public RegAllocInterface regNumber assignCopyRegMinimal(RefPosition* refPosition); bool isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPosition); +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + bool areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2); +#endif bool isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord); void checkAndAssignInterval(RegRecord* regRec, Interval* interval); void assignPhysReg(RegRecord* regRec, Interval* interval); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7856d4302d8aa3..b9b94c75911547 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1457,6 +1457,139 @@ int LinearScan::BuildNode(GenTree* tree) #include "hwintrinsic.h" +#if defined(FEATURE_MASKED_HW_INTRINSICS) +constexpr int PFalseConstantPattern = -1; + +struct SveMaskConstant +{ + insOpts opt; + int pattern; +}; + +//------------------------------------------------------------------------ +// NormalizeSveMaskPattern: Canonicalize equivalent SVE mask patterns. +// +// Arguments: +// pattern - The SVE mask pattern. +// +// Return Value: +// The canonical pattern. +// +static int NormalizeSveMaskPattern(int pattern) +{ + if (pattern == SveMaskPatternLargestPowerOf2) + { + return SveMaskPatternAll; + } + + return pattern; +} + +//------------------------------------------------------------------------ +// TryGetSvePTrueOpt: Get the instruction option for an SVE ptrue base type. +// +// Arguments: +// baseType - The SVE element type. +// opt - [out] The corresponding instruction option. +// +// Return Value: +// True if baseType is supported; otherwise false. +// +static bool TryGetSvePTrueOpt(var_types baseType, insOpts* opt) +{ + switch (baseType) + { + case TYP_BYTE: + case TYP_UBYTE: + *opt = INS_OPTS_SCALABLE_B; + return true; + case TYP_SHORT: + case TYP_USHORT: + *opt = INS_OPTS_SCALABLE_H; + return true; + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + *opt = INS_OPTS_SCALABLE_S; + return true; + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + *opt = INS_OPTS_SCALABLE_D; + return true; + default: + return false; + } +} + +//------------------------------------------------------------------------ +// TryGetSveMaskConstant: Get the instruction option and pattern represented by an SVE mask node. +// +// Arguments: +// node - The mask node. +// value - [out] The mask constant description. +// +// Return Value: +// True if node represents an SVE mask constant; otherwise false. +// +static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) +{ + if (!node->OperIs(GT_CNS_MSK)) + { + return false; + } + +#ifdef DEBUG + // Scalable mask constants do not use the legacy fixed-size mask representation. + if (JitConfig.JitUseScalableVectorT()) + { + return false; + } +#endif + + GenTreeMskCon* mask = node->AsMskCon(); + if (mask->IsZero()) + { + value->opt = INS_OPTS_SCALABLE_B; + value->pattern = PFalseConstantPattern; + return true; + } + + const var_types types[] = {TYP_BYTE, TYP_SHORT, TYP_INT, TYP_LONG}; + for (var_types type : types) + { + SveMaskPattern pattern = EvaluateSimdMaskToPattern(type, mask->gtSimdMaskVal); + if (pattern != SveMaskPatternNone) + { + bool found = TryGetSvePTrueOpt(type, &value->opt); + value->pattern = NormalizeSveMaskPattern(static_cast(pattern)); + assert(found); + return true; + } + } + return false; +} + +//------------------------------------------------------------------------ +// areMatchingSveMaskConstants: Check whether two nodes materialize the same SVE mask constant. +// +// Arguments: +// tree1 - The first node. +// tree2 - The second node. +// +// Return Value: +// True if both nodes represent the same SVE mask constant; otherwise false. +// +bool LinearScan::areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2) +{ + SveMaskConstant value1; + SveMaskConstant value2; + return TryGetSveMaskConstant(tree1, &value1) && TryGetSveMaskConstant(tree2, &value2) && + (value1.opt == value2.opt) && (value1.pattern == value2.pattern); +} + +#endif // FEATURE_MASKED_HW_INTRINSICS + //------------------------------------------------------------------------ // BuildHWIntrinsic: Set the NodeInfo for a GT_HWINTRINSIC tree. // diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj index 8202afed636e8f..2a23f58397625f 100644 --- a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj @@ -10,6 +10,5 @@ - From da7c439a01fd7d927aebfe525e139b6222d93d2e Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 3 Aug 2026 13:33:40 +0000 Subject: [PATCH 5/7] Refine SVE mask constant comparison --- src/coreclr/jit/hwintrinsicarm64.cpp | 1 - src/coreclr/jit/lsraarm64.cpp | 116 ++------------------------- src/coreclr/jit/simd.h | 45 +++++++++++ 3 files changed, 52 insertions(+), 110 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 91a3458999f833..3b26efbea79805 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1035,7 +1035,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Sve_CreateTrueMaskUInt64: { assert(sig->numArgs == 1); - assert(retType == TYP_MASK); op1 = impPopStack().val; // Where possible, import a constant vector to allow for optimisations. diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index b9b94c75911547..4676face40af18 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1458,83 +1458,19 @@ int LinearScan::BuildNode(GenTree* tree) #include "hwintrinsic.h" #if defined(FEATURE_MASKED_HW_INTRINSICS) -constexpr int PFalseConstantPattern = -1; - -struct SveMaskConstant -{ - insOpts opt; - int pattern; -}; - -//------------------------------------------------------------------------ -// NormalizeSveMaskPattern: Canonicalize equivalent SVE mask patterns. -// -// Arguments: -// pattern - The SVE mask pattern. -// -// Return Value: -// The canonical pattern. -// -static int NormalizeSveMaskPattern(int pattern) -{ - if (pattern == SveMaskPatternLargestPowerOf2) - { - return SveMaskPatternAll; - } - - return pattern; -} - -//------------------------------------------------------------------------ -// TryGetSvePTrueOpt: Get the instruction option for an SVE ptrue base type. -// -// Arguments: -// baseType - The SVE element type. -// opt - [out] The corresponding instruction option. -// -// Return Value: -// True if baseType is supported; otherwise false. -// -static bool TryGetSvePTrueOpt(var_types baseType, insOpts* opt) -{ - switch (baseType) - { - case TYP_BYTE: - case TYP_UBYTE: - *opt = INS_OPTS_SCALABLE_B; - return true; - case TYP_SHORT: - case TYP_USHORT: - *opt = INS_OPTS_SCALABLE_H; - return true; - case TYP_INT: - case TYP_UINT: - case TYP_FLOAT: - *opt = INS_OPTS_SCALABLE_S; - return true; - case TYP_LONG: - case TYP_ULONG: - case TYP_DOUBLE: - *opt = INS_OPTS_SCALABLE_D; - return true; - default: - return false; - } -} - //------------------------------------------------------------------------ -// TryGetSveMaskConstant: Get the instruction option and pattern represented by an SVE mask node. +// areMatchingSveMaskConstants: Check whether two nodes materialize the same SVE mask constant. // // Arguments: -// node - The mask node. -// value - [out] The mask constant description. +// tree1 - The first node. +// tree2 - The second node. // // Return Value: -// True if node represents an SVE mask constant; otherwise false. +// True if both nodes represent the same SVE mask constant; otherwise false. // -static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) +bool LinearScan::areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2) { - if (!node->OperIs(GT_CNS_MSK)) + if (!tree1->OperIs(GT_CNS_MSK) || !tree2->OperIs(GT_CNS_MSK)) { return false; } @@ -1547,45 +1483,7 @@ static bool TryGetSveMaskConstant(GenTree* node, SveMaskConstant* value) } #endif - GenTreeMskCon* mask = node->AsMskCon(); - if (mask->IsZero()) - { - value->opt = INS_OPTS_SCALABLE_B; - value->pattern = PFalseConstantPattern; - return true; - } - - const var_types types[] = {TYP_BYTE, TYP_SHORT, TYP_INT, TYP_LONG}; - for (var_types type : types) - { - SveMaskPattern pattern = EvaluateSimdMaskToPattern(type, mask->gtSimdMaskVal); - if (pattern != SveMaskPatternNone) - { - bool found = TryGetSvePTrueOpt(type, &value->opt); - value->pattern = NormalizeSveMaskPattern(static_cast(pattern)); - assert(found); - return true; - } - } - return false; -} - -//------------------------------------------------------------------------ -// areMatchingSveMaskConstants: Check whether two nodes materialize the same SVE mask constant. -// -// Arguments: -// tree1 - The first node. -// tree2 - The second node. -// -// Return Value: -// True if both nodes represent the same SVE mask constant; otherwise false. -// -bool LinearScan::areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2) -{ - SveMaskConstant value1; - SveMaskConstant value2; - return TryGetSveMaskConstant(tree1, &value1) && TryGetSveMaskConstant(tree2, &value2) && - (value1.opt == value2.opt) && (value1.pattern == value2.pattern); + return tree1->AsMskCon()->gtSimdMaskVal.EqualsSveMask(tree2->AsMskCon()->gtSimdMaskVal); } #endif // FEATURE_MASKED_HW_INTRINSICS diff --git a/src/coreclr/jit/simd.h b/src/coreclr/jit/simd.h index 463125b2a8aafe..b8d13bf2274130 100644 --- a/src/coreclr/jit/simd.h +++ b/src/coreclr/jit/simd.h @@ -333,6 +333,10 @@ struct simdmask_t return !(*this == other); } +#if defined(TARGET_ARM64) + bool EqualsSveMask(const simdmask_t& other) const; +#endif + static uint64_t GetBitMask(uint32_t elementCount) { assert((elementCount >= 1) && (elementCount <= 64)); @@ -2122,6 +2126,47 @@ SveMaskPattern EvaluateSimdMaskToPattern(var_types baseType, simdmask_t arg0) } } +//------------------------------------------------------------------------ +// simdmask_t::EqualsSveMask: Check whether two masks represent the same SVE mask constant. +// +// Arguments: +// other - The other mask. +// +// Return Value: +// True if both masks represent the same SVE mask constant; otherwise false. +// +inline bool simdmask_t::EqualsSveMask(const simdmask_t& other) const +{ + if (IsZero() || other.IsZero()) + { + return IsZero() && other.IsZero(); + } + + auto tryGetPattern = [](const simdmask_t& mask, var_types* baseType, SveMaskPattern* pattern) { + const var_types types[] = {TYP_BYTE, TYP_SHORT, TYP_INT, TYP_LONG}; + for (var_types type : types) + { + SveMaskPattern result = EvaluateSimdMaskToPattern(type, mask); + if (result != SveMaskPatternNone) + { + *baseType = type; + *pattern = result; + return true; + } + } + + return false; + }; + + var_types thisBaseType; + SveMaskPattern thisPattern; + var_types otherBaseType; + SveMaskPattern otherPattern; + + return tryGetPattern(*this, &thisBaseType, &thisPattern) && tryGetPattern(other, &otherBaseType, &otherPattern) && + (thisBaseType == otherBaseType) && (thisPattern == otherPattern); +} + // Functionality for handling constant vectors of unknown size enum SimdScalableKind : uint8_t From cd1599a334013394f3098370b4f623636fe1262c Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 3 Aug 2026 13:33:49 +0000 Subject: [PATCH 6/7] Limit SVE mask constant reuse to basic blocks --- src/coreclr/jit/lsra.cpp | 65 ++++++---------------- src/coreclr/jit/lsra.h | 9 +-- src/tests/JIT/opt/SVE/ConstantMaskReuse.cs | 41 -------------- 3 files changed, 20 insertions(+), 95 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 16d8f9423ef7fe..3f5668ef4e6c50 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -3952,15 +3952,6 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) assert(currentBlock != nullptr); markBlockVisited(currentBlock); - BasicBlock* nextBlock = getNextBlock(); - bool preserveMaskConstants = false; -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - preserveMaskConstants = m_compiler->opts.OptimizationEnabled() && (nextBlock != nullptr) && - (nextBlock->GetUniquePred(m_compiler) == currentBlock) && - !blockInfo[nextBlock->bbNum].hasEHBoundaryIn && - !blockInfo[currentBlock->bbNum].hasEHBoundaryOut; -#endif - if (localVarsEnregistered) { processBlockEndLocations(currentBlock); @@ -3969,14 +3960,15 @@ void LinearScan::processBlockEndAllocation(BasicBlock* currentBlock) // When the last block in the method has successors, there will be a final "RefTypeBB" to // ensure that we get the varToRegMap set appropriately, but in that case we don't need // to worry about "nextBlock". + BasicBlock* nextBlock = getNextBlock(); if (nextBlock != nullptr) { - processBlockStartLocations(nextBlock, preserveMaskConstants); + processBlockStartLocations(nextBlock); } } else { - resetAllRegistersState(preserveMaskConstants); + resetAllRegistersState(); } } @@ -4225,13 +4217,10 @@ void LinearScan::unassignIntervalBlockStart(RegRecord* regRecord, VarToRegMap in } //------------------------------------------------------------------------ -// resetAllRegistersState: Resets the next interval ref and spill cost, and optionally -// preserves mask constants. -// -// Arguments: -// preserveMaskConstants - whether mask constants survive from the preceding block +// resetAllRegistersState: Resets the next interval ref, spill cost and clears +// the constant registers. // -void LinearScan::resetAllRegistersState(bool preserveMaskConstants) +void LinearScan::resetAllRegistersState() { assert(!enregisterLocalVars); // Just clear any constant registers and return. @@ -4241,29 +4230,21 @@ void LinearScan::resetAllRegistersState(bool preserveMaskConstants) int regIndex = REG_FIRST; for (regNumber reg = REG_FIRST; reg < AVAILABLE_REG_COUNT; NEXT_REGISTER(reg, regIndex)) { - RegRecord* physRegRecord = getRegisterRecord(reg); - Interval* assignedInterval = physRegRecord->assignedInterval; + RegRecord* physRegRecord = getRegisterRecord(reg); #ifdef DEBUG + Interval* assignedInterval = physRegRecord->assignedInterval; assert(assignedInterval == nullptr || assignedInterval->isConstant); #endif - if (preserveMaskConstants && (assignedInterval != nullptr) && varTypeIsMask(assignedInterval->registerType)) - { - setConstantReg(reg, assignedInterval->registerType); - } - else - { - physRegRecord->assignedInterval = nullptr; - } + physRegRecord->assignedInterval = nullptr; } } //------------------------------------------------------------------------ -// processBlockStartLocations: Update var locations on entry to 'currentBlock' and update -// the constant register state. +// processBlockStartLocations: Update var locations on entry to 'currentBlock' and clear constant +// registers. // // Arguments: -// currentBlock - the BasicBlock we are about to allocate registers for -// preserveMaskConstants - whether mask constants survive from the preceding block +// currentBlock - the BasicBlock we are about to allocate registers for // // Return Value: // None @@ -4275,7 +4256,7 @@ void LinearScan::resetAllRegistersState(bool preserveMaskConstants) // modify the inVarToRegMap in cases where a lclVar was spilled after the block had been // completed. // -void LinearScan::processBlockStartLocations(BasicBlock* currentBlock, bool preserveMaskConstants) +void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) { // We should only call this method if we have register candidates. @@ -4582,9 +4563,9 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock, bool prese // Only focus on actual registers present deadCandidates &= actualRegistersMask; - handleDeadCandidates(deadCandidates.getLow(), REG_LOW_BASE, inVarToRegMap, preserveMaskConstants); + handleDeadCandidates(deadCandidates.getLow(), REG_LOW_BASE, inVarToRegMap); #ifdef HAS_MORE_THAN_64_REGISTERS - handleDeadCandidates(deadCandidates.getHigh(), REG_HIGH_BASE, inVarToRegMap, preserveMaskConstants); + handleDeadCandidates(deadCandidates.getHigh(), REG_HIGH_BASE, inVarToRegMap); #endif // HAS_MORE_THAN_64_REGISTERS #endif // TARGET_ARM } @@ -4596,15 +4577,11 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock, bool prese // deadCandidates - mask of registers. // regBase - base register number. // inVarToRegMap - variable to register map. -// preserveMaskConstants - whether mask constants survive from the preceding block. // // Return Value: // None // -void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, - int regBase, - VarToRegMap inVarToRegMap, - bool preserveMaskConstants) +void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBase, VarToRegMap inVarToRegMap) { while (deadCandidates != RBM_NONE) { @@ -4618,14 +4595,6 @@ void LinearScan::handleDeadCandidates(SingleTypeRegSet deadCandidates, { assert(assignedInterval->isLocalVar || assignedInterval->isConstant || assignedInterval->IsUpperVector()); - if (preserveMaskConstants && assignedInterval->isConstant && varTypeIsMask(assignedInterval->registerType)) - { - // Keep the mask constant associated with this available register so a - // matching definition in the successor can reuse it. - setConstantReg(reg, assignedInterval->registerType); - continue; - } - if (!assignedInterval->isConstant && assignedInterval->assignedReg == physRegRecord) { assignedInterval->isActive = false; @@ -7958,7 +7927,7 @@ void LinearScan::resolveRegisters() curBBStartLocation = currentRefPosition->nodeLocation; if (block != m_compiler->fgFirstBB) { - processBlockStartLocations(block, false); + processBlockStartLocations(block); } // Handle the DummyDefs, updating the incoming var location. diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 38af3cc15b4fd5..29d9b1d1c5efa1 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -997,14 +997,11 @@ class LinearScan : public RegAllocInterface void processBlockEndAllocation(BasicBlock* current); // Record variable locations at start/end of block - void processBlockStartLocations(BasicBlock* current, bool preserveMaskConstants); + void processBlockStartLocations(BasicBlock* current); - FORCEINLINE void handleDeadCandidates(SingleTypeRegSet deadCandidates, - int regBase, - VarToRegMap inVarToRegMap, - bool preserveMaskConstants); + FORCEINLINE void handleDeadCandidates(SingleTypeRegSet deadCandidates, int regBase, VarToRegMap inVarToRegMap); void processBlockEndLocations(BasicBlock* current); - void resetAllRegistersState(bool preserveMaskConstants); + void resetAllRegistersState(); #ifdef TARGET_ARM bool isSecondHalfReg(RegRecord* regRec, Interval* interval); diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs index 4c5fb4c805fcdf..621e5ce20dc078 100644 --- a/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs @@ -50,9 +50,6 @@ public static void TestEntryPoint() Consume(PTrueDifferentElementSizes(a)); Consume(PTrueMultipleConversionTrueMask(mask1, mask2)); Consume(PTrueSeparatedByCall(a, b)); - Consume(PTrueSeparateBlocks(a, Environment.TickCount != 0)); - Consume(PTrueUniquePredecessor(a, Environment.TickCount != 0)); - Consume(PTrueJoin(a, Environment.TickCount != 0)); if (Sve2.IsSupported) { @@ -80,44 +77,6 @@ private static Vector PTrueSeparatedByCall(Vector a, Vector b) return result1; } - [MethodImpl(MethodImplOptions.NoInlining)] - private static Vector PTrueSeparateBlocks(Vector value, bool condition) - { - //ARM64: ptrue {{p[0-9]+}}.s - //ARM64: ptrue {{p[0-9]+}}.s - //ARM64-NOT: ptrue {{p[0-9]+}}.s - if (condition) - { - return Sve.Abs(value); - } - - return Sve.Negate(value); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static Vector PTrueUniquePredecessor(Vector value, bool condition) - { - //ARM64: ptrue {{p[0-9]+}}.s - //ARM64-NOT: ptrue {{p[0-9]+}}.s - Vector result = Sve.Abs(value); - if (condition) - { - return result; - } - - return Sve.Negate(result); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static Vector PTrueJoin(Vector value, bool condition) - { - //ARM64: ptrue {{p[0-9]+}}.s - //ARM64: ptrue {{p[0-9]+}}.s - //ARM64-NOT: ptrue {{p[0-9]+}}.s - Vector result = condition ? Sve.Abs(value) : Sve.Negate(value); - return Sve.Add(result, value); - } - [MethodImpl(MethodImplOptions.NoInlining)] private static Vector PTrueSingleCompareMask(Vector a, Vector b) { From 48a44ed1c1ec244d4c7d00a368dbba7a8fd7b26f Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 4 Aug 2026 07:36:44 +0000 Subject: [PATCH 7/7] Use existing LSRA mask constant comparison --- src/coreclr/jit/lsra.cpp | 7 ------ src/coreclr/jit/lsra.h | 3 --- src/coreclr/jit/lsraarm64.cpp | 31 ------------------------ src/coreclr/jit/simd.h | 45 ----------------------------------- 4 files changed, 86 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 3f5668ef4e6c50..38db856f55f6d5 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2717,13 +2717,6 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo GenTree* otherTreeNode = physRegRecord->assignedInterval->firstRefPosition->treeNode; noway_assert(otherTreeNode != nullptr); -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - if (areMatchingSveMaskConstants(refPosition->treeNode, otherTreeNode)) - { - return true; - } -#endif - if (refPosition->treeNode->OperGet() != otherTreeNode->OperGet()) { return false; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 29d9b1d1c5efa1..3541e1a2309349 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1168,9 +1168,6 @@ class LinearScan : public RegAllocInterface regNumber assignCopyRegMinimal(RefPosition* refPosition); bool isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPosition); -#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) - bool areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2); -#endif bool isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord); void checkAndAssignInterval(RegRecord* regRec, Interval* interval); void assignPhysReg(RegRecord* regRec, Interval* interval); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 4676face40af18..7856d4302d8aa3 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1457,37 +1457,6 @@ int LinearScan::BuildNode(GenTree* tree) #include "hwintrinsic.h" -#if defined(FEATURE_MASKED_HW_INTRINSICS) -//------------------------------------------------------------------------ -// areMatchingSveMaskConstants: Check whether two nodes materialize the same SVE mask constant. -// -// Arguments: -// tree1 - The first node. -// tree2 - The second node. -// -// Return Value: -// True if both nodes represent the same SVE mask constant; otherwise false. -// -bool LinearScan::areMatchingSveMaskConstants(GenTree* tree1, GenTree* tree2) -{ - if (!tree1->OperIs(GT_CNS_MSK) || !tree2->OperIs(GT_CNS_MSK)) - { - return false; - } - -#ifdef DEBUG - // Scalable mask constants do not use the legacy fixed-size mask representation. - if (JitConfig.JitUseScalableVectorT()) - { - return false; - } -#endif - - return tree1->AsMskCon()->gtSimdMaskVal.EqualsSveMask(tree2->AsMskCon()->gtSimdMaskVal); -} - -#endif // FEATURE_MASKED_HW_INTRINSICS - //------------------------------------------------------------------------ // BuildHWIntrinsic: Set the NodeInfo for a GT_HWINTRINSIC tree. // diff --git a/src/coreclr/jit/simd.h b/src/coreclr/jit/simd.h index b8d13bf2274130..463125b2a8aafe 100644 --- a/src/coreclr/jit/simd.h +++ b/src/coreclr/jit/simd.h @@ -333,10 +333,6 @@ struct simdmask_t return !(*this == other); } -#if defined(TARGET_ARM64) - bool EqualsSveMask(const simdmask_t& other) const; -#endif - static uint64_t GetBitMask(uint32_t elementCount) { assert((elementCount >= 1) && (elementCount <= 64)); @@ -2126,47 +2122,6 @@ SveMaskPattern EvaluateSimdMaskToPattern(var_types baseType, simdmask_t arg0) } } -//------------------------------------------------------------------------ -// simdmask_t::EqualsSveMask: Check whether two masks represent the same SVE mask constant. -// -// Arguments: -// other - The other mask. -// -// Return Value: -// True if both masks represent the same SVE mask constant; otherwise false. -// -inline bool simdmask_t::EqualsSveMask(const simdmask_t& other) const -{ - if (IsZero() || other.IsZero()) - { - return IsZero() && other.IsZero(); - } - - auto tryGetPattern = [](const simdmask_t& mask, var_types* baseType, SveMaskPattern* pattern) { - const var_types types[] = {TYP_BYTE, TYP_SHORT, TYP_INT, TYP_LONG}; - for (var_types type : types) - { - SveMaskPattern result = EvaluateSimdMaskToPattern(type, mask); - if (result != SveMaskPatternNone) - { - *baseType = type; - *pattern = result; - return true; - } - } - - return false; - }; - - var_types thisBaseType; - SveMaskPattern thisPattern; - var_types otherBaseType; - SveMaskPattern otherPattern; - - return tryGetPattern(*this, &thisBaseType, &thisPattern) && tryGetPattern(other, &otherBaseType, &otherPattern) && - (thisBaseType == otherBaseType) && (thisPattern == otherPattern); -} - // Functionality for handling constant vectors of unknown size enum SimdScalableKind : uint8_t