diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index c612170643fecd..32654f7776f34a 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8681,7 +8681,6 @@ void CodeGen::genCodeForReuseVal(GenTree* treeNode) { assert(treeNode->IsReuseRegVal()); - // For now, this is only used for constant nodes. #if defined(FEATURE_MASKED_HW_INTRINSICS) assert(treeNode->OperIs(GT_CNS_INT, GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK)); #elif defined(FEATURE_SIMD) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index ab02812f445863..43b456bba294cc 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -2546,7 +2546,6 @@ struct GenTree bool IsReuseRegVal() const { - // This can be extended to non-constant nodes, but not to local or indir nodes. return OperIsConst() && ((gtFlags & GTF_REUSE_REG_VAL) != 0); } diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 91a3458999f833..3b26efbea79805 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1035,7 +1035,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Sve_CreateTrueMaskUInt64: { assert(sig->numArgs == 1); - assert(retType == TYP_MASK); op1 = impPopStack().val; // Where possible, import a constant vector to allow for optimisations. diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d3955c7861ae1d..e493b7c112416b 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1473,6 +1473,27 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); +#if defined(FEATURE_MASKED_HW_INTRINSICS) + if (intrinsicId == NI_Sve_ConversionTrueMask) + { + GenTree* trueMask = m_compiler->gtNewSimdTrueMaskNode(node->GetSimdBaseType()); + BlockRange().InsertBefore(node, trueMask); + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + use.ReplaceWith(trueMask); + } + else + { + trueMask->SetUnusedValue(); + } + + BlockRange().Remove(node); + return LowerNode(trueMask); + } +#endif // FEATURE_MASKED_HW_INTRINSICS + bool isScalar = false; genTreeOps oper = node->GetOperForHWIntrinsicId(&isScalar); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b9fcf5c6e459dd..38db856f55f6d5 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -13738,6 +13738,31 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* freeCandidates = linearScan->getFreeCandidates(candidates, regType); } + if (freeCandidates != RBM_NONE) + { + // Set the 'matchingConstants' set. + if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) + { + matchingConstants = linearScan->getMatchingConstants(candidates, currentInterval, refPosition); + } + +#if defined(TARGET_ARM64) && defined(FEATURE_MASKED_HW_INTRINSICS) + if (linearScan->m_compiler->opts.OptimizationEnabled() && varTypeIsMask(regType)) + { + // Avoid overwriting an available mask constant when another free register exists. A later + // definition can then reuse the constant; matching constants remain preferred candidates. + SingleTypeRegSet constantsToPreserve = + linearScan->m_RegistersWithConstants.GetRegSetForType(regType) & ~matchingConstants & ~fixedRegMask; + SingleTypeRegSet remainingFreeCandidates = freeCandidates & ~constantsToPreserve; + if (remainingFreeCandidates != RBM_NONE) + { + candidates &= ~constantsToPreserve; + freeCandidates = remainingFreeCandidates; + } + } +#endif + } + // If no free candidates, then double check if refPosition is an actual ref. if (freeCandidates == RBM_NONE) { @@ -13748,14 +13773,6 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* return RBM_NONE; } } - else - { - // Set the 'matchingConstants' set. - if (currentInterval->isConstant && RefTypeIsDef(refPosition->refType)) - { - matchingConstants = linearScan->getMatchingConstants(candidates, currentInterval, refPosition); - } - } #define IF_FOUND_GOTO_DONE \ if (found) \ diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs new file mode 100644 index 00000000000000..621e5ce20dc078 --- /dev/null +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.cs @@ -0,0 +1,345 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +public class ConstantMaskReuse +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Consume(T value) + { + } + + [Fact] + public static void TestEntryPoint() + { + if (!Sve.IsSupported) + { + return; + } + + Vector a = Vector.Create(11); + Vector b = Vector.Create(22); + Vector c = Vector.Create(33); + int[] values = new int[32]; + Vector mask1 = Vector128.CreateScalar(0x1UL).AsVector(); + Vector mask2 = Vector128.CreateScalar(0x2UL).AsVector(); + + Consume(PTrueSingleCompareMask(a, b)); + Consume(PTrueSingleCreateTrueMask(a, b)); + Consume(PTrueSingleCreateTrueMaskPattern(a, b)); + Consume(PFalseSingleCreateFalseMask(a, b)); + Consume(PTrueSingleAllBitsMask(a, b)); + Consume(PTrueSingleEmbeddedMask(a, b)); + Consume(PTrueSingleConversionTrueMask(mask1)); + + Consume(PTrueMultipleCompareMask(a, b)); + Consume(PTrueMultipleCreateTrueMask(a, b, c)); + Consume(PTrueMultipleCreateTrueMaskPattern(a, b, c)); + Consume(PFalseMultipleCreateFalseMask(a, b, c)); + Consume(PFalseMultipleLoadMasks(values)); + Consume(PTrueMultipleAllBitsMask(a, b, c)); + Consume(PTrueMultipleEmbeddedMask(a, b, c)); + Consume(PTrueMultipleMixedSources(a, b, c)); + Consume(PTrueDifferentPatterns(a, b, c)); + Consume(PTrueDifferentElementSizes(a)); + Consume(PTrueMultipleConversionTrueMask(mask1, mask2)); + Consume(PTrueSeparatedByCall(a, b)); + + if (Sve2.IsSupported) + { + Vector d = Vector.Create(42.0); + Vector e = Vector.Create((byte)42); + + Consume(PTrueMultipleSve2Log2Negate(d)); + Consume(PTrueMultipleSve2ZeroExtend8(e)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector Identity(Vector value) => value; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSeparatedByCall(Vector a, Vector b) + { + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64: ptrue {{p[0-9]+}}.s + //ARM64-NOT: ptrue {{p[0-9]+}}.s + Vector result1 = Sve.Abs(a); + Consume(result1); + Vector result2 = Sve.Negate(Identity(b)); + Consume(result2); + return result1; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleCompareMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 + return Sve.CompareGreaterThan(a, b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleCreateTrueMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 + return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleCreateTrueMaskPattern(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 + return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PFalseSingleCreateFalseMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: pfalse {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/z, #-1 + return Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateFalseMaskInt32()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleAllBitsMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + return Sve.ConditionalSelect(Vector.AllBitsSet, Sve.AbsoluteDifference(a, b), a); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueSingleEmbeddedMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + return Sve.AbsoluteDifference(a, b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong PTrueSingleConversionTrueMask(Vector mask) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: cmpne {{p[0-9]+}}.d, {{p[0-9]+}}/z, {{z[0-9]+}}.d, #0 + //ARM64-FULL-LINE-NEXT: mov {{x[0-9]+}}, xzr + //ARM64-FULL-LINE-NEXT: sqdecp {{x[0-9]+}}, {{p[0-9]+}}.d + return Sve.SaturatingDecrementByActiveElementCount(0UL, mask); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleCompareMask(Vector a, Vector b) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: mov {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CompareGreaterThan(a, b); + Vector mask2 = Sve.CompareLessThan(a, b); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, a); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleCreateTrueMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateTrueMaskInt32()); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleCreateTrueMaskPattern(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PFalseMultipleCreateFalseMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: pfalse {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateFalseMaskInt32()); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateFalseMaskInt32()); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector PFalseMultipleLoadMasks(int[] values) + { + //ARM64-FULL-LINE: pfalse {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: wrffr {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: ldnf1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [{{x[0-9]+}}] + //ARM64-FULL-LINE-NEXT: add {{x[0-9]+}}, {{x[0-9]+}}, #4 + //ARM64-FULL-LINE-NEXT: ldnf1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [{{x[0-9]+}}] + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + fixed (int* ptr = values) + { + Vector result1 = Sve.LoadVectorNonFaulting(Sve.CreateFalseMaskInt32(), ptr); + Vector result2 = Sve.LoadVectorNonFaulting(Vector.Zero, ptr + 1); + return Sve.Add(result1, result2); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleAllBitsMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector result1 = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.AbsoluteDifference(a, b), a); + Vector result2 = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.Abs(c), c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleEmbeddedMask(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector result1 = Sve.AbsoluteDifference(a, b); + Vector result2 = Sve.Abs(c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleMixedSources(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32()); + Vector result1 = Sve.ConditionalSelect(mask, Sve.AbsoluteDifference(a, b), a); + Vector result2 = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.Abs(c), c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueDifferentPatterns(Vector a, Vector b, Vector c) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.b, vl1 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: cmpgt {{p[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.s, vl2 + //ARM64-FULL-LINE-NEXT: brka {{p[0-9]+}}.b, {{p[0-9]+}}/z, {{p[0-9]+}}.b + //ARM64-FULL-LINE-NEXT: sel {{z[0-9]+}}.s, {{p[0-9]+}}, {{z[0-9]+}}.s, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector mask1 = Sve.CreateBreakAfterMask(Sve.CompareGreaterThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount1)); + Vector mask2 = Sve.CreateBreakAfterMask(Sve.CompareLessThan(a, b), Sve.CreateTrueMaskInt32(SveMaskPattern.VectorCount2)); + Vector result1 = Sve.ConditionalSelect(mask1, a, b); + Vector result2 = Sve.ConditionalSelect(mask2, b, c); + return Sve.Add(result1, result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueDifferentElementSizes(Vector value) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: abs {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s + //ARM64-FULL-LINE-NEXT: ptrue {{p[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: neg {{z[0-9]+}}.h, {{p[0-9]+}}/m, {{z[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: add {{z[0-9]+}}.s, {{z[0-9]+}}.s, {{z[0-9]+}}.s + Vector result1 = Sve.Abs(value); + Vector result2 = Sve.Negate((Vector)value); + return Sve.Add(result1, (Vector)result2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong PTrueMultipleConversionTrueMask(Vector mask1, Vector mask2) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: cmpne {{p[0-9]+}}.d, {{p[0-9]+}}/z, {{z[0-9]+}}.d, #0 + //ARM64-FULL-LINE-NEXT: mov {{x[0-9]+}}, xzr + //ARM64-FULL-LINE-NEXT: sqdecp {{x[0-9]+}}, {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: cmpne {{p[0-9]+}}.d, {{p[0-9]+}}/z, {{z[0-9]+}}.d, #0 + //ARM64-FULL-LINE-NEXT: mov {{x[0-9]+}}, xzr + //ARM64-FULL-LINE-NEXT: sqdecp {{x[0-9]+}}, {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: add {{x[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}} + ulong result1 = Sve.SaturatingDecrementByActiveElementCount(0UL, mask1); + ulong result2 = Sve.SaturatingDecrementByActiveElementCount(0UL, mask2); + return result1 + result2; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleSve2Log2Negate(Vector value) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: flogb {{z[0-9]+}}.d, {{p[0-9]+}}/m, {{z[0-9]+}}.d + //ARM64-FULL-LINE-NEXT: neg {{z[0-9]+}}.d, {{p[0-9]+}}/m, {{z[0-9]+}}.d + Vector exponent = Sve2.Log2(value); + Vector scale = Sve.Negate(exponent); + return Sve.Scale(value, scale); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector PTrueMultipleSve2ZeroExtend8(Vector value) + { + //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: uxtb {{z[0-9]+}}.h, {{p[0-9]+}}/m, {{z[0-9]+}}.h + //ARM64-FULL-LINE-NEXT: uxtb {{z[0-9]+}}.h, {{p[0-9]+}}/m, {{z[0-9]+}}.h + Vector result1 = Sve.ZeroExtend8((Vector)value); + Vector result2 = Sve.ZeroExtend8((Vector)Sve.Add(value, value)); + return Sve.Add(result1, result2); + } +} diff --git a/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj new file mode 100644 index 00000000000000..2a23f58397625f --- /dev/null +++ b/src/tests/JIT/opt/SVE/ConstantMaskReuse.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + +