Skip to content

SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TIHan merged 24 commits into from
May 29, 2024
Merged

SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs #102180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
36d0aea
Initial work for sve loadvectorx APIs
TIHan May 13, 2024
b50e4ff
Formatting
TIHan May 13, 2024
9672e98
Added loadvectorx* intrinsic tests
TIHan May 13, 2024
ac509ac
Change test template name
TIHan May 13, 2024
9b92c12
Added loadvectorx* APIs to ref
TIHan May 13, 2024
68b0ec7
Fixed build
TIHan May 13, 2024
511efc2
Added APIs to PlatformNotSupported
TIHan May 13, 2024
ed8a928
Fixed test template
TIHan May 13, 2024
2f0efe7
Feedback
TIHan May 14, 2024
99fe12a
Handling multi-reg
TIHan May 15, 2024
d01738b
Remove layout
TIHan May 15, 2024
6cc2fa1
This might fix it
TIHan May 15, 2024
692660e
LoadVectorx2/3/4 is working
TIHan May 16, 2024
8fd8319
Revert dstCount changes
TIHan May 16, 2024
0f969a9
Formatting
TIHan May 16, 2024
3f80fc6
Feedback and re-naming APIs
TIHan May 17, 2024
bb85fd4
Fix build
TIHan May 17, 2024
b7aed29
Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2
TIHan May 17, 2024
da15488
Trying to fix failing test
TIHan May 20, 2024
aa82bab
Tweaks
TIHan May 21, 2024
bdb3b13
quick fix
TIHan May 21, 2024
e4ecf43
Merging
TIHan May 23, 2024
ce587af
Added FalseMask and NonFaulting test cases
TIHan May 25, 2024
d36b653
Feedback
TIHan May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
case NI_Sve_Load2xVectorAndUnzip:
case NI_Sve_Load3xVectorAndUnzip:
case NI_Sve_Load4xVectorAndUnzip:
addr = Op(2);
break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
return compiler->typGetBlkLayout(64);

case NI_Sve_Load2xVectorAndUnzip:
return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.
case NI_Sve_Load3xVectorAndUnzip:
return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
case NI_Sve_Load4xVectorAndUnzip:
return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

default:
Expand Down
8 changes: 8 additions & 0 deletions src/coreclr/jit/hwintrinsic.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
// Op1 input is a vector. HWInstrinsic requires a mask.
retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
}

if (HWIntrinsicInfo::IsMultiReg(intrinsic))
{
assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
retNode =
impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
}
}

if (retType != nodeRetType)
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/jit/hwintrinsic.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
case NI_AdvSimd_LoadAndReplicateToVector64x2:
case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
case NI_Sve_Load2xVectorAndUnzip:
return 2;

case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
case NI_AdvSimd_LoadAndReplicateToVector64x3:
case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
case NI_Sve_Load3xVectorAndUnzip:
return 3;

case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
case NI_AdvSimd_LoadAndReplicateToVector64x4:
case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
case NI_Sve_Load4xVectorAndUnzip:
return 4;
#endif

Expand Down
29 changes: 29 additions & 0 deletions src/coreclr/jit/hwintrinsicarm64.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
break;
}

case NI_Sve_Load2xVectorAndUnzip:
case NI_Sve_Load3xVectorAndUnzip:
case NI_Sve_Load4xVectorAndUnzip:
{
info.compNeedsConsecutiveRegisters = true;

assert(sig->numArgs == 2);

op2 = impPopStack().val;
op1 = impPopStack().val;

if (op2->OperIs(GT_CAST))
{
// Although the API specifies a pointer, if what we have is a BYREF, that's what
// we really want, so throw away the cast.
if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
{
op2 = op2->gtGetOp1();
}
}

assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
break;
}

case NI_AdvSimd_LoadAndInsertScalarVector64x2:
case NI_AdvSimd_LoadAndInsertScalarVector64x3:
case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down
20 changes: 20 additions & 0 deletions src/coreclr/jit/hwintrinsiccodegenarm64.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
}
break;

case NI_Sve_Load2xVectorAndUnzip:
case NI_Sve_Load3xVectorAndUnzip:
case NI_Sve_Load4xVectorAndUnzip:
{
#ifdef DEBUG
// Validates that consecutive registers were used properly.

assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

regNumber argReg = targetReg;
for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
{
assert(argReg == node->GetRegNumByIdx(i));
argReg = getNextSIMDRegWithWraparound(argReg);
}
#endif // DEBUG
GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
break;
}

case NI_Sve_StoreAndZipx2:
case NI_Sve_StoreAndZipx3:
case NI_Sve_StoreAndZipx4:
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/jit/hwintrinsiclistarm64sve.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down
13 changes: 13 additions & 0 deletions src/coreclr/jit/lsraarm64.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
break;
}

case NI_Sve_Load2xVectorAndUnzip:
case NI_Sve_Load3xVectorAndUnzip:
case NI_Sve_Load4xVectorAndUnzip:
{
assert(intrin.op1 != nullptr);
assert(intrin.op2 != nullptr);
assert(intrinsicTree->OperIsMemoryLoadOrStore());
srcCount += BuildAddrUses(intrin.op2);
BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
*pDstCount = dstCount;
break;
}

case NI_Sve_StoreAndZipx2:
case NI_Sve_StoreAndZipx3:
case NI_Sve_StoreAndZipx4:
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
SVE: Added `Load2xVectorAndUnzip`, `Load3xVectorAndUnzip`, `Load4xVectorAndUnzip` APIs by TIHan · Pull Request #102180 · dotnet/runtime · GitHub
Skip to content
SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs#102180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

TIHan
merged 24 commits into

from

May 29, 2024
 Merged


SVE: Added Load2xVectorAndUnzip, Load3xVectorAndUnzip, Load4xVectorAndUnzip APIs
#102180
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


24 commits


Select commit
Hold shift + click to select a range
36d0aea

Initial work for sve loadvectorx APIs


TIHan May 13, 2024
b50e4ff

Formatting


TIHan May 13, 2024
9672e98

Added loadvectorx* intrinsic tests


TIHan May 13, 2024
ac509ac

Change test template name


TIHan May 13, 2024
9b92c12

Added loadvectorx* APIs to ref


TIHan May 13, 2024
68b0ec7

Fixed build


TIHan May 13, 2024
511efc2

Added APIs to PlatformNotSupported


TIHan May 13, 2024
ed8a928

Fixed test template


TIHan May 13, 2024
2f0efe7

Feedback


TIHan May 14, 2024
99fe12a

Handling multi-reg


TIHan May 15, 2024
d01738b

Remove layout


TIHan May 15, 2024
6cc2fa1

This might fix it


TIHan May 15, 2024
692660e

LoadVectorx2/3/4 is working


TIHan May 16, 2024
8fd8319

Revert dstCount changes


TIHan May 16, 2024
0f969a9

Formatting


TIHan May 16, 2024
3f80fc6

Feedback and re-naming APIs


TIHan May 17, 2024
bb85fd4

Fix build


TIHan May 17, 2024
b7aed29

Merge remote-tracking branch 'upstream/main' into arm64-sve-loadvectorx2


TIHan May 17, 2024
da15488

Trying to fix failing test


TIHan May 20, 2024
aa82bab

Tweaks


TIHan May 21, 2024
bdb3b13

quick fix


TIHan May 21, 2024
e4ecf43

Merging


TIHan May 23, 2024
ce587af

Added FalseMask and NonFaulting test cases


TIHan May 25, 2024
d36b653

Feedback


TIHan May 28, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 10 additions & 0 deletions
src/coreclr/jit/gentree.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26594,6 +26594,9 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
 case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 addr = Op(2);
 break;
#endif // TARGET_ARM64
Expand DownExpand Up@@ -27115,6 +27118,13 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 return compiler->typGetBlkLayout(64);

 case NI_Sve_Load2xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 2);
Comment thread
TIHan marked this conversation as resolved.

 case NI_Sve_Load3xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 3);
 case NI_Sve_Load4xVectorAndUnzip:
 return compiler->typGetBlkLayout(compiler->getVectorTByteLength() * 4);

#endif // TARGET_ARM64

 default:
Expand Down

8 changes: 8 additions & 0 deletions
src/coreclr/jit/hwintrinsic.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1603,6 +1603,14 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 // Op1 input is a vector. HWInstrinsic requires a mask.
 retNode->AsHWIntrinsic()->Op(1) = gtNewSimdConvertVectorToMaskNode(retType, op1, simdBaseJitType, simdSize);
 }

 if (HWIntrinsicInfo::IsMultiReg(intrinsic))
 {
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 assert(HWIntrinsicInfo::IsMultiReg(retNode->AsHWIntrinsic()->GetHWIntrinsicId()));
 retNode =
 impStoreMultiRegValueToVar(retNode, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 }
 }

 if (retType != nodeRetType)
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsic.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -831,6 +831,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
 case NI_AdvSimd_LoadAndReplicateToVector64x2:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
 case NI_Sve_Load2xVectorAndUnzip:
 return 2;

 case NI_AdvSimd_LoadVector64x3AndUnzip:
Expand All@@ -841,6 +842,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
 case NI_AdvSimd_LoadAndReplicateToVector64x3:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
 case NI_Sve_Load3xVectorAndUnzip:
 return 3;

 case NI_AdvSimd_LoadVector64x4AndUnzip:
Expand All@@ -851,6 +853,7 @@ struct HWIntrinsicInfo
 case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
 case NI_AdvSimd_LoadAndReplicateToVector64x4:
 case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
 case NI_Sve_Load4xVectorAndUnzip:
 return 4;
#endif

Expand Down

29 changes: 29 additions & 0 deletions
src/coreclr/jit/hwintrinsicarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2296,6 +2296,35 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
 retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 info.compNeedsConsecutiveRegisters = true;

 assert(sig->numArgs == 2);

 op2 = impPopStack().val;
 op1 = impPopStack().val;

 if (op2->OperIs(GT_CAST))
 {
 // Although the API specifies a pointer, if what we have is a BYREF, that's what
 // we really want, so throw away the cast.
 if (op2->gtGetOp1()->TypeGet() == TYP_BYREF)
 {
 op2 = op2->gtGetOp1();
 }
 }

 assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
 assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic));

 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize);
 break;
 }

 case NI_AdvSimd_LoadAndInsertScalarVector64x2:
 case NI_AdvSimd_LoadAndInsertScalarVector64x3:
 case NI_AdvSimd_LoadAndInsertScalarVector64x4:
Expand Down

20 changes: 20 additions & 0 deletions
src/coreclr/jit/hwintrinsiccodegenarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1356,6 +1356,26 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
 }
 break;

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
#ifdef DEBUG
 // Validates that consecutive registers were used properly.

 assert(node->GetMultiRegCount(compiler) == (unsigned int)GetEmitter()->insGetSveReg1ListSize(ins));

 regNumber argReg = targetReg;
 for (unsigned int i = 0; i < node->GetMultiRegCount(compiler); i++)
 {
 assert(argReg == node->GetRegNumByIdx(i));
 argReg = getNextSIMDRegWithWraparound(argReg);
 }
#endif // DEBUG
 GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt);
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down

3 changes: 3 additions & 0 deletions
src/coreclr/jit/hwintrinsiclistarm64sve.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -93,6 +93,9 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32,
HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, true, {INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, true, {INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, true, {INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, false, {INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
Expand Down

13 changes: 13 additions & 0 deletions
src/coreclr/jit/lsraarm64.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1780,6 +1780,19 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
 break;
 }

 case NI_Sve_Load2xVectorAndUnzip:
 case NI_Sve_Load3xVectorAndUnzip:
 case NI_Sve_Load4xVectorAndUnzip:
 {
 assert(intrin.op1 != nullptr);
 assert(intrin.op2 != nullptr);
 assert(intrinsicTree->OperIsMemoryLoadOrStore());
 srcCount += BuildAddrUses(intrin.op2);
 BuildConsecutiveRegistersForDef(intrinsicTree, dstCount);
 *pDstCount = dstCount;
 break;
 }

 case NI_Sve_StoreAndZipx2:
 case NI_Sve_StoreAndZipx3:
 case NI_Sve_StoreAndZipx4:
Expand Down
Loading