Skip to content

Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakobbotsch merged 9 commits into from
Nov 25, 2022
Merged

Move the [With|Get]Element(SIMD) folding to local morph #76491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6d1cab0
Migrate [Get|With]Element to local morph
SingleAccretion Nov 6, 2022
19072fd
Delete code from morph
SingleAccretion Nov 6, 2022
ff2849f
Move fgMorphCombineSIMDFieldAssignments to local morph
SingleAccretion Nov 6, 2022
0a9f164
Merge branch 'main' into No-HWI-Get-IN-MinOpts
SingleAccretion Nov 11, 2022
95ca016
Merge branch 'main' into No-HWI-Get-IN-MinOpts
SingleAccretion Nov 12, 2022
bb7eaad
Save uses in Values instead of nodes
SingleAccretion Nov 24, 2022
ed7dbfd
Remove ReplaceWith
SingleAccretion Nov 24, 2022
8f77462
Add ifdefs for clarity and to avoid regressing ARM
SingleAccretion Nov 24, 2022
69936a5
Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…
SingleAccretion Nov 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
unsigned* indexOut,
unsigned* simdSizeOut,
bool ignoreUsedInSIMDIntrinsic = false);
GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down
267 changes: 221 additions & 46 deletions src/coreclr/jit/lclmorph.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
class Value
{
GenTree* m_node;
unsigned m_lclNum;
unsigned m_offset;
bool m_address;
INDEBUG(bool m_consumed;)
GenTree** m_use;
unsigned m_lclNum;
unsigned m_offset;
bool m_address;
INDEBUG(bool m_consumed);

public:
// Produce an unknown value associated with the specified node.
Value(GenTree* node)
: m_node(node)
Value(GenTree** use)
: m_use(use)
, m_lclNum(BAD_VAR_NUM)
, m_offset(0)
, m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
{
}

// Get the use for the node that produced this value.
GenTree** Use() const
{
return m_use;
}

// Get the node that produced this value.
GenTree* Node() const
{
return m_node;
return *m_use;
}

// Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
None,
Nop,
BitCast,
#ifdef FEATURE_HW_INTRINSICS
GetElement,
WithElement,
#endif // FEATURE_HW_INTRINSICS
LclVar,
LclFld
};
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
}
}

PushValue(node);
PushValue(use);

return Compiler::WALK_CONTINUE;
}
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
}

private:
void PushValue(GenTree* node)
void PushValue(GenTree** use)
{
m_valueStack.Push(node);
m_valueStack.Push(use);
}

Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

case IndirTransform::BitCast:
indir->ChangeOper(GT_BITCAST);
indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
lclNode = indir->gtGetOp1()->AsLclVarCommon();
lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
break;

#ifdef FEATURE_HW_INTRINSICS
case IndirTransform::GetElement:
{
var_types elementType = indir->TypeGet();
assert(elementType == TYP_FLOAT);

lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
/* isSimdAsHWIntrinsic */ false);
indir = hwiNode;
*val.Use() = hwiNode;
}
break;

case IndirTransform::WithElement:
{
assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
var_types elementType = indir->TypeGet();
assert(elementType == TYP_FLOAT);

lclNode = BashToLclVar(indir, lclNum);
GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
GenTree* elementNode = user->gtGetOp2();
user->AsOp()->gtOp2 =
m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
/* isSimdAsHWIntrinsic */ false);
user->ChangeType(varDsc->TypeGet());
}
break;
#endif // FEATURE_HW_INTRINSICS

case IndirTransform::LclVar:
// TODO-ADDR: use "BashToLclVar" here.
if (indir->TypeGet() != varDsc->TypeGet())
{
assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
return IndirTransform::LclVar;
}

if (varTypeIsSIMD(varDsc))
{
// TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
// fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
// here.
return IndirTransform::None;
}

// Bool and ubyte are the same type.
if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
(indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
{
return IndirTransform::LclVar;
}

bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

// For small locals on the LHS we can ignore the signed/unsigned diff.
if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
(varTypeToSigned(indir) == varTypeToSigned(varDsc)))
if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
{
assert(varTypeIsSmall(indir));
return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
return IndirTransform::LclFld;
}

#ifdef FEATURE_HW_INTRINSICS
if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
m_compiler->IsBaselineSimdIsaSupported())
{
return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
}
#endif // FEATURE_HW_INTRINSICS

// Turn this into a bitcast if we can.
if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
{
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
// the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
// nodes for the outer struct "A".
//
if (indir->TypeIs(TYP_STRUCT))
// TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
{
// TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
if (indir->OperIs(GT_IND))
{
// We do not have a layout for this node.
return;
}
return;
}

ClassLayout* layout = indir->GetLayout(m_compiler);
ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
if (indSize > genTypeSize(fieldType))
{
// Retargeting this indirection to reference the promoted field would make it
// "wide", address-exposing the whole parent struct (with all of its fields).
return;
}

if (indir->TypeIs(TYP_STRUCT))
{
indir->SetOper(GT_OBJ);
indir->AsBlk()->SetLayout(layout);
indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
{
return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
}

//------------------------------------------------------------------------
// BashToLclVar: Bash node to a LCL_VAR.
//
// Arguments:
// node - the node to bash
// lclNum - the local's number
//
// Return Value:
// The bashed node.
//
GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
{
LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

node->ChangeOper(GT_LCL_VAR);
node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
node->AsLclVar()->SetLclNum(lclNum);

return node->AsLclVar();
}
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
bool madeChanges = false;
LocalAddressVisitor visitor(this);

for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

for (Statement* const stmt : block->Statements())
{
#ifdef FEATURE_SIMD
if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
stmt->GetRootNode()->OperIs(GT_ASG))
{
madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
}
#endif

visitor.VisitStmt(stmt);
}
}

return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
madeChanges |= visitor.MadeChanges();

return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
LocalAddressVisitor visitor(this);
visitor.VisitStmt(stmt);
GenTree* tree = stmt->GetRootNode();
assert(tree->OperGet() == GT_ASG);

GenTree* originalLHS = tree->AsOp()->gtOp1;
GenTree* prevLHS = tree->AsOp()->gtOp1;
GenTree* prevRHS = tree->AsOp()->gtOp2;
unsigned index = 0;
CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
unsigned simdSize = 0;
GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
{
// if the RHS is not from a SIMD vector field X, then there is no need to check further.
return false;
}

var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
var_types simdType = getSIMDTypeForSize(simdSize);
int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
int remainingAssignments = assignmentsCount;
Statement* curStmt = stmt->GetNextStmt();
Statement* lastStmt = stmt;

while (curStmt != nullptr && remainingAssignments > 0)
{
GenTree* exp = curStmt->GetRootNode();
if (exp->OperGet() != GT_ASG)
{
break;
}
GenTree* curLHS = exp->gtGetOp1();
GenTree* curRHS = exp->gtGetOp2();

if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
{
break;
}

remainingAssignments--;
prevLHS = curLHS;
prevRHS = curRHS;

lastStmt = curStmt;
curStmt = curStmt->GetNextStmt();
}

if (remainingAssignments > 0)
{
// if the left assignments number is bigger than zero, then this means
// that the assignments are not assigning to the contiguously memory
// locations from same vector.
return false;
}

JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

for (int i = 0; i < assignmentsCount; i++)
{
fgRemoveStmt(block, stmt->GetNextStmt());
}

GenTree* dstNode;

if (originalLHS->OperIs(GT_LCL_FLD))
{
dstNode = originalLHS;
dstNode->gtType = simdType;
}
else
{
GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
}

JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
DISPSTMT(stmt);

assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
simdStructNode->ClearDoNotCSE();

tree = gtNewAssignNode(dstNode, simdStructNode);

stmt->SetRootNode(tree);

JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
DISPSTMT(stmt);

return true;
}
#endif // FEATURE_SIMD
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
Move the `[With|Get]Element(SIMD)` folding to local morph by SingleAccretion · Pull Request #76491 · dotnet/runtime · GitHub
Skip to content
Move the [With|Get]Element(SIMD) folding to local morph#76491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jakobbotsch
merged 9 commits into

from

Nov 25, 2022
 Merged


Move the [With|Get]Element(SIMD) folding to local morph
#76491
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
6d1cab0

Migrate [Get|With]Element to local morph


SingleAccretion Nov 6, 2022
19072fd

Delete code from morph


SingleAccretion Nov 6, 2022
ff2849f

Move fgMorphCombineSIMDFieldAssignments to local morph


SingleAccretion Nov 6, 2022
0a9f164

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 11, 2022
95ca016

Merge branch 'main' into No-HWI-Get-IN-MinOpts


SingleAccretion Nov 12, 2022
bb7eaad

Save uses in Values instead of nodes


SingleAccretion Nov 24, 2022
ed7dbfd

Remove ReplaceWith


SingleAccretion Nov 24, 2022
8f77462

Add ifdefs for clarity and to avoid regressing ARM


SingleAccretion Nov 24, 2022
69936a5

Merge branch 'No-HWI-Get-IN-MinOpts' of https://github.com/SingleAccr…


SingleAccretion Nov 24, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 0 additions & 2 deletions
src/coreclr/jit/compiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5701,8 +5701,6 @@ class Compiler
 unsigned* indexOut,
 unsigned* simdSizeOut,
 bool ignoreUsedInSIMDIntrinsic = false);
 GenTree* fgMorphFieldAssignToSimdSetElement(GenTree* tree);
 GenTree* fgMorphFieldToSimdGetElement(GenTree* tree);
 bool fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt);
 void impMarkContiguousSIMDFieldAssignments(Statement* stmt);

Expand Down

267 changes: 221 additions & 46 deletions
src/coreclr/jit/lclmorph.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 //
 class Value
 {
 GenTree* m_node;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed;)
 GenTree** m_use;
 unsigned m_lclNum;
 unsigned m_offset;
 bool m_address;
 INDEBUG(bool m_consumed);

 public:
 // Produce an unknown value associated with the specified node.
 Value(GenTree* node)
 : m_node(node)
 Value(GenTree** use)
 : m_use(use)
 , m_lclNum(BAD_VAR_NUM)
 , m_offset(0)
 , m_address(false)
Expand All@@ -53,10 +53,16 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 }

 // Get the use for the node that produced this value.
 GenTree** Use() const
 {
 return m_use;
 }

 // Get the node that produced this value.
 GenTree* Node() const
 {
 return m_node;
 return *m_use;
 }

 // Does this value represent a location?
Expand DownExpand Up@@ -294,6 +300,10 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 None,
 Nop,
 BitCast,
#ifdef FEATURE_HW_INTRINSICS
 GetElement,
 WithElement,
#endif // FEATURE_HW_INTRINSICS
 LclVar,
 LclFld
 };
Expand DownExpand Up@@ -418,7 +428,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }
 }

 PushValue(node);
 PushValue(use);

 return Compiler::WALK_CONTINUE;
 }
Expand DownExpand Up@@ -557,9 +567,9 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 }

private:
 void PushValue(GenTree* node)
 void PushValue(GenTree** use)
 {
 m_valueStack.Push(node);
 m_valueStack.Push(use);
 }

 Value& TopValue(unsigned index)
Expand DownExpand Up@@ -909,13 +919,46 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>

 case IndirTransform::BitCast:
 indir->ChangeOper(GT_BITCAST);
 indir->gtGetOp1()->ChangeOper(GT_LCL_VAR);
 indir->gtGetOp1()->ChangeType(varDsc->TypeGet());
 indir->gtGetOp1()->AsLclVar()->SetLclNum(lclNum);
 lclNode = indir->gtGetOp1()->AsLclVarCommon();
 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 break;

#ifdef FEATURE_HW_INTRINSICS
 case IndirTransform::GetElement:
 {
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir->gtGetOp1(), lclNum);
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 indir = hwiNode;
 *val.Use() = hwiNode;
 }
 break;

 case IndirTransform::WithElement:
 {
 assert(user->OperIs(GT_ASG) && (user->gtGetOp1() == indir));
 var_types elementType = indir->TypeGet();
 assert(elementType == TYP_FLOAT);

 lclNode = BashToLclVar(indir, lclNum);
 GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet());
 GenTree* indexNode = m_compiler->gtNewIconNode(val.Offset() / genTypeSize(elementType));
 GenTree* elementNode = user->gtGetOp2();
 user->AsOp()->gtOp2 =
 m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode,
 CORINFO_TYPE_FLOAT, genTypeSize(varDsc),
 /* isSimdAsHWIntrinsic */ false);
 user->ChangeType(varDsc->TypeGet());
 }
 break;
#endif // FEATURE_HW_INTRINSICS

 case IndirTransform::LclVar:
 // TODO-ADDR: use "BashToLclVar" here.
 if (indir->TypeGet() != varDsc->TypeGet())
 {
 assert(genTypeSize(indir) == genTypeSize(varDsc)); // BOOL <-> UBYTE.
Expand DownExpand Up@@ -996,24 +1039,17 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclVar;
 }

 if (varTypeIsSIMD(varDsc))
 {
 // TODO-ADDR: skip SIMD variables for now, fgMorphFieldAssignToSimdSetElement and
 // fgMorphFieldToSimdGetElement need to be updated to recognize LCL_FLDs or moved
 // here.
 return IndirTransform::None;
 }

 // Bool and ubyte are the same type.
 if ((indir->TypeIs(TYP_BOOL) && (varDsc->TypeGet() == TYP_UBYTE)) ||
 (indir->TypeIs(TYP_UBYTE) && (varDsc->TypeGet() == TYP_BOOL)))
 {
 return IndirTransform::LclVar;
 }

 bool isDef = user->OperIs(GT_ASG) && (user->gtGetOp1() == indir);

 // For small locals on the LHS we can ignore the signed/unsigned diff.
 if (user->OperIs(GT_ASG) && (user->gtGetOp1() == indir) &&
 (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 if (isDef && (varTypeToSigned(indir) == varTypeToSigned(varDsc)))
 {
 assert(varTypeIsSmall(indir));
 return IndirTransform::LclVar;
Expand All@@ -1024,6 +1060,14 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 return IndirTransform::LclFld;
 }

#ifdef FEATURE_HW_INTRINSICS
 if (varTypeIsSIMD(varDsc) && indir->TypeIs(TYP_FLOAT) && ((val.Offset() % genTypeSize(TYP_FLOAT)) == 0) &&
 m_compiler->IsBaselineSimdIsaSupported())
 {
 return isDef ? IndirTransform::WithElement : IndirTransform::GetElement;
 }
#endif // FEATURE_HW_INTRINSICS

 // Turn this into a bitcast if we can.
 if ((genTypeSize(indir) == genTypeSize(varDsc)) && (varTypeIsFloating(indir) || varTypeIsFloating(varDsc)))
 {
Expand DownExpand Up@@ -1139,16 +1183,23 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 // the promoted local would look like "{ int a, B }", while the IR would contain "FIELD"
 // nodes for the outer struct "A".
 //
 if (indir->TypeIs(TYP_STRUCT))
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND) && indir->TypeIs(TYP_STRUCT))
 {
 // TODO-1stClassStructs: delete this once "IND<struct>" nodes are no more.
 if (indir->OperIs(GT_IND))
 {
 // We do not have a layout for this node.
 return;
 }
 return;
 }

 ClassLayout* layout = indir->GetLayout(m_compiler);
 ClassLayout* layout = indir->TypeIs(TYP_STRUCT) ? indir->GetLayout(m_compiler) : nullptr;
 unsigned indSize = indir->TypeIs(TYP_STRUCT) ? layout->GetSize() : genTypeSize(indir);
 if (indSize > genTypeSize(fieldType))
 {
 // Retargeting this indirection to reference the promoted field would make it
 // "wide", address-exposing the whole parent struct (with all of its fields).
 return;
 }

 if (indir->TypeIs(TYP_STRUCT))
 {
 indir->SetOper(GT_OBJ);
 indir->AsBlk()->SetLayout(layout);
 indir->AsBlk()->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid;
Expand DownExpand Up@@ -1298,6 +1349,27 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
 {
 return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node));
 }

 //------------------------------------------------------------------------
 // BashToLclVar: Bash node to a LCL_VAR.
 //
 // Arguments:
 // node - the node to bash
 // lclNum - the local's number
 //
 // Return Value:
 // The bashed node.
 //
 GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum)
 {
 LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum);

 node->ChangeOper(GT_LCL_VAR);
 node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc));
 node->AsLclVar()->SetLclNum(lclNum);

 return node->AsLclVar();
 }
};

//------------------------------------------------------------------------
Expand All@@ -1314,6 +1386,7 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
//
PhaseStatus Compiler::fgMarkAddressExposedLocals()
{
 bool madeChanges = false;
 LocalAddressVisitor visitor(this);

 for (BasicBlock* const block : Blocks())
Expand All@@ -1323,27 +1396,129 @@ PhaseStatus Compiler::fgMarkAddressExposedLocals()

 for (Statement* const stmt : block->Statements())
 {
#ifdef FEATURE_SIMD
 if (opts.OptimizationEnabled() && stmt->GetRootNode()->TypeIs(TYP_FLOAT) &&
 stmt->GetRootNode()->OperIs(GT_ASG))
 {
 madeChanges |= fgMorphCombineSIMDFieldAssignments(block, stmt);
 }
#endif

 visitor.VisitStmt(stmt);
 }
 }

 return visitor.MadeChanges() ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
 madeChanges |= visitor.MadeChanges();

 return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING;
}

//------------------------------------------------------------------------
// fgMarkAddressExposedLocals: Traverses the specified statement and marks address
// exposed locals.
#ifdef FEATURE_SIMD
//-----------------------------------------------------------------------------------
// fgMorphCombineSIMDFieldAssignments:
// If the RHS of the input stmt is a read for simd vector X Field, then this
// function will keep reading next few stmts based on the vector size(2, 3, 4).
// If the next stmts LHS are located contiguous and RHS are also located
// contiguous, then we replace those statements with one store.
//
// Arguments:
// stmt - the statement to traverse
// Argument:
// block - BasicBlock*. block which stmt belongs to
// stmt - Statement*. the stmt node we want to check
//
// Notes:
// Trees such as IND(ADDR(LCL_VAR)), that morph is expected to fold
// to just LCL_VAR, do not result in the involved local being marked
// address exposed.
// Return Value:
// Whether the assignments were successfully coalesced.
//
void Compiler::fgMarkAddressExposedLocals(Statement* stmt)
bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, Statement* stmt)
{
 LocalAddressVisitor visitor(this);
 visitor.VisitStmt(stmt);
 GenTree* tree = stmt->GetRootNode();
 assert(tree->OperGet() == GT_ASG);

 GenTree* originalLHS = tree->AsOp()->gtOp1;
 GenTree* prevLHS = tree->AsOp()->gtOp1;
 GenTree* prevRHS = tree->AsOp()->gtOp2;
 unsigned index = 0;
 CorInfoType simdBaseJitType = CORINFO_TYPE_UNDEF;
 unsigned simdSize = 0;
 GenTree* simdStructNode = getSIMDStructFromField(prevRHS, &simdBaseJitType, &index, &simdSize, true);

 if ((simdStructNode == nullptr) || (index != 0) || (simdBaseJitType != CORINFO_TYPE_FLOAT))
 {
 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
 return false;
 }

 var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
 var_types simdType = getSIMDTypeForSize(simdSize);
 int assignmentsCount = simdSize / genTypeSize(simdBaseType) - 1;
 int remainingAssignments = assignmentsCount;
 Statement* curStmt = stmt->GetNextStmt();
 Statement* lastStmt = stmt;

 while (curStmt != nullptr && remainingAssignments > 0)
 {
 GenTree* exp = curStmt->GetRootNode();
 if (exp->OperGet() != GT_ASG)
 {
 break;
 }
 GenTree* curLHS = exp->gtGetOp1();
 GenTree* curRHS = exp->gtGetOp2();

 if (!areArgumentsContiguous(prevLHS, curLHS) || !areArgumentsContiguous(prevRHS, curRHS))
 {
 break;
 }

 remainingAssignments--;
 prevLHS = curLHS;
 prevRHS = curRHS;

 lastStmt = curStmt;
 curStmt = curStmt->GetNextStmt();
 }

 if (remainingAssignments > 0)
 {
 // if the left assignments number is bigger than zero, then this means
 // that the assignments are not assigning to the contiguously memory
 // locations from same vector.
 return false;
 }

 JITDUMP("\nFound contiguous assignments from a SIMD vector to memory.\n");
 JITDUMP("From " FMT_BB ", " FMT_STMT " to " FMT_STMT "\n", block->bbNum, stmt->GetID(), lastStmt->GetID());

 for (int i = 0; i < assignmentsCount; i++)
 {
 fgRemoveStmt(block, stmt->GetNextStmt());
 }

 GenTree* dstNode;

 if (originalLHS->OperIs(GT_LCL_FLD))
 {
 dstNode = originalLHS;
 dstNode->gtType = simdType;
 }
 else
 {
 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
 dstNode = gtNewOperNode(GT_IND, simdType, copyBlkDst);
 }

 JITDUMP("\n" FMT_BB " " FMT_STMT " (before):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 assert(!simdStructNode->CanCSE() && varTypeIsSIMD(simdStructNode));
 simdStructNode->ClearDoNotCSE();

 tree = gtNewAssignNode(dstNode, simdStructNode);

 stmt->SetRootNode(tree);

 JITDUMP("\nReplaced " FMT_BB " " FMT_STMT " (after):\n", block->bbNum, stmt->GetID());
 DISPSTMT(stmt);

 return true;
}
#endif // FEATURE_SIMD
Loading