') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); [release/6.0] Port stackoverflow fix from Roslyn to SourceGenerator PolyFill by CyrusNajmabadi · Pull Request #76946 · dotnet/runtime · GitHub
Skip to content

[release/6.0] Port stackoverflow fix from Roslyn to SourceGenerator PolyFill - #76946

Merged
carlossanlop merged 1 commit into
dotnet:release/6.0from
CyrusNajmabadi:generatorStackOverflow
Oct 13, 2022
Merged

[release/6.0] Port stackoverflow fix from Roslyn to SourceGenerator PolyFill#76946
carlossanlop merged 1 commit into
dotnet:release/6.0from
CyrusNajmabadi:generatorStackOverflow

Conversation

@CyrusNajmabadi

@CyrusNajmabadiCyrusNajmabadi commented Oct 12, 2022•
edited by joperezr
Loading

Copy link
Copy Markdown
Contributor

Fixes#76953
Ports dotnet/roslyn#64322 back to the runtime polyfill.

Corresponding 7.0 fix is here: #76954

Customer Impact

Customers using 6.0.x sdk can run into stack-overflows in the Source-Generator helper code code, in both VS and compiler, when compiling code with certain problematic code-constructs. This happens for tehse customers even if they are not using Source-Generators themselves as the overflow occurs in the code that is trying to determine if the generator should run.

This commonly happens when working with, or trying to compile, generated code produced by tools like T4 or Antlr (which commonly generate deeply recursive trees).

Examples of code that causes this are:

stringdata="..."+"..."+"..."+/* thousands more concatenations */+"..."+"..."+"...";

In a case like this, the concatenation code is not a balanced tree but instead, effectively, a linear tree like so:

image

The existing code works by recursing the user's parse tree, which ends up blowing the stack in cases like these.

As this is a stack overflow, it is fairly catastrophic for the user. Absent them changing this code outside of VS, which may not be possible for tool-generated code, the only workaround is to disable these generators (like System.Text.Json). Disabling the generator may be suitable for customers that are not using that generator, but will completely break users who are using it and who do have these code constructs.

Testing

The affected source generators have automation tests which provide some coverage for the main scenarios. Additionally, we performed manual testing, specifically against the reported repro projects, and after the changes on this PR, we have validated that the issue no longer repros.

Risk

Low-Medium. This is a simple port of a change made in roslyn to switch from implicit recursion, to using an explicit stack. This is a refactoring that roslyn is familiar with as stack-overflows are not uncommon for us as users do have these sorts of files, and recursive solutions have often had to be refactored in a similar fashion. The reason why we add Medium to the risk, is only because we don't currently have full automation tests for running against older versions of VS, but we have validated that those work manually.

@ghost

ghost commented Oct 12, 2022

Copy link
Copy Markdown

I couldn't figure out the best area label to add to this PR. If you have write-permissions please help me learn by adding exactly one area label.

@stephentoub

stephentoub commented Oct 12, 2022

Copy link
Copy Markdown
Member

What's the empty csproj being added?

@CyrusNajmabadi

CyrusNajmabadi commented Oct 12, 2022

Copy link
Copy Markdown
ContributorAuthor

What's the empty csproj being added?

No clue. Must have been teh command line operations i was performing.

@joperezr

joperezr commented Oct 12, 2022

Copy link
Copy Markdown
Member

@CyrusNajmabadi is this a direct port of the Roslyn change? Or did you have to make some adjustments? Mainly asking to see if we have some blindspots as this code is not very well tested in runtime.

@CyrusNajmabadi

CyrusNajmabadi commented Oct 12, 2022

Copy link
Copy Markdown
ContributorAuthor

@CyrusNajmabadi is this a direct port of the Roslyn change? Or did you have to make some adjustments? Mainly asking to see if we have some blindspots as this code is not very well tested in runtime.

Definitely had to make some adjustments. Specifically:

  1. i have to use ValueListBuilder
  2. i need to pass items by ref across the helper functions.

That said, it is otherwise identical. These changes are the same ones i needed to make when just doing the original port as well. And the core logic is the same.

@carlossanlop

carlossanlop commented Oct 12, 2022

Copy link
Copy Markdown
Contributor

@CyrusNajmabadi when this is ready, please add the servicing-consider label, fill out the template in the PR description, then send an email to Tactics requesting approval.

@carlossanlopcarlossanlop changed the title Port stackoverflow fix from Roslyn to SourceGenerator PolyFill[release/6.0] Port stackoverflow fix from Roslyn to SourceGenerator PolyFillOct 12, 2022
@joperezr

joperezr commented Oct 12, 2022

Copy link
Copy Markdown
Member

I see. Let's make sure that we thoroughly test this manually to make sure things are working as expected. I'll sync with you offline to do that.

@CyrusNajmabadiCyrusNajmabadi added the Servicing-consider Issue for next servicing release review label Oct 12, 2022
@CyrusNajmabadi

CyrusNajmabadi commented Oct 12, 2022

Copy link
Copy Markdown
ContributorAuthor

fill out the template in the PR description

Happy to. Where is this template?

@carlossanlop

carlossanlop commented Oct 12, 2022

Copy link
Copy Markdown
Contributor

@joperezr just added the headers to the description to help.

If you need inspiration, here are some past examples.

This was referenced Oct 12, 2022
Merged
Merged
@joperezr

joperezr commented Oct 12, 2022

Copy link
Copy Markdown
Member

This change will not be enough for .NET 6. We will also need to service all NuGet packages that use the polyfill approach. I'll push the relevant changes in a bit

@joperezr

joperezr commented Oct 12, 2022

Copy link
Copy Markdown
Member

Looks like the only 2 packages affected by this are System.Text.Json and Microsoft.Extensions.Logging.Abstractions, and both of them are already inline to get serviced in the next servicing train, so no need for further changes here.

@build-analysisbuild-analysisBot mentioned this pull request Oct 12, 2022
Closed
joperezr
joperezr reviewed Oct 12, 2022
View reviewed changes
Comment threadsrc/libraries/Common/src/Roslyn/SyntaxValueProvider_ForAttributeWithSimpleName.cs
localAliases.Length = localAliasCount;
}
else if (syntaxHelper.IsAttributeList(node))
processCompilationOrNamespaceMembers(

@joperezrjoperezrOct 12, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about this recursion? Can't you still have stackoverflow if some generated code has a lot of namespaces nested within each other?

@CyrusNajmabadiCyrusNajmabadiOct 12, 2022

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the goal is not to prevent all recursion, it's to prevent recursion on cases that occur in the wild in reasonable scenarios. So, a user having a highly recursive expression (like a + b+ c+ ...) happens in teh wild and is a problem. We do not see users having insanely deeply recursive namespaces. Roslyn itself does not guard against that here or for other recursive cases.

Note: this algorithm is hard to make 'fully recursive' due to how it pushes/pops state as it enters/exits namespaces. THat's why the namespace portion stays implicitly recursive, but the rest of the code becomes explicitly recursive.

@joperezrjoperezrOct 13, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, my suggestion wasn't really to not make this recursive or to prevent recursion, but instead to protect against a stack overflow in places were it is likely to happen. If the answer is: we don't expect ever to get a stackoverflow here, then I'm fine with not guarding against it. My main concern is that this code is used by analyzers that automatically run on all projects, so if we ever think it may be possible to hit, we should be defensive. That is just my 2 cents.

joperezr
joperezr approved these changes Oct 12, 2022
View reviewed changes

@joperezrjoperezr left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Left a small comment, but looks good otherwise. I've tested these changes locally against the reported repro case, and after this, the issue no longer repros.

@joperezr

joperezr commented Oct 12, 2022

Copy link
Copy Markdown
Member

@carlossanlop are all of the failures here known issues? I've looked at them and none of them seem to be related to what is being changed here.

@carlossanlop

carlossanlop commented Oct 12, 2022

Copy link
Copy Markdown
Contributor

Lots of network issues. Re-run might be the best course of action.

@build-analysisbuild-analysisBot mentioned this pull request Oct 13, 2022
Closed
2 tasks
@joperezrjoperezr added Servicing-approved Approved for servicing release and removed Servicing-consider Issue for next servicing release review labels Oct 13, 2022
@joperezr

joperezr commented Oct 13, 2022

Copy link
Copy Markdown
Member

Approved via email.

@carlossanlop

carlossanlop commented Oct 13, 2022

Copy link
Copy Markdown
Contributor

CI green, approved by Tactics, signed off. Ready to merge. :shipit:

@carlossanlop
carlossanlop merged commit ff4038f into dotnet:release/6.0Oct 13, 2022
@CyrusNajmabadiCyrusNajmabadi mentioned this pull request Oct 13, 2022
Closed
@CyrusNajmabadi
CyrusNajmabadi deleted the generatorStackOverflow branch October 13, 2022 19:18
@ghostghost locked as resolved and limited conversation to collaborators Nov 13, 2022
Sign up for freeto subscribe to this conversation on GitHub. Already have an account? Sign in.

Reviewers

@joperezrjoperezrjoperezr approved these changes

Assignees

@CyrusNajmabadiCyrusNajmabadi

Labels

Servicing-approvedApproved for servicing release

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

4 participants

@CyrusNajmabadi@stephentoub@joperezr@carlossanlop