') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Expose onion messages in public API by valentinewallace · Pull Request #1650 · lightningdevkit/rust-lightning · GitHub
Skip to content

Expose onion messages in public API - #1650

Merged
TheBlueMatt merged 4 commits into
lightningdevkit:mainfrom
valentinewallace:2022-08-take-onionmsgs-public
Sep 2, 2022
Merged

Expose onion messages in public API #1650
TheBlueMatt merged 4 commits into
lightningdevkit:mainfrom
valentinewallace:2022-08-take-onionmsgs-public

Conversation

@valentinewallace

@valentinewallacevalentinewallace commented Aug 4, 2022•
edited
Loading

Copy link
Copy Markdown
Contributor

This completes the final anti-DoS steps needed for us to be able to make the onion messages API public and integrate the OnionMessenger with the PeerManager. See commit messages for more info.

TODO: add tests for the last commit after it's conceptually ACK'd

Blocked on #1604 and #1648

@valentinewallacevalentinewallace mentioned this pull request Aug 4, 2022
Merged
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 848a6d9 to d93d568CompareAugust 4, 2022 21:14
TheBlueMatt
TheBlueMatt reviewed Aug 7, 2022
View reviewed changes
Comment threadlightning/src/onion_message/messenger.rs Outdated
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch 2 times, most recently from 4e8e973 to 3209ac5CompareAugust 9, 2022 19:23
TheBlueMatt
TheBlueMatt reviewed Aug 10, 2022
View reviewed changes
Comment threadlightning/src/ln/features.rs Outdated
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 3209ac5 to 6b01547CompareAugust 15, 2022 18:27
@jkczyz
jkczyz self-requested a review August 19, 2022 19:47
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 6b01547 to ee37e44CompareAugust 26, 2022 15:16
@valentinewallace
valentinewallace marked this pull request as ready for review August 26, 2022 15:17
@valentinewallace

valentinewallace commented Aug 26, 2022

Copy link
Copy Markdown
ContributorAuthor

Now based on #1604

@TheBlueMatt

TheBlueMatt commented Aug 29, 2022

Copy link
Copy Markdown
Collaborator

Needs rebase.

TheBlueMatt
TheBlueMatt reviewed Aug 29, 2022
View reviewed changes
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/ln/features.rs Outdated
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from ee37e44 to 664f139CompareAugust 31, 2022 17:40
@codecov-commenter

codecov-commenter commented Aug 31, 2022•
edited
Loading

Copy link
Copy Markdown

Codecov Report

Merging #1650 (447ba29) into main (0624cf9) will increase coverage by 0.13%.
The diff coverage is 90.58%.

❗ Current head 447ba29 differs from pull request most recent head a8ea0bd. Consider uploading reports for the commit a8ea0bd to get more accurate results

@@ Coverage Diff @@## main #1650 +/- ##
==========================================
+ Coverage 90.85% 90.98% +0.13% 
==========================================
Files 85 85 Lines 45948 46681 +733 Branches 45948 46681 +733 ==========================================
+ Hits 41746 42474 +728 - Misses 4202 4207 +5
Impacted FilesCoverage Δ
lightning/src/lib.rs100.00% <ø> (ø)
lightning/src/ln/features.rs99.46% <ø> (ø)
lightning/src/ln/msgs.rs86.24% <ø> (+0.49%)⬆️
lightning/src/onion_message/packet.rs72.52% <50.00%> (+7.69%)⬆️
lightning/src/onion_message/messenger.rs89.55% <87.27%> (+0.01%)⬆️
lightning/src/ln/peer_handler.rs57.02% <100.00%> (+0.25%)⬆️
lightning/src/onion_message/functional_tests.rs96.33% <100.00%> (+0.45%)⬆️
lightning/src/routing/gossip.rs91.43% <0.00%> (-0.24%)⬇️
lightning/src/ln/functional_tests.rs96.96% <0.00%> (-0.15%)⬇️
lightning/src/routing/scoring.rs96.14% <0.00%> (+0.01%)⬆️
... and 7 more

Help us with your feedback. Take ten seconds to tell us how you rate us. Have a feature suggestion? Share it here.

@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 664f139 to ec1af89CompareAugust 31, 2022 17:47
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from ec1af89 to 366662eCompareAugust 31, 2022 18:59
@valentinewallacevalentinewallace mentioned this pull request Aug 31, 2022
Merged
@valentinewallace

valentinewallace commented Aug 31, 2022

Copy link
Copy Markdown
ContributorAuthor

@TheBlueMatt should we only be connecting peers to OnionMessenger if and when we have an open channel? Or does this new approach make that less important?

@TheBlueMatt

TheBlueMatt commented Sep 1, 2022

Copy link
Copy Markdown
Collaborator

We should consider some kind of only-channel-peers thing in the future, I think, but just as a way to give those peers larger buffers and give other peers almost no buffer space. That said, we need to limit the buffer in OnionMessager to some size. Given we're in OM-specific stuff, let's do it as some number of bytes, not a number of messages (ie we have to walk the pending messages map before deciding to insert). Should be super trivial but needs to happen here I think.

Picking a random number for the buffer limit, let's go with 256KiB for now. Default open files limit on my local machine in front of me is 1024, which means we'd max out at 256MiB in buffer size.

@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 366662e to 0ed9416CompareSeptember 1, 2022 19:28
@TheBlueMatt

TheBlueMatt commented Sep 1, 2022

Copy link
Copy Markdown
Collaborator

Looks like the fuzzing seeds need updating.

@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 0ed9416 to abf3517CompareSeptember 1, 2022 20:05
TheBlueMatt
TheBlueMatt reviewed Sep 2, 2022
View reviewed changes
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs Outdated
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch 3 times, most recently from 5251618 to 3a4a6ebCompareSeptember 2, 2022 19:24
TheBlueMatt
TheBlueMatt reviewed Sep 2, 2022
View reviewed changes
Comment threadlightning/src/onion_message/messenger.rs Outdated
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 3a4a6eb to 447ba29CompareSeptember 2, 2022 19:31
TheBlueMatt
TheBlueMatt previously approved these changes Sep 2, 2022
View reviewed changes
jkczyz
jkczyz reviewed Sep 2, 2022
View reviewed changes

@jkczyzjkczyz left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No major concerns.

Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs Outdated
Comment threadlightning/src/onion_message/functional_tests.rs Outdated
Comment threadlightning/src/onion_message/messenger.rs
let mut peer_buffered_bytes = 0;
for (pk, peer_buf) in buffer {
for om in peer_buf {
let om_len = om.serialized_length();

@jkczyzjkczyzSep 2, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we optimize this by implmenting serialized_length on OnionMessage and Packet? Seems a bit wasteful to serialize all pending messages any time an onion message is sent.

@TheBlueMattTheBlueMattSep 2, 2022

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd put good money on it being free. In the past I've investigated this for a few things and LLVM is pretty good at dealing with the simple cases. As the comment for serialized_length says "Note that LLVM optimizes this away in most cases! Check that it isn't before you override!"

@jkczyzjkczyzSep 2, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, right. Was just looking at the implementation and noticed that comment. Feel free to disregard!

@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 447ba29 to 235f4a5CompareSeptember 2, 2022 20:29
TheBlueMatt
TheBlueMatt previously approved these changes Sep 2, 2022
View reviewed changes
@valentinewallace
Limit OnionMessenger outbound buffer size
a8ea0bd
Drop OMs if they push us over the max OnionMessenger outbound buffer size
@valentinewallace
valentinewallaceforce-pushed the 2022-08-take-onionmsgs-public branch from 235f4a5 to a8ea0bdCompareSeptember 2, 2022 20:35
@TheBlueMatt
TheBlueMatt merged commit fc7b14b into lightningdevkit:mainSep 2, 2022
@jkczyzjkczyz mentioned this pull request May 10, 2023
Open
60 tasks
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@TheBlueMattTheBlueMattTheBlueMatt approved these changes
@jkczyzjkczyzjkczyz approved these changes

Assignees

@TheBlueMattTheBlueMatt

@jkczyzjkczyz

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

4 participants

@valentinewallace@TheBlueMatt@codecov-commenter@jkczyz