') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Ensure derive_channel_keys doesn't panic if per-run seed is high by TheBlueMatt · Pull Request #1935 · lightningdevkit/rust-lightning · GitHub
Skip to content

Ensure derive_channel_keys doesn't panic if per-run seed is high - #1935

Merged
TheBlueMatt merged 2 commits into
lightningdevkit:mainfrom
TheBlueMatt:2022-12-no-non-time-panic
Jan 3, 2023
Merged

Ensure derive_channel_keys doesn't panic if per-run seed is high#1935
TheBlueMatt merged 2 commits into
lightningdevkit:mainfrom
TheBlueMatt:2022-12-no-non-time-panic

Conversation

@TheBlueMatt

Copy link
Copy Markdown
Collaborator

b04d1b8 changed the way we
calculate the channel_keys_id to include the 128-bit
user_channel_id as well, shifting the counter up four bytes and
the starting_time_nanos field up into the second four bytes.

In derive_channel_keys we hash the full channel_keys_id with an
HD-derived key from our master seed. Previously, that key was
derived with an index of the per-restart counter, re-calculated by
pulling the second four bytes out of the user_channel_id. Because
the channel_keys_id fields were shifted up four bytes, that is
now a reference to the starting_time_nanos value. This should be
fine, the derivation doesn't really add any value here, its all
being hashed anyway, except that derivation IDs must be below 2^31.
This implies that we panic if the user passes a
starting_time_nanos which has the high bit set. For those using
the nanosecond part of the current time this isn't an issue - the
value cannot exceed 1_000_000, which does not have the high bit
set, however, some users may use some other per-run seed.

Thus, here we simply drop the high bit from the seed, ensuring we
don't panic. Note that this is backwards compatible as it only
changes the key derivation in cases where we previously panicked.

Ideally we'd drop the derivation entirely, but that would break
backwards compatibility of key derivation.

I'm a little less concerned about this for rust users, but I do think we should backport this to 113 bindings, as bindings users more often use non-time for the starting_time_nanos field.

b04d1b8 changed the way we
calculate the `channel_keys_id` to include the 128-bit
`user_channel_id` as well, shifting the counter up four bytes and
the `starting_time_nanos` field up into the second four bytes.
In `derive_channel_keys` we hash the full `channel_keys_id` with an
HD-derived key from our master seed. Previously, that key was
derived with an index of the per-restart counter, re-calculated by
pulling the second four bytes out of the `user_channel_id`. Because
the `channel_keys_id` fields were shifted up four bytes, that is
now a reference to the `starting_time_nanos` value. This should be
fine, the derivation doesn't really add any value here, its all
being hashed anyway, except that derivation IDs must be below 2^31.
This implies that we panic if the user passes a
`starting_time_nanos` which has the high bit set. For those using
the nanosecond part of the current time this isn't an issue - the
value cannot exceed 1_000_000, which does not have the high bit
set, however, some users may use some other per-run seed.
Thus, here we simply drop the high bit from the seed, ensuring we
don't panic. Note that this is backwards compatible as it only
changes the key derivation in cases where we previously panicked.
Ideally we'd drop the derivation entirely, but that would break
backwards compatibility of key derivation.
@TheBlueMattTheBlueMatt added this to the 0.0.114 milestone Dec 28, 2022
@codecov-commenter

codecov-commenter commented Dec 28, 2022

Copy link
Copy Markdown

Codecov Report

Base: 90.77% // Head: 91.72% // Increases project coverage by +0.94% 🎉

Coverage data is based on head (5dde803) compared to base (f6a9382).
Patch coverage: 83.33% of modified lines in pull request are covered.

Additional details and impacted files
@@ Coverage Diff @@## main #1935 +/- ##
==========================================
+ Coverage 90.77% 91.72% +0.94% 
==========================================
Files 94 96 +2 Lines 49603 57157 +7554 Branches 49603 57157 +7554 ==========================================
+ Hits 45028 52427 +7399 - Misses 4575 4730 +155 
Impacted FilesCoverage Δ
lightning/src/chain/keysinterface.rs86.02% <83.33%> (+2.87%)⬆️
lightning/src/util/events.rs29.35% <0.00%> (-0.23%)⬇️
lightning/src/chain/onchaintx.rs95.17% <0.00%> (-0.21%)⬇️
lightning/src/ln/reorg_tests.rs100.00% <0.00%> (ø)
lightning/src/offers/refund.rs93.73% <0.00%> (ø)
lightning/src/ln/outbound_payment.rs89.26% <0.00%> (ø)
lightning/src/ln/channel.rs89.03% <0.00%> (+0.17%)⬆️
lightning/src/ln/monitor_tests.rs99.84% <0.00%> (+0.27%)⬆️
lightning/src/ln/inbound_payment.rs93.78% <0.00%> (+0.29%)⬆️
lightning/src/offers/offer.rs93.91% <0.00%> (+0.46%)⬆️
... and 12 more

Help us with your feedback. Take ten seconds to tell us how you rate us. Have a feature suggestion? Share it here.

☔ View full report at Codecov.
📢 Do you have feedback about the report comment? Let us know in this issue.

@tnulltnull left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks good, two nits.

Comment threadlightning/src/chain/keysinterface.rs
Comment threadlightning/src/chain/keysinterface.rs Outdated
// starting_time provided in the constructor) to be unique.
let child_privkey = self.channel_master_key.ckd_priv(&self.secp_ctx, ChildNumber::from_hardened_idx(chan_id as u32).expect("key space exhausted")).expect("Your RNG is busted");
let child_privkey = self.channel_master_key.ckd_priv(&self.secp_ctx,
ChildNumber::from_hardened_idx((chan_id as u32) % (1 << 31)).expect("key space exhausted")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

great catch, but this is getting too complicated. Let's extract chan_id % (1 << 31) into a variable, and then let's extract ChildNumber::from_hardened_idx into a separate variable, and pass that to the method in one line.

Copy link
Copy Markdown
CollaboratorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see how more variables makes things more readable, the indentation as-is makes clear what's being expected where, no?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, most people find that more comfortable to read, but the PR's perfectly fine as is.

Previously, the `derive_channel_keys` derivation ID asserted that
the high bit of the per-channel key derivation counter doesn't
role over as it checked the 31st bit was zero. As we no longer do
that, we should ensure the assertion in `generate_channel_keys_id`
asserts that we don't role over.
@TheBlueMatt
TheBlueMattforce-pushed the 2022-12-no-non-time-panic branch from f544b3b to 5dde803CompareJanuary 3, 2023 17:34
@TheBlueMatt

Copy link
Copy Markdown
CollaboratorAuthor

Pushed the fixup of the comments in the last commit -

diff --git a/lightning/src/chain/keysinterface.rs b/lightning/src/chain/keysinterface.rs
index 2a3fb2cc8..d260d0294 100644
--- a/lightning/src/chain/keysinterface.rs+++ b/lightning/src/chain/keysinterface.rs@@ -1264,8 +1264,9 @@ impl KeysInterface for KeysManager {
fn generate_channel_keys_id(&self, _inbound: bool, _channel_value_satoshis: u64, user_channel_id: u128) -> [u8; 32] {
let child_idx = self.channel_child_index.fetch_add(1, Ordering::AcqRel);
- // child_idx is the only thing guaranteed to make each channel unique without a restart+ // `child_idx` is the only thing guaranteed to make each channel unique without a restart
// (though `user_channel_id` should help, depending on user behavior). If it manages to
- // role over, we're screwed. Because we only support 32-bit+ systems, assert that our- // AtomicUsize doesn't reach u32::MAX.+ // roll over, we may generate duplicate keys for two different channels, which could result+ // in loss of funds. Because we only support 32-bit+ systems, assert that our `AtomicUsize`+ // doesn't reach `u32::MAX`.
assert!(child_idx < core::u32::MAX as usize, "2^32 channels opened without restart");
let mut id = [0; 32];

@TheBlueMatt
TheBlueMatt merged commit e1208bf into lightningdevkit:mainJan 3, 2023
MaxFangX added a commit to lexe-app/rust-lightning that referenced this pull request Jan 12, 2023
phlip9 pushed a commit to lexe-app/lexe-public that referenced this pull request Jan 23, 2023
Like 0.0.112, 0.0.113 is broken; LDK panics with 50% probability if we
use random nanos:
lightningdevkit/rust-lightning#1935
This PR updates LDK past 0.0.113 to the commit that merges the fix.
PXplod pushed a commit to bitlightlabs/rust-lightning that referenced this pull request Sep 30, 2024
0.0.114 - Mar 3, 2023 - "Faster Async BOLT12 Retries"
API Updates
===========
* `InvoicePayer` has been removed and its features moved directly into
`ChannelManager`. As such it now requires a simplified `Router` and supports
`send_payment_with_retry` (and friends). `ChannelManager::retry_payment` was
removed in favor of the automated retries. Invoice payment utilities in
`lightning-invoice` now call the new code (lightningdevkit#1812, lightningdevkit#1916, lightningdevkit#1929, lightningdevkit#2007, etc).
* `Sign`/`BaseSign` has been renamed `ChannelSigner`, with `EcdsaChannelSigner`
split out in anticipation of future schnorr/taproot support (lightningdevkit#1967).
* The catch-all `KeysInterface` was split into `EntropySource`, `NodeSigner`,
and `SignerProvider`. `KeysManager` implements all three (lightningdevkit#1910, lightningdevkit#1930).
* `KeysInterface::get_node_secret` is now `KeysManager::get_node_secret_key`
and is no longer required for external signers (lightningdevkit#1951, lightningdevkit#2070).
* A `lightning-transaction-sync` crate has been added which implements keeping
LDK in sync with the chain via an esplora server (lightningdevkit#1870). Note that it can
only be used on nodes that *never* ran a previous version of LDK.
* `Score` is updated in `BackgroundProcessor` instead of via `Router` (lightningdevkit#1996).
* `ChainAccess::get_utxo` (now `UtxoAccess`) can now be resolved async (lightningdevkit#1980).
* BOLT12 `Offer`, `InvoiceRequest`, `Invoice` and `Refund` structs as well as
associated builders have been added. Such invoices cannot yet be paid due to
missing support for blinded path payments (lightningdevkit#1927, lightningdevkit#1908, lightningdevkit#1926).
* A `lightning-custom-message` crate has been added to make combining multiple
custom messages into one enum/handler easier (lightningdevkit#1832).
* `Event::PaymentPathFailure` is now generated for failure to send an HTLC
over the first hop on our local channel (lightningdevkit#2014, lightningdevkit#2043).
* `lightning-net-tokio` no longer requires an `Arc` on `PeerManager` (lightningdevkit#1968).
* `ChannelManager::list_recent_payments` was added (lightningdevkit#1873).
* `lightning-background-processor` `std` is now optional in async mode (lightningdevkit#1962).
* `create_phantom_invoice` can now be used in `no-std` (lightningdevkit#1985).
* The required final CLTV delta on inbound payments is now configurable (lightningdevkit#1878)
* bitcoind RPC error code and message are now surfaced in `block-sync` (lightningdevkit#2057).
* Get `historical_estimated_channel_liquidity_probabilities` was added (lightningdevkit#1961).
* `ChannelManager::fail_htlc_backwards_with_reason` was added (lightningdevkit#1948).
* Macros which implement serialization using TLVs or straight writing of struct
fields are now public (lightningdevkit#1823, lightningdevkit#1976, lightningdevkit#1977).
Backwards Compatibility
=======================
* Any inbound payments with a custom final CLTV delta will be rejected by LDK
if you downgrade prior to receipt (lightningdevkit#1878).
* `Event::PaymentPathFailed::network_update` will always be `None` if an
0.0.114-generated event is read by a prior version of LDK (lightningdevkit#2043).
* `Event::PaymentPathFailed::all_paths_removed` will always be false if an
0.0.114-generated event is read by a prior version of LDK. Users who rely on
it to determine payment retries should migrate to `Event::PaymentFailed`, in
a separate release prior to upgrading to LDK 0.0.114 if downgrading is
supported (lightningdevkit#2043).
Performance Improvements
========================
* Channel data is now stored per-peer and channel updates across multiple
peers can be operated on simultaneously (lightningdevkit#1507).
* Routefinding is roughly 1.5x faster (lightningdevkit#1799).
* Deserializing a `NetworkGraph` is roughly 6x faster (lightningdevkit#2016).
* Memory usage for a `NetworkGraph` has been reduced substantially (lightningdevkit#2040).
* `KeysInterface::get_secure_random_bytes` is roughly 200x faster (lightningdevkit#1974).
Bug Fixes
=========
* Fixed a bug where a delay in processing a `PaymentSent` event longer than the
time taken to persist a `ChannelMonitor` update, when occurring immediately
prior to a crash, may result in the `PaymentSent` event being lost (lightningdevkit#2048).
* Fixed spurious rejections of rapid gossip sync data when the graph has been
updated by other means between gossip syncs (lightningdevkit#2046).
* Fixed a panic in `KeysManager` when the high bit of `starting_time_nanos`
is set (lightningdevkit#1935).
* Resolved an issue where the `ChannelManager::get_persistable_update_future`
future would fail to wake until a second notification occurs (lightningdevkit#2064).
* Resolved a memory leak when using `ChannelManager::send_probe` (lightningdevkit#2037).
* Fixed a deadlock on some platforms at least when using async `ChannelMonitor`
updating (lightningdevkit#2006).
* Removed debug-only assertions which were reachable in threaded code (lightningdevkit#1964).
* In some cases when payment sending fails on our local channel retries no
longer take the same path and thus never succeed (lightningdevkit#2014).
* Retries for spontaneous payments have been fixed (lightningdevkit#2002).
* Return an `Err` if `lightning-persister` fails to read the directory listing
rather than panicing (lightningdevkit#1943).
* `peer_disconnected` will now never be called without `peer_connected` (lightningdevkit#2035)
Security
========
0.0.114 fixes several denial-of-service vulnerabilities which are reachable from
untrusted input from channel counterparties or in deployments accepting inbound
connections or channels. It also fixes a denial-of-service vulnerability in rare
cases in the route finding logic.
* The number of pending un-funded channels as well as peers without funded
channels is now limited to avoid denial of service (lightningdevkit#1988).
* A second `channel_ready` message received immediately after the first could
lead to a spurious panic (lightningdevkit#2071). This issue was introduced with 0conf
support in LDK 0.0.107.
* A division-by-zero issue was fixed in the `ProbabilisticScorer` if the amount
being sent (including previous-hop fees) is equal to a channel's capacity
while walking the graph (lightningdevkit#2072). The division-by-zero was introduced with
historical data tracking in LDK 0.0.112.
In total, this release features 130 files changed, 21457 insertions, 10113
deletions in 343 commits from 18 authors, in alphabetical order:
* Alec Chen
* Allan Douglas R. de Oliveira
* Andrei
* Arik Sosman
* Daniel Granhão
* Duncan Dean
* Elias Rohrer
* Jeffrey Czyz
* John Cantrell
* Kurtsley
* Matt Corallo
* Max Fang
* Omer Yacine
* Valentine Wallace
* Viktor Tigerström
* Wilmer Paulino
* benthecarman
* jurvis
phlip9 pushed a commit to lexe-app/lexe-public that referenced this pull request Nov 26, 2024
Like 0.0.112, 0.0.113 is broken; LDK panics with 50% probability if we
use random nanos:
lightningdevkit/rust-lightning#1935
This PR updates LDK past 0.0.113 to the commit that merges the fix.
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

4 participants

@TheBlueMatt@codecov-commenter@tnull@arik-so