Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); connection: add per-peer exponential reconnection backoff by Jolah1 · Pull Request #951 · lightningdevkit/ldk-node · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -87,6 +87,9 @@ pub(crate) const LDK_PAYMENT_RETRY_TIMEOUT: Duration = Duration::from_secs(10);
// The time in-between peer reconnection attempts.
pub(crate) const PEER_RECONNECTION_INTERVAL: Duration = Duration::from_secs(60);

// The upper bound on the per-peer exponential backoff applied to failed reconnection attempts.
pub(crate) const PEER_RECONNECTION_MAX_INTERVAL: Duration = Duration::from_secs(60 * 30);

// The time in-between RGS sync attempts.
pub(crate) const RGS_SYNC_INTERVAL: Duration = Duration::from_secs(60 * 60);

Expand Down
153 changes: 150 additions & 3 deletions src/connection.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,22 +8,51 @@
use std::collections::hash_map::{self, HashMap};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::time::{Duration, Instant};

use bitcoin::secp256k1::PublicKey;
use lightning::ln::msgs::SocketAddress;

use crate::config::TorConfig;
use crate::config::{TorConfig, PEER_RECONNECTION_INTERVAL, PEER_RECONNECTION_MAX_INTERVAL};
use crate::logger::{log_debug, log_error, log_info, LdkLogger};
use crate::types::{KeysManager, PeerManager};
use crate::Error;

struct PeerReconnectState {
consecutive_failures: u32,
next_retry_at: Instant,
next_backoff: Duration,
}

impl PeerReconnectState {
fn new(now: Instant) -> Self {
Self {
consecutive_failures: 0,
next_retry_at: now,
next_backoff: PEER_RECONNECTION_INTERVAL,
}
}

/// Bumps the failure count, schedules `next_retry_at` to `now + current backoff`,
/// and doubles the backoff for the following failure (capped at
/// [`PEER_RECONNECTION_MAX_INTERVAL`]). Returns the backoff that was scheduled.
fn record_failure(&mut self, now: Instant) -> Duration {
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
let scheduled_backoff = self.next_backoff;
self.next_retry_at = now + scheduled_backoff;
self.next_backoff =
std::cmp::min(scheduled_backoff.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
scheduled_backoff
}
}

pub(crate) struct ConnectionManager<L: Deref + Clone + Sync + Send>
where
L::Target: LdkLogger,
{
pending_connections:
Mutex<HashMap<PublicKey, Vec<tokio::sync::oneshot::Sender<Result<(), Error>>>>>,
reconnect_state: Mutex<HashMap<PublicKey, PeerReconnectState>>,
peer_manager: Arc<PeerManager>,
tor_proxy_config: Option<TorConfig>,
keys_manager: Arc<KeysManager>,
Expand All@@ -39,14 +68,81 @@ where
keys_manager: Arc<KeysManager>, logger: L,
) -> Self {
let pending_connections = Mutex::new(HashMap::new());
let reconnect_state = Mutex::new(HashMap::new());

Self { pending_connections, peer_manager, tor_proxy_config, keys_manager, logger }
Self {
pending_connections,
reconnect_state,
peer_manager,
tor_proxy_config,
keys_manager,
logger,
}
}

/// Returns whether the background reconnection task should attempt to reconnect
/// to `node_id` now, based on per-peer exponential backoff state.
pub(crate) fn is_reconnect_due(&self, node_id: &PublicKey) -> bool {
self.reconnect_state
.lock()
.expect("lock")
.get(node_id)
.map_or(true, |state| Instant::now() >= state.next_retry_at)
}

/// Records a failed background reconnection attempt: the per-peer retry interval
/// is doubled (up to [`PEER_RECONNECTION_MAX_INTERVAL`]) and the next retry is
/// scheduled `now + <previous interval>`. `now` should be the instant of the
/// reconnection loop tick that scheduled the attempt, so that the resulting
/// schedule stays aligned with the loop's wakeups rather than drifting by the
/// attempt's duration.
///
/// Successful connects clear any backoff state via [`Self::do_connect_peer`].
pub(crate) fn record_reconnect_failure(&self, node_id: &PublicKey, now: Instant) {
let mut state_lock = self.reconnect_state.lock().expect("lock");
let state = state_lock.entry(*node_id).or_insert_with(|| PeerReconnectState::new(now));
let scheduled_backoff = state.record_failure(now);

log_debug!(
self.logger,
"Reconnection to peer {} failed ({} consecutive failures); next retry in {}s",
node_id,
state.consecutive_failures,
scheduled_backoff.as_secs(),
);
}

/// Removes any per-peer backoff state for `node_id`, so a subsequent attempt
/// is treated as a fresh first try. Called when a peer is removed from the
/// persisted peer store.
pub(crate) fn clear_reconnect_state(&self, node_id: &PublicKey) {
self.reconnect_state.lock().expect("lock").remove(node_id);
}

/// Drops backoff state for any peer not in `persisted_peers`. This guards against
/// entries resurrected by a failed attempt that raced the peer's removal from the
/// store, which would otherwise linger forever.
pub(crate) fn prune_reconnect_state(&self, persisted_peers: &[PublicKey]) {
self.reconnect_state
.lock()
.expect("lock")
.retain(|node_id, _| persisted_peers.contains(node_id));
}

/// Returns whether an outbound connection attempt to `node_id` is currently in
/// flight, i.e., whether a [`Self::do_connect_peer`] call would merely subscribe
/// to another task's attempt (possibly targeting a different address).
pub(crate) fn has_pending_connection(&self, node_id: &PublicKey) -> bool {
self.pending_connections.lock().expect("lock").contains_key(node_id)
}

pub(crate) async fn connect_peer_if_necessary(
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
if self.peer_manager.peer_by_node_id(&node_id).is_some() {
// The peer is demonstrably reachable: reset any backoff so a subsequent
// drop is retried promptly.
self.clear_reconnect_state(&node_id);
return Ok(());
}

Expand All@@ -57,6 +153,11 @@ where
&self, node_id: PublicKey, addr: SocketAddress,
) -> Result<(), Error> {
let res = self.do_connect_peer_internal(node_id, addr).await;
if res.is_ok() {
// Any successful connect (including user-initiated ones) resets backoff so the
// background reconnection loop retries promptly if the peer drops again.
self.clear_reconnect_state(&node_id);
}
self.propagate_result_to_subscribers(&node_id, res);
res
}
Expand DownExpand Up@@ -273,3 +374,49 @@ where
}
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn reconnect_state_doubles_until_capped() {
let start = Instant::now();
let mut state = PeerReconnectState::new(start);

let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL);
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_retry_at, start + PEER_RECONNECTION_INTERVAL);

let mut expected = PEER_RECONNECTION_INTERVAL;
for failure_count in 2..32 {
expected = std::cmp::min(expected.saturating_mul(2), PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, expected);
assert_eq!(state.consecutive_failures, failure_count);
assert_eq!(state.next_retry_at, start + expected);
assert!(state.next_backoff <= PEER_RECONNECTION_MAX_INTERVAL);
}

// Once capped, further failures stay at the cap.
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
let scheduled = state.record_failure(start);
assert_eq!(scheduled, PEER_RECONNECTION_MAX_INTERVAL);
assert_eq!(state.next_backoff, PEER_RECONNECTION_MAX_INTERVAL);
}

#[test]
fn reconnect_state_schedules_relative_to_failure_time() {
let t0 = Instant::now();
let mut state = PeerReconnectState::new(t0);

let _ = state.record_failure(t0);
assert_eq!(state.next_retry_at, t0 + PEER_RECONNECTION_INTERVAL);

let t1 = t0 + Duration::from_secs(5);
let scheduled = state.record_failure(t1);
assert_eq!(scheduled, PEER_RECONNECTION_INTERVAL * 2);
assert_eq!(state.next_retry_at, t1 + PEER_RECONNECTION_INTERVAL * 2);
}
}
1 change: 1 addition & 0 deletions src/event.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -1655,6 +1655,7 @@ where
);
return Err(ReplayEvent());
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);
}
}

Expand Down
39 changes: 37 additions & 2 deletions src/lib.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -484,11 +484,45 @@ impl Node {
.map(|peer| peer.counterparty_node_id)
.collect::<Vec<_>>();

for peer_info in connect_peer_store.list_peers().iter().filter(|info| !pm_peers.contains(&info.node_id)) {
let _ = connect_cm.do_connect_peer(
let persisted_peers = connect_peer_store.list_peers();
let persisted_node_ids = persisted_peers
.iter()
.map(|peer| peer.node_id)
.collect::<Vec<_>>();
connect_cm.prune_reconnect_state(&persisted_node_ids);

// Anchor this tick's backoff bookkeeping to the tick instant
// rather than each attempt's completion time, so retries stay
// aligned with the loop's wakeups.
let tick_start = Instant::now();

for peer_info in persisted_peers.iter() {
if pm_peers.contains(&peer_info.node_id) {
// A connected peer (e.g., via an inbound connection) is
// proven reachable: reset any backoff so a future
// disconnect is retried promptly again. Note we only
// observe this at tick time: an inbound connection that
// comes and goes entirely within one tick keeps its
// backoff.
connect_cm.clear_reconnect_state(&peer_info.node_id);
continue;
}
if connect_cm.has_pending_connection(&peer_info.node_id) {
// Another task is already dialing this peer, possibly
// towards a different address. Don't join it: its result
// shouldn't drive our backoff for the persisted address.
continue;
}
if !connect_cm.is_reconnect_due(&peer_info.node_id) {
continue;
}
let res = connect_cm.do_connect_peer(
peer_info.node_id,
peer_info.address.clone(),
).await;
if res.is_err() {
connect_cm.record_reconnect_failure(&peer_info.node_id, tick_start);
}
}
}
}
Expand DownExpand Up@@ -1221,6 +1255,7 @@ impl Node {
log_error!(self.logger, "Failed to remove peer {}: {}", counterparty_node_id, e)
},
}
self.connection_manager.clear_reconnect_state(&counterparty_node_id);

self.peer_manager.disconnect_by_node_id(counterparty_node_id);
Ok(())
Expand Down
Loading