diff --git a/Cargo.lock b/Cargo.lock index 2d24781..9fbd646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1945,7 +1945,7 @@ dependencies = [ [[package]] name = "flextunnel-cli" -version = "0.0.78" +version = "0.0.79" dependencies = [ "anyhow", "clap", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "flextunnel-core" -version = "0.0.78" +version = "0.0.79" dependencies = [ "anyhow", "askama", @@ -1992,7 +1992,7 @@ dependencies = [ [[package]] name = "flextunnel-desktop" -version = "0.0.78" +version = "0.0.79" dependencies = [ "aes-gcm", "anyhow", @@ -2021,7 +2021,7 @@ dependencies = [ [[package]] name = "flextunnel-ffi" -version = "0.0.78" +version = "0.0.79" dependencies = [ "flextunnel-core", "iroh", diff --git a/Cargo.toml b/Cargo.toml index c08e6d7..ef6bf0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ default-members = [ ] [workspace.package] -version = "0.0.78" +version = "0.0.79" edition = "2024" description = "SOCKS5/HTTP-proxy-over-QUIC split tunnel via iroh P2P" diff --git a/README.md b/README.md index fd975d0..502a835 100644 --- a/README.md +++ b/README.md @@ -416,8 +416,8 @@ Client auth keypairs are generated with the standalone | `--relay-url ` | Custom relay URLs (repeatable; at least two distinct relays, the same set as the server). Configuring custom relays disables n0 internet discovery (the server is reached via relay hints); mDNS local discovery stays on. | | `--relay-auth-token ` | Shared bearer token sent to every custom relay's WebSocket upgrade. Only valid with `--relay-url` (rejected with the default relays). | | `--auto-reconnect` | Force auto-reconnect on (overrides `auto_reconnect = false` in the config). | -| `--no-auto-reconnect` | Exit on the first disconnection instead of reconnecting. | -| `--max-reconnect-attempts ` | Cap reconnect attempts between successful connections (unlimited if unset). | +| `--no-auto-reconnect` | Exit on the first failed connection attempt or drop instead of retrying. | +| `--max-reconnect-attempts ` | Cap consecutive retries before giving up (unlimited if unset). | | `--quick` | Self-contained ephemeral session (pairs with `server start --quick`): ignore any saved config, print this client's EndpointId (enter it at the quick server's prompt — that allowlist entry is the credential; no auth keypair), prompt for the server EndpointId, then run the live control panel in this terminal. Needs an interactive terminal. Takes no lock and opens no control socket; quitting the panel disconnects. Nothing is persisted. Conflicts with `-c`/`--auth-key(-file)`. | `flextunnel client start` needs at least one flag — run with no arguments and it @@ -616,13 +616,21 @@ Auto-reconnect is **enabled by default** (`auto_reconnect = true`); pass `--no-auto-reconnect` (or set `auto_reconnect = false`) to disable it, and `--auto-reconnect` to force it on over a config that disabled it. -- The **first** connection must succeed. If it fails — bad node id, wrong - relay, server down, or a rejected token — the client **exits immediately** - rather than retrying blindly. -- Once connected at least once, a transient drop triggers reconnection with - **exponential backoff + jitter** (1s → 60s), indefinitely, unless - `--max-reconnect-attempts` caps it or auto-reconnect is disabled. -- A permanent error (auth/config) never retries. +- A failed connection attempt — the **first one included** — or a lost + connection is retried with **exponential backoff + jitter** (1s doubling to + 5 min), indefinitely, unless `--max-reconnect-attempts` caps it or + auto-reconnect is disabled. A server that is down, or not up yet, is the + ordinary case, not a reason to exit: the client waits it out and connects + when the server appears. +- A long outage is cheap to sit through: once the backoff reaches its cap the + client makes one bounded connect attempt every five minutes. Repeated + failures escalate to rebuilding the iroh endpoint from scratch after the + third one, and then at most every 30 minutes for as long as the outage + lasts (see [`docs/architecture.md`](docs/architecture.md#reconnect-policy-client)). +- A permanent error (a rejected key, a malformed config) never retries; that + is the only kind of error the client exits on. +- The control panel shows the outage's progress: failed attempts so far, the + last error, and when the next attempt is due. - The local proxy listeners stay bound across reconnects. Off-list targets keep connecting directly; on-list requests are held for the reconnect (up to 45s) and only then fail with a network-unreachable reply. diff --git a/client.toml.example b/client.toml.example index 24fe5e7..f81245f 100644 --- a/client.toml.example +++ b/client.toml.example @@ -47,8 +47,8 @@ auth_key_file = "~/.config/flextunnel/client.key" # custom relay_urls (rejected with the default iroh relays). # relay_urls = ["https://relay.example"] # relay_auth_token = "shared-relay-secret" -# auto_reconnect = true # default: true. set false to exit on first disconnect -# max_reconnect_attempts = 10 # default: unlimited (omit for no cap) +# auto_reconnect = true # default: true. set false to exit on the first failed attempt or drop +# max_reconnect_attempts = 10 # default: unlimited (omit for no cap). retries before giving up # The split-tunnel routed set (the "tunnel set") is configured on the server and # pushed to this client during the handshake — there is no client-side routed-set diff --git a/crates/flextunnel-cli/src/client_session.rs b/crates/flextunnel-cli/src/client_session.rs index 5bbc11a..6aef318 100644 --- a/crates/flextunnel-cli/src/client_session.rs +++ b/crates/flextunnel-cli/src/client_session.rs @@ -20,7 +20,7 @@ use flextunnel_core::forwards::{ }; use flextunnel_core::config::ForwardConfig; use flextunnel_core::iroh::SecretKey; -use flextunnel_core::proxy::{ClientAuth, ClientConfig, ProxyClient, reserved}; +use flextunnel_core::proxy::{ClientAuth, ClientConfig, ProxyClient, ReconnectStatus, reserved}; use flextunnel_core::transport::endpoint::{ ClientEndpoint, RelayConfig, create_client_endpoint, create_quick_client_endpoint, }; @@ -259,7 +259,7 @@ async fn build_session( http_addr, ever_connected: false, connected_since: None, - last_error: None, + reconnect: ReconnectStatus::default(), disabled_reasons: HashMap::new(), }; @@ -346,6 +346,7 @@ async fn drive_session( fwd_mgr.apply(&forwards); } state.observe_connection(routes.lock().map(|r| r.connected).unwrap_or(false)); + state.reconnect = client.reconnect_status(); // The reconnect loop rebuilds the endpoint after repeated // failures, which changes the (ephemeral) node id — keep the // status display current. @@ -448,7 +449,9 @@ struct SessionState { http_addr: Option, ever_connected: bool, connected_since: Option, - last_error: Option, + /// The core's reconnect progress (failed attempts, last error, next + /// attempt), polled by the ticker; all defaults while connected. + reconnect: ReconnectStatus, /// Bind-failure reasons of forwards switched off by the ticker, keyed by /// forward id, shown next to their rows for the rest of the session. disabled_reasons: HashMap, @@ -494,7 +497,12 @@ impl SessionState { socks_addr: self.socks_addr, http_addr: self.http_addr, status_page_host: reserved::STATUS_HOST.to_string(), - last_error: self.last_error.clone(), + failed_attempts: self.reconnect.failed_attempts, + next_attempt_secs: self + .reconnect + .next_attempt_at + .map(|at| at.saturating_duration_since(Instant::now()).as_secs()), + last_error: self.reconnect.last_error.clone(), routes: wire_routes(routes), forwards: forwards .iter() diff --git a/crates/flextunnel-cli/src/ipc.rs b/crates/flextunnel-cli/src/ipc.rs index 26104ac..a7351e0 100644 --- a/crates/flextunnel-cli/src/ipc.rs +++ b/crates/flextunnel-cli/src/ipc.rs @@ -121,6 +121,13 @@ pub struct StatusSnapshot { pub http_addr: Option, /// Reserved host that is always tunneled to the server's status page. pub status_page_host: String, + /// Consecutive failed connection attempts in the current outage (0 while + /// connected). + pub failed_attempts: u32, + /// Seconds until the next connection attempt is due while backing off; + /// `Some(0)` while an attempt is in progress, `None` when none is pending. + pub next_attempt_secs: Option, + /// What the last connection attempt failed with, while the tunnel is down. pub last_error: Option, pub routes: WireRoutes, pub forwards: Vec, @@ -599,6 +606,8 @@ mod tests { socks_addr: Some("127.0.0.1:1080".parse().unwrap()), http_addr: None, status_page_host: "flextunnel.internal".into(), + failed_attempts: 0, + next_attempt_secs: None, last_error: None, routes: WireRoutes { domains: vec!["*.internal".into()], diff --git a/crates/flextunnel-cli/src/main.rs b/crates/flextunnel-cli/src/main.rs index 4ecf7fa..c8b9f0f 100644 --- a/crates/flextunnel-cli/src/main.rs +++ b/crates/flextunnel-cli/src/main.rs @@ -175,10 +175,10 @@ enum ClientAction { /// Force auto-reconnect on (overrides `auto_reconnect = false` in the config). #[arg(long, conflicts_with = "no_auto_reconnect")] auto_reconnect: bool, - /// Disable auto-reconnect (exit on the first disconnection). + /// Disable auto-reconnect (exit on the first failed connection attempt or drop). #[arg(long, conflicts_with = "auto_reconnect")] no_auto_reconnect: bool, - /// Cap on reconnect attempts between successful connections (unlimited if unset). + /// Cap on consecutive retries before giving up (unlimited if unset). #[arg(long)] max_reconnect_attempts: Option, /// Ignore any saved config, print this client's EndpointId (enter it on diff --git a/crates/flextunnel-cli/src/tui/view.rs b/crates/flextunnel-cli/src/tui/view.rs index 7f3ae9e..cb50f97 100644 --- a/crates/flextunnel-cli/src/tui/view.rs +++ b/crates/flextunnel-cli/src/tui/view.rs @@ -105,6 +105,17 @@ fn header_lines(s: &StatusSnapshot) -> Vec> { if let Some(secs) = s.connected_secs { first.push(Span::styled(format!(" for {}", format_uptime(secs)), DIM)); } + // While down, say how far the retry loop has got and when it tries again, + // so a backoff step of minutes reads as waiting, not stuck. + if s.failed_attempts > 0 { + let next = match s.next_attempt_secs { + Some(secs) if secs > 0 => format!("next in {}", format_uptime(secs)), + _ => "trying now".to_string(), + }; + let n = s.failed_attempts; + let s_ = if n == 1 { "" } else { "s" }; + first.push(Span::styled(format!(" {n} failed attempt{s_}, {next}"), DIM)); + } let proxy = |name: &str, addr: Option| match addr { Some(addr) => Span::raw(format!("{name} {addr}")), diff --git a/crates/flextunnel-core/src/config.rs b/crates/flextunnel-core/src/config.rs index c1bd355..8bf3d62 100644 --- a/crates/flextunnel-core/src/config.rs +++ b/crates/flextunnel-core/src/config.rs @@ -150,9 +150,11 @@ pub struct ClientConfig { /// Shared bearer token sent to every custom relay's WebSocket upgrade. Only /// valid with custom `relay_urls`; rejected with the default iroh relays. pub relay_auth_token: Option, - /// Reconnect with backoff on a transient drop (default true). + /// Retry failed connection attempts and lost connections with backoff + /// (default true; `false` exits on the first failure of either kind). pub auto_reconnect: Option, - /// Cap on reconnect attempts between successful connections. + /// Cap on consecutive retries before the client gives up (default + /// unlimited). pub max_reconnect_attempts: Option, /// Server-direct port forwards (`[[forwards]]` tables). Config-file only — /// there is no CLI flag. diff --git a/crates/flextunnel-core/src/proxy/bridge.rs b/crates/flextunnel-core/src/proxy/bridge.rs index 615ed2d..b5bbecf 100644 --- a/crates/flextunnel-core/src/proxy/bridge.rs +++ b/crates/flextunnel-core/src/proxy/bridge.rs @@ -8,16 +8,17 @@ //! //! [`BRIDGE_ALPN`]: crate::transport::BRIDGE_ALPN //! -//! The connect/auth/heartbeat machinery mirrors [`super::client`], with two -//! deliberate differences in reconnect policy: a bridge retries **forever** (no -//! fail-fast first connect, no attempt cap). The peer server may simply not be -//! up yet, and a server daemon must not exit — or stop serving its other -//! routes — because a peer is down. While the upstream is down, matching -//! streams fail with host-unreachable (see `route_to_bridge` in -//! [`super::server`]). And a bridge never escalates to the client's endpoint -//! rebuild: it dials on the **server's own endpoint**, which is also accepting -//! inbound clients on its persistent identity — rebuilding it to unwedge one -//! upstream would sever every connected client. +//! The connect/auth/heartbeat machinery mirrors [`super::client`], including +//! its backoff series (shared `calculate_backoff`), with two deliberate +//! differences in reconnect policy: a bridge retries **forever** — no +//! auto-reconnect switch, no retry cap. The peer server may simply not be up +//! yet, and a server daemon must not exit — or stop serving its other routes — +//! because a peer is down. While the upstream is down, matching streams fail +//! with host-unreachable (see `route_to_bridge` in [`super::server`]). And a +//! bridge never escalates to the client's endpoint rebuild: it dials on the +//! **server's own endpoint**, which is also accepting inbound clients on its +//! persistent identity — rebuilding it to unwedge one upstream would sever +//! every connected client. use crate::error::{ProxyError, ProxyResult}; use crate::proxy::client::{calculate_backoff, client_heartbeat_loop, connect_with_timeout}; diff --git a/crates/flextunnel-core/src/proxy/client.rs b/crates/flextunnel-core/src/proxy/client.rs index 862380f..10e3cf9 100644 --- a/crates/flextunnel-core/src/proxy/client.rs +++ b/crates/flextunnel-core/src/proxy/client.rs @@ -18,25 +18,43 @@ use std::net::SocketAddr; use std::num::NonZeroU32; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpListener; #[cfg(unix)] use tokio::net::UnixListener; use tokio::sync::{Semaphore, watch}; -/// Reconnect backoff: base 1s, doubling per attempt, capped at 60s. -const RECONNECT_BACKOFF_MAX: u64 = 60; -/// Escalate to a full endpoint rebuild every this many consecutive failed -/// reconnect attempts. The early attempts get the cheap `network_change()` -/// nudge, which repairs dead UDP sockets; a wedge that survives the nudge plus -/// two full connect timeouts is endpoint state a rebind cannot fix — a relay -/// link lost to a ping timeout and never re-established, stale cached paths -/// for the server — which only a fresh endpoint repairs (observed as "restart -/// the client process and it connects instantly"; the rebuild is that restart, -/// in-process). Rebuilding every Nth attempt (not once) keeps a long outage -/// retrying from fresh state without paying the rebuild on every backoff. +/// Reconnect backoff: base 1s, doubling per consecutive failed attempt, capped +/// here. The early steps (1s, 2s, 4s, …) catch a server restart within a +/// minute or two; past them the doubling runs on up to the cap, so a server +/// that stays down for hours or days is probed once every five minutes for as +/// long as it takes. Each probe is one bounded connect ([`CONNECT_TIMEOUT`]) +/// on the endpoint the client already holds, so an outage of any length is +/// cheap to sit through — at the price of noticing the server's return up to +/// five minutes late. Events that make an earlier attempt worthwhile cut the +/// wait short (see [`ProxyClient::wait_backoff`]). +const RECONNECT_BACKOFF_MAX: Duration = Duration::from_secs(300); +/// Escalate to a full endpoint rebuild once this many consecutive attempts of +/// an outage have failed. The attempts before it get the cheap +/// `network_change()` nudge, which repairs dead UDP sockets; a wedge that +/// survives the nudge plus two full connect timeouts is endpoint state a +/// rebind cannot fix — a relay link lost to a ping timeout and never +/// re-established, stale cached paths for the server — which only a fresh +/// endpoint repairs (observed as "restart the client process and it connects +/// instantly"; the rebuild is that restart, in-process). const REBUILD_ENDPOINT_ATTEMPTS: u32 = 3; +/// After an outage's first rebuild, rebuild again no more often than this for +/// as long as the outage lasts. The rebuild is the expensive step — fresh +/// sockets and relay connections, a new ephemeral identity the relay has to +/// learn, the old endpoint's close — and it repairs a wedged *endpoint*; a +/// freshly built endpoint that still cannot connect has ruled that out, and +/// no amount of rebuilding brings back a server (or a network) that is down. +/// The nudge still runs before every attempt, so a network that changes +/// mid-outage is picked up without a rebuild; this is only the backstop for a +/// wedge that develops during a long outage. The budget is per outage: a +/// successful connection resets it. +const REBUILD_ENDPOINT_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60); /// Max jitter (ms) added to each backoff to avoid thundering reconnects. const RECONNECT_JITTER_MAX_MS: u64 = 500; /// Deadline for the server's handshake response. The QUIC keep-alive keeps the @@ -175,9 +193,12 @@ pub struct ClientConfig { /// Shared bearer token sent to every custom relay's WebSocket upgrade. /// Only valid alongside custom `relay_urls`; ignored with the default relays. pub relay_auth_token: Option, - /// Reconnect with backoff on a transient failure instead of exiting. + /// Retry a failed connection attempt or a lost connection with backoff + /// instead of ending the session (`false`: the first failure of either + /// kind ends it — including a server that is merely not up yet). pub auto_reconnect: bool, - /// Cap on reconnect attempts between successful connections (unlimited if None). + /// Cap on consecutive retries — attempts after a failure, before the next + /// success — after which the session ends (unlimited if `None`). pub max_reconnect_attempts: Option, } @@ -234,12 +255,47 @@ impl ServerForwarder { /// Shared with [`crate::proxy::bridge`], whose reconnect policy mirrors the /// client's. pub(crate) fn calculate_backoff(attempt: u32) -> Duration { - let shift = attempt.saturating_sub(1).min(6); // cap the doubling at 2^6 = 64 - let secs = (1u64 << shift).min(RECONNECT_BACKOFF_MAX); + // 2^(attempt-1) seconds; the shift is bounded well past where the cap + // takes over, so it can never overflow. + let shift = attempt.saturating_sub(1).min(16); + let secs = (1u64 << shift).min(RECONNECT_BACKOFF_MAX.as_secs()); let jitter = rand::rng().random_range(0..=RECONNECT_JITTER_MAX_MS); Duration::from_secs(secs) + Duration::from_millis(jitter) } +/// Whether the reconnect loop should rebuild the endpoint before its next +/// attempt: the outage has reached [`REBUILD_ENDPOINT_ATTEMPTS`] consecutive +/// failures, and it either has not rebuilt yet or last did so at least +/// [`REBUILD_ENDPOINT_MIN_INTERVAL`] ago. +fn rebuild_due(attempt: u32, last_rebuild: Option) -> bool { + attempt >= REBUILD_ENDPOINT_ATTEMPTS + && last_rebuild.is_none_or(|at| at.elapsed() >= REBUILD_ENDPOINT_MIN_INTERVAL) +} + +/// An event that cuts a reconnect backoff short (see +/// [`ProxyClient::wait_backoff`]). +enum BackoffWake { + /// The device lost its network path mid-sleep: park until it returns. + PathLost, + /// The embedding app came to the foreground: attempt right away. + Foregrounded, +} + +/// What the reconnect loop is doing while the tunnel is down, for status +/// displays (the CLI panel). All fields are at their defaults while connected. +#[derive(Clone, Debug, Default)] +pub struct ReconnectStatus { + /// Consecutive failed connection attempts in the current outage: 0 while + /// connected, or before the first attempt has failed. + pub failed_attempts: u32, + /// What the most recent attempt failed with. + pub last_error: Option, + /// When the next attempt is due. Already in the past while an attempt is + /// in progress (bounded by [`CONNECT_TIMEOUT`] + [`HANDSHAKE_TIMEOUT`]); + /// `None` when there is no attempt to wait for. + pub next_attempt_at: Option, +} + /// Snapshot of what the tunnel currently forwards: the split-tunnel set the /// server pushed on the last successful handshake, plus whether a connection is /// live right now. Shared with the FFI so the app can display the routed @@ -313,6 +369,9 @@ pub struct ProxyClient { /// timers at all — retrying into a dead path is pure battery burn — and a /// flip back to available reconnects immediately with a fresh backoff. network_available: watch::Sender, + /// The reconnect loop's progress through the current outage, for status + /// displays ([`Self::reconnect_status`]). + reconnect: Mutex, } impl ProxyClient { @@ -327,14 +386,18 @@ impl ProxyClient { local_close: watch::Sender::new(false), background: watch::Sender::new(false), network_available: watch::Sender::new(true), + reconnect: Mutex::new(ReconnectStatus::default()), } } /// Report the embedding app's scene state. Backgrounded, the heartbeat /// slows to [`HEARTBEAT_INTERVAL_IDLE`] (one radio wake a minute instead of /// six); foregrounded, it snaps back to [`HEARTBEAT_INTERVAL`] — a beat - /// already overdue at the faster cadence is sent immediately. Safe to call - /// repeatedly with the same value. + /// already overdue at the faster cadence is sent immediately — and a + /// reconnect backoff in progress ends early: the next attempt runs at + /// once, with a fresh backoff series (the user is looking, and a long + /// backoff step sized for an unattended outage should not keep them + /// waiting). Safe to call repeatedly with the same value. pub fn set_background(&self, background: bool) { self.background.send_replace(background); } @@ -451,17 +514,36 @@ impl ProxyClient { } } + /// A snapshot of the reconnect loop's progress through the current outage + /// (all defaults while connected). + pub fn reconnect_status(&self) -> ReconnectStatus { + self.reconnect + .lock() + .map(|s| s.clone()) + .unwrap_or_default() + } + + fn set_reconnect_status(&self, status: ReconnectStatus) { + if let Ok(mut s) = self.reconnect.lock() { + *s = status; + } + } + /// Bind the local SOCKS5 listener (and the optional HTTP listener) once, then - /// connect to the server and serve them. Reconnect policy (matching ezvpn): - /// the **first** connection must succeed — if it fails, exit immediately (a - /// bad node id, wrong relay, or down server is not worth retrying blindly). - /// Once connected at least once, transient drops are retried with exponential - /// backoff, indefinitely (unless `--max-reconnect-attempts` caps it or - /// `--no-auto-reconnect` is set). The listeners stay bound across reconnects: - /// off-list targets keep connecting directly, while on-list requests are held - /// for the reconnect (failing with network-unreachable only after - /// [`TUNNEL_RECOVERY_HOLD`]). Reconnects that keep failing escalate to a - /// full endpoint rebuild every [`REBUILD_ENDPOINT_ATTEMPTS`] attempts. + /// connect to the server and serve them. Reconnect policy: every + /// recoverable failure — a connection attempt that fails, the first one + /// included, or an established connection that drops — is retried with + /// exponential backoff (1s doubling to [`RECONNECT_BACKOFF_MAX`]), + /// indefinitely, unless `--max-reconnect-attempts` caps it or + /// `--no-auto-reconnect` is set. A server that is down, or not up yet, is + /// the ordinary case, not an error to exit on; only a permanent error (a + /// rejected credential, a malformed config) ends the session. The + /// listeners stay bound across reconnects: off-list targets keep + /// connecting directly, while on-list requests are held for the reconnect + /// (failing with network-unreachable only after [`TUNNEL_RECOVERY_HOLD`]). + /// An outage that reaches [`REBUILD_ENDPOINT_ATTEMPTS`] failures escalates + /// to a full endpoint rebuild, repeated at most every + /// [`REBUILD_ENDPOINT_MIN_INTERVAL`] for as long as the outage lasts. pub async fn run(&self, endpoint: &ClientEndpoint) -> ProxyResult<()> { let socks = match self.config.socks_listen { Some(addr) => Some(TcpListener::bind(addr).await?), @@ -636,35 +718,41 @@ impl ProxyClient { } /// Maintain the server connection: (re)establish + authenticate, publish the - /// live connection and tunnel set for the accept loop, and reconnect with - /// backoff on drops. Reconnect policy is unchanged: the **first** connection - /// must succeed (fail fast); once connected, transient drops are retried. + /// live connection and tunnel set for the accept loop, and retry with + /// backoff when an attempt fails or the connection drops (see + /// [`Self::handle_failure`] for what is retried and [`Self::run`] for the + /// policy). async fn manage_connection( &self, endpoint: &ClientEndpoint, current: &SharedConn, routed_set_shared: &SharedRoutedSet, ) -> ProxyResult<()> { - let mut ever_connected = false; + // Consecutive failed attempts in the current outage; 0 once connected. let mut attempt: u32 = 0; - // Set when the last backoff resumed from a parked (path-lost) state: - // the backoff series was reset, but the endpoint still needs the rebind - // nudge below — the network genuinely changed underneath it. Every - // retry passes through `wait_backoff`, which reassigns it. + // When the current outage last rebuilt the endpoint (`None`: not yet). + let mut last_rebuild: Option = None; + // Set when the last backoff was cut short by an event that reset the + // series (a restored network path, a foregrounded app): the endpoint + // still needs the rebind nudge below — the network may well have + // changed underneath it. Every retry passes through `wait_backoff`, + // which reassigns it. let mut path_returned = false; loop { // Until (re)authenticated, nothing is being forwarded. self.set_connected(false); current.send_replace(None); - if attempt > 0 && attempt.is_multiple_of(REBUILD_ENDPOINT_ATTEMPTS) { + if rebuild_due(attempt, last_rebuild) { // Escalation: the nudge below wasn't enough — rebuild the - // endpoint from scratch (see [`REBUILD_ENDPOINT_ATTEMPTS`]). - // On a rebuild failure (e.g. no route to bind on a dead - // network) the current endpoint stays in place and this - // attempt proceeds with it — the next multiple retries the - // rebuild. - log::warn!("Reconnect still failing after {attempt} attempts; rebuilding the endpoint from scratch"); + // endpoint from scratch (see [`REBUILD_ENDPOINT_ATTEMPTS`] and + // [`REBUILD_ENDPOINT_MIN_INTERVAL`]). On a rebuild failure + // (e.g. no route to bind on a dead network) the current + // endpoint stays in place and this attempt proceeds with it; + // the failed rebuild still counts against the interval — a + // network that dead is a matter for the per-attempt nudge. + log::warn!("Still failing after {attempt} attempts; rebuilding the endpoint from scratch"); + last_rebuild = Some(Instant::now()); if let Err(e) = endpoint.rebuild().await { log::warn!("Endpoint rebuild failed ({e:#}); retrying with the current endpoint"); } @@ -687,11 +775,13 @@ impl ProxyClient { .await { Ok(established) => { - ever_connected = true; - attempt = 0; // reset backoff on a successful connection + // Connected: the outage, if there was one, is over. + attempt = 0; + last_rebuild = None; + self.set_reconnect_status(ReconnectStatus::default()); established } - Err(e) => match self.handle_failure(e, ever_connected, &mut attempt) { + Err(e) => match self.handle_failure(e, &mut attempt) { Ok(backoff) => { path_returned = self.wait_backoff(backoff, &mut attempt).await; continue; @@ -716,7 +806,7 @@ impl ProxyClient { self.set_connected(false); current.send_replace(None); if let Err(e) = maintained { - match self.handle_failure(e, ever_connected, &mut attempt) { + match self.handle_failure(e, &mut attempt) { Ok(backoff) => { path_returned = self.wait_backoff(backoff, &mut attempt).await; continue; @@ -729,34 +819,38 @@ impl ProxyClient { } } - /// Decide what to do with a connection error: `Ok(backoff)` to retry after - /// the given delay, or `Err(e)` to give up. + /// Decide what to do with a failed attempt or a lost connection: + /// `Ok(backoff)` to retry after that delay, or `Err(e)` to end the session. /// - /// Gives up when: the first connection never succeeded (`!ever_connected` — - /// fail fast), auto-reconnect is disabled, the error is permanent - /// (auth/config), or an explicit attempt cap was reached. Otherwise retries. - fn handle_failure( - &self, - e: ProxyError, - ever_connected: bool, - attempt: &mut u32, - ) -> Result { - if !ever_connected || !self.config.auto_reconnect || !e.is_recoverable() { + /// Every recoverable error (`ProxyError::is_recoverable`: a connect that + /// failed or timed out, a dropped connection) is retried, on the first + /// attempt exactly as after a year connected — a server that is down, or + /// not up yet, is the ordinary case — unless auto-reconnect is off or the + /// retry cap is reached. A permanent error (a rejected credential, a + /// malformed config) ends the session: the same credential and config + /// would fail the same way every time. + fn handle_failure(&self, e: ProxyError, attempt: &mut u32) -> Result { + if !self.config.auto_reconnect || !e.is_recoverable() { return Err(e); } *attempt += 1; if let Some(max) = self.config.max_reconnect_attempts && *attempt > max.get() { - log::error!("Giving up after {} reconnect attempt(s): {e}", max.get()); + log::error!("Giving up after {} retries: {e}", max.get()); return Err(e); } let backoff = calculate_backoff(*attempt); log::warn!( - "Connection lost ({e}); reconnecting in {:.1}s (attempt {})", + "{e}; retrying in {:.1}s (attempt {})", backoff.as_secs_f64(), *attempt ); + self.set_reconnect_status(ReconnectStatus { + failed_attempts: *attempt, + last_error: Some(e.to_string()), + next_attempt_at: Some(Instant::now() + backoff), + }); Ok(backoff) } @@ -766,33 +860,68 @@ impl ProxyClient { /// nothing) and return as soon as the path comes back — resetting the /// backoff series so the restored network gets an immediate, fresh /// reconnect. While the path is up this is a plain backoff sleep, except - /// that a mid-sleep loss switches to parking. + /// that a mid-sleep loss switches to parking, and that the embedding app + /// coming to the foreground ends the sleep the same way a restored path + /// does — a backoff step sized for an unattended outage (up to + /// [`RECONNECT_BACKOFF_MAX`]) must not keep a user who is looking waiting. /// - /// Returns whether the wait resumed from a parked state — i.e. the network - /// went away and came back — so the caller can nudge - /// `Endpoint::network_change()` even though the attempt counter was reset. + /// Returns whether the wait was cut short by one of those events (the + /// backoff series was reset), so the caller can nudge + /// `Endpoint::network_change()` even though the attempt counter is 0. async fn wait_backoff(&self, backoff: Duration, attempt: &mut u32) -> bool { let mut available = self.network_available.subscribe(); if !*available.borrow() { - log::info!("Network unavailable; pausing reconnects until a path returns"); - // The sender lives in self, so this cannot error while we run. - let _ = available.wait_for(|a| *a).await; - log::info!("Network available again; reconnecting now"); - *attempt = 0; + self.park_until_path_returns(&mut available, attempt).await; return true; } - let lost_mid_sleep = tokio::select! { - _ = tokio::time::sleep(backoff) => false, - r = available.wait_for(|a| !*a) => r.is_ok(), + // A foreground flip only matters if the app is backgrounded right now; + // otherwise this branch stays inert. The senders live in self, so + // `wait_for` cannot error while we run. + let mut background = self.background.subscribe(); + let foregrounded = async { + if *background.borrow() && background.wait_for(|b| !*b).await.is_ok() { + return; + } + std::future::pending::<()>().await }; - if lost_mid_sleep { - log::info!("Network unavailable; pausing reconnects until a path returns"); - let _ = available.wait_for(|a| *a).await; - log::info!("Network available again; reconnecting now"); - *attempt = 0; - return true; + let cut_short = tokio::select! { + _ = tokio::time::sleep(backoff) => None, + r = available.wait_for(|a| !*a) => r.is_ok().then_some(BackoffWake::PathLost), + _ = foregrounded => Some(BackoffWake::Foregrounded), + }; + match cut_short { + None => false, + Some(BackoffWake::PathLost) => { + self.park_until_path_returns(&mut available, attempt).await; + true + } + Some(BackoffWake::Foregrounded) => { + log::info!("App foregrounded; reconnecting now"); + self.reset_backoff(attempt); + true + } + } + } + + /// Park (no timers) until the device reports a usable path again, then + /// reset the backoff series for an immediate, fresh attempt. + async fn park_until_path_returns( + &self, + available: &mut watch::Receiver, + attempt: &mut u32, + ) { + log::info!("Network unavailable; pausing reconnects until a path returns"); + let _ = available.wait_for(|a| *a).await; + log::info!("Network available again; reconnecting now"); + self.reset_backoff(attempt); + } + + /// Start a fresh backoff series with an attempt due right now. + fn reset_backoff(&self, attempt: &mut u32) { + *attempt = 0; + if let Ok(mut s) = self.reconnect.lock() { + s.next_attempt_at = Some(Instant::now()); } - false } /// Connect to the server and complete the auth handshake, returning the @@ -1992,6 +2121,159 @@ mod tests { assert!(!resumed, "an uninterrupted sleep saw no path return"); } + /// The app coming to the foreground ends a backoff sleep early, with the + /// series reset and the cut-short reported, so a user who is looking never + /// waits out a step sized for an unattended outage. + #[tokio::test] + async fn backoff_ends_early_when_app_foregrounded() { + let client = Arc::new(test_client()); + client.set_background(true); + let c = client.clone(); + let task = tokio::spawn(async move { + let mut attempt = 9; + let cut_short = c.wait_backoff(Duration::from_secs(300), &mut attempt).await; + (attempt, cut_short) + }); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!task.is_finished(), "backoff ended while still backgrounded"); + client.set_background(false); + let (attempt, cut_short) = tokio::time::timeout(Duration::from_secs(5), task) + .await + .expect("backoff did not end on the foreground flip") + .unwrap(); + assert_eq!(attempt, 0, "a foreground flip should reset the backoff series"); + assert!(cut_short); + assert_eq!( + client.reconnect_status().next_attempt_at.map(|t| t <= Instant::now()), + Some(true), + "the next attempt reads as due now" + ); + } + + /// A foreground flip while the device has no path changes nothing: the + /// wait stays parked (there is nothing to attempt into) until the path + /// returns. + #[tokio::test] + async fn parked_backoff_ignores_foreground_flip() { + let client = Arc::new(test_client()); + client.set_background(true); + client.set_network_available(false); + let c = client.clone(); + let task = tokio::spawn(async move { + let mut attempt = 5; + c.wait_backoff(Duration::from_millis(50), &mut attempt).await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + client.set_background(false); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!task.is_finished(), "a parked backoff resumed without a path"); + client.set_network_available(true); + assert!( + tokio::time::timeout(Duration::from_secs(5), task) + .await + .expect("backoff did not resume on path return") + .unwrap() + ); + } + + /// The backoff doubles from 1s and settles at the cap, jitter aside. + #[test] + fn backoff_doubles_to_the_cap() { + let jitter = Duration::from_millis(RECONNECT_JITTER_MAX_MS); + for (attempt, secs) in [(0, 1), (1, 1), (2, 2), (3, 4), (7, 64), (9, 256)] { + let b = calculate_backoff(attempt); + let base = Duration::from_secs(secs); + assert!(b >= base && b <= base + jitter, "attempt {attempt}: {b:?}"); + } + for attempt in [10, 11, 20, u32::MAX] { + let b = calculate_backoff(attempt); + assert!( + b >= RECONNECT_BACKOFF_MAX && b <= RECONNECT_BACKOFF_MAX + jitter, + "attempt {attempt}: {b:?}" + ); + } + } + + /// An outage's first rebuild comes after the third consecutive failure; + /// later ones are spaced by the minimum interval, however many attempts + /// fail in between. + #[test] + fn rebuild_after_third_failure_then_at_most_every_interval() { + assert!(!rebuild_due(0, None)); + assert!(!rebuild_due(REBUILD_ENDPOINT_ATTEMPTS - 1, None)); + assert!(rebuild_due(REBUILD_ENDPOINT_ATTEMPTS, None)); + assert!(rebuild_due(REBUILD_ENDPOINT_ATTEMPTS + 7, None)); + + let just_now = Some(Instant::now()); + assert!(!rebuild_due(REBUILD_ENDPOINT_ATTEMPTS, just_now)); + assert!(!rebuild_due(REBUILD_ENDPOINT_ATTEMPTS * 4, just_now)); + + let long_ago = Instant::now().checked_sub(REBUILD_ENDPOINT_MIN_INTERVAL); + assert!(long_ago.is_some(), "test host has been up for over an hour"); + assert!(rebuild_due(REBUILD_ENDPOINT_ATTEMPTS, long_ago)); + assert!(!rebuild_due(REBUILD_ENDPOINT_ATTEMPTS - 1, long_ago)); + } + + /// A recoverable failure is retried from the very first attempt — a server + /// that is not up yet is not a reason to exit — and the reconnect status + /// reflects it; a permanent error, or auto-reconnect being off, ends the + /// session on that same first failure. + #[test] + fn first_failure_is_retried_unless_permanent_or_reconnect_is_off() { + let client = test_client(); + let mut attempt = 0; + let backoff = client + .handle_failure(ProxyError::Signaling("server not up yet".into()), &mut attempt) + .expect("a transient first failure is retried"); + assert!(backoff >= Duration::from_secs(1)); + assert_eq!(attempt, 1); + let status = client.reconnect_status(); + assert_eq!(status.failed_attempts, 1); + assert!(status.last_error.unwrap().contains("server not up yet")); + assert!(status.next_attempt_at.is_some()); + + assert!( + client + .handle_failure(ProxyError::AuthenticationFailed("rejected".into()), &mut attempt) + .is_err(), + "a permanent error is never retried" + ); + + let no_retry = ProxyClient::new(ClientConfig { + auto_reconnect: false, + ..test_client().config + }); + let mut attempt = 0; + assert!( + no_retry + .handle_failure(ProxyError::ConnectionLost("dropped".into()), &mut attempt) + .is_err() + ); + } + + /// `max_reconnect_attempts` caps consecutive retries: that many are + /// granted, the next failure ends the session. + #[test] + fn retry_cap_ends_the_session_after_that_many_retries() { + let client = ProxyClient::new(ClientConfig { + max_reconnect_attempts: NonZeroU32::new(2), + ..test_client().config + }); + let mut attempt = 0; + for _ in 0..2 { + assert!( + client + .handle_failure(ProxyError::ConnectionLost("dropped".into()), &mut attempt) + .is_ok() + ); + } + assert!( + client + .handle_failure(ProxyError::ConnectionLost("dropped".into()), &mut attempt) + .is_err() + ); + } + /// While the tunnel stays down a held request must wait out the full /// recovery window before giving up — the hold is a real wait, not a /// fail-fast with extra steps. diff --git a/crates/flextunnel-core/src/proxy/e2e_tests.rs b/crates/flextunnel-core/src/proxy/e2e_tests.rs index ce704b7..9812cf9 100644 --- a/crates/flextunnel-core/src/proxy/e2e_tests.rs +++ b/crates/flextunnel-core/src/proxy/e2e_tests.rs @@ -560,6 +560,96 @@ async fn reconnect_rebuilds_a_dead_endpoint() { let _ = std::fs::remove_file(bl_path); } +/// A client started before its server is up keeps trying until the server +/// appears: a failed first connection is retried like any later drop instead +/// of ending the session (the boot-order case — a unit starting ahead of the +/// server, or a server down for maintenance when the client comes up). +/// +/// The stand-in for "not up yet" is the server's own endpoint with nothing +/// serving on it: every accepted connection is closed on the spot, so an +/// attempt fails fast (the alternative, a dead address, would sit out the +/// full connect timeout per attempt). The server then comes up on that same +/// endpoint — same identity, same address — exactly as a late-starting +/// service would. +#[tokio::test] +async fn client_retries_first_connect_until_server_is_up() { + let server_ep = loopback_endpoint(SecretKey::generate(), true).await; + let server_id = server_ep.id(); + let server_addr = EndpointAddr::new(server_id).with_ip_addr(server_ep.bound_sockets()[0]); + let bl_path = temp_blocklist("retry-first-connect"); + let (routed_set, routed_cidrs) = loopback_cidr_set(); + let params = ProxyServerParams { + routed_set, + routed_cidrs, + ..base_params(server_id, bl_path.clone()) + }; + + let (up_tx, mut up_rx) = tokio::sync::watch::channel(false); + tokio::spawn({ + let server_ep = server_ep.clone(); + async move { + loop { + // Only the accept wait is cancellable: an accepted connection + // is always answered (closed), never dropped on the floor. + let incoming = tokio::select! { + incoming = server_ep.accept() => incoming, + _ = up_rx.wait_for(|up| *up) => break, + }; + let Some(incoming) = incoming else { return }; + if let Ok(conn) = incoming.await { + conn.close(0u32.into(), b"not up yet"); + } + } + let server = ProxyServer::new(params); + if let Err(e) = server.run(&server_ep).await { + eprintln!("e2e retry test server task ended: {e}"); + } + } + }); + + let client_ep = ClientEndpoint::from_parts( + loopback_endpoint_seeded(SecretKey::generate(), false, vec![server_addr]).await, + Arc::new(|| Box::pin(async { anyhow::bail!("no rebuild expected in this test") })), + ); + let client = Arc::new(ProxyClient::new(ClientConfig { + server_node_id: server_id.to_string(), + auth: ClientAuth::Key(Box::new(test_client_key().clone())), + socks_listen: None, + http_listen: None, + relay_urls: Vec::new(), + relay_auth_token: None, + auto_reconnect: true, + max_reconnect_attempts: None, + })); + let socks_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let session = tokio::spawn({ + let (client, ep) = (client.clone(), client_ep.clone()); + async move { client.run_with_listener(&ep, socks_listener).await } + }); + + // The session survives its first failure and reports it, rather than + // ending with an error. + wait_until("the first attempt to fail", || { + client.reconnect_status().failed_attempts >= 1 + }) + .await; + assert!(!session.is_finished(), "the session ended on a failed first connect"); + let status = client.reconnect_status(); + assert!(status.last_error.is_some()); + assert!(status.next_attempt_at.is_some()); + + // The server comes up; the next attempt (1s backoff) lands. + up_tx.send_replace(true); + let connected = || client.routes().lock().unwrap().connected; + wait_until("the client to connect once the server is up", connected).await; + let status = client.reconnect_status(); + assert_eq!(status.failed_attempts, 0, "a connection clears the outage"); + assert!(status.last_error.is_none()); + + session.abort(); + let _ = std::fs::remove_file(bl_path); +} + /// Deploy-style connection holding: a SOCKS request for an on-list target /// arriving while the tunnel link is down is *held* for the client's own /// reconnect and then proceeds transparently on the fresh connection, instead diff --git a/crates/flextunnel-core/src/proxy/mod.rs b/crates/flextunnel-core/src/proxy/mod.rs index 2c8c14f..0420acf 100644 --- a/crates/flextunnel-core/src/proxy/mod.rs +++ b/crates/flextunnel-core/src/proxy/mod.rs @@ -18,7 +18,9 @@ pub mod status_page; mod e2e_tests; pub use bridge::{BridgeUpstream, BridgeUpstreamConfig}; -pub use client::{ClientAuth, ClientConfig, ProxyClient, ServerForwarder, TunnelRoutes}; +pub use client::{ + ClientAuth, ClientConfig, ProxyClient, ReconnectStatus, ServerForwarder, TunnelRoutes, +}; pub use dns_forward::DnsForwarder; pub use forward::{ForwardManager, ForwardSpec, ForwardState, ForwardStatus}; pub use routed_set::RoutedSet; diff --git a/crates/flextunnel-ffi/src/lib.rs b/crates/flextunnel-ffi/src/lib.rs index a164fc0..c3bd2ec 100644 --- a/crates/flextunnel-ffi/src/lib.rs +++ b/crates/flextunnel-ffi/src/lib.rs @@ -11,7 +11,7 @@ //! instance may run at a time (a process-global guard rejects a second). //! 2. [`flextunnel_set_forwards`] — reconcile server-direct local forwards. //! 3. [`flextunnel_health`] — cheap liveness probe: is the serve loop still -//! running, or did it give up (bad node id / auth / unreachable server)? +//! running, or did it give up (a rejected key, a malformed config)? //! 4. [`flextunnel_routes`] — snapshot the server-pushed split-tunnel set for UI. //! 5. [`flextunnel_conn_path`] — snapshot the live iroh path(s) (relay/direct) //! for an on-demand "connection path" status readout. @@ -551,8 +551,11 @@ pub unsafe extern "C" fn flextunnel_close_listeners(handle: *const FlextunnelHan /// is backgrounded, `0` when foregrounded. Backgrounded, the app-level /// heartbeat — the connection's only periodic traffic — slows from 10s to 60s, /// keeping the cellular radio in its low-power state almost the whole time; the -/// foreground flip snaps it back and sends any overdue beat immediately. -/// Idempotent; safe to call with the same value repeatedly. +/// foreground flip snaps it back and sends any overdue beat immediately, and if +/// the core is sitting out a reconnect backoff (up to 5 min once a long outage +/// has pushed it to the cap) it ends that wait and attempts at once with a +/// fresh backoff series. Idempotent; safe to call with the same value +/// repeatedly. /// /// Returns 1 on success and -1 for a null handle. /// @@ -615,9 +618,9 @@ pub unsafe extern "C" fn flextunnel_stop(handle: *mut FlextunnelHandle) { /// Liveness probe for the running proxy. /// /// Returns `1` while the connect/serve loop is still running, `0` once it has -/// ended (it gives up on a fatal error — bad node id, auth failure, or an -/// unreachable server on the *first* connect; transient drops after a successful -/// connect keep retrying and stay `1`), and `-1` for a null handle. +/// ended (it gives up only on a permanent error — a bad node id, a rejected +/// key; an unreachable server, on the first attempt or after a drop, keeps +/// retrying with backoff and stays `1`), and `-1` for a null handle. /// /// # Safety /// `handle` must be a valid pointer returned by [`flextunnel_start`] and not yet diff --git a/docs/architecture.md b/docs/architecture.md index 253849a..9a51d09 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -222,24 +222,39 @@ atomically (temp + rename) and loaded at startup. Implemented in `ProxyClient::run` / `handle_failure`: -- The **first** connection must succeed; if it fails (even a transient error), - the client exits — a bad node id, wrong relay, or down server is not worth - retrying blindly. -- After at least one success, transient drops (`ConnectionLost` / `Network` / - `Signaling` — see `ProxyError::is_recoverable`) are retried with **exponential - backoff + jitter** (1s → 60s), indefinitely, unless `--max-reconnect-attempts` - caps it or `--no-auto-reconnect` disables it. -- Permanent errors (`AuthenticationFailed` / `Config`) never retry. -- Every **third** consecutive failure escalates to a **full endpoint rebuild** - (`ClientEndpoint::rebuild`); the other retries nudge - `Endpoint::network_change()` (rebinds dead UDP sockets) instead. The rebuild - swaps in a freshly bound endpoint — new sockets, new +- Every recoverable failure (`ConnectionLost` / `Network` / `Signaling` — see + `ProxyError::is_recoverable`) is retried with **exponential backoff + + jitter** (1s doubling to `RECONNECT_BACKOFF_MAX`, 5 min), indefinitely, + unless `--max-reconnect-attempts` caps it or `--no-auto-reconnect` disables + it. The first attempt is no different from any later one: a server that is + down or not up yet is the ordinary case (boot order, maintenance), not an + error to exit on. A bad node id or relay URL is a `Config` error and still + fails on the first attempt. +- Permanent errors (`AuthenticationFailed` / `Config`) never retry — the same + credential and config would fail the same way every time. +- A long outage costs one bounded connect (`CONNECT_TIMEOUT`) per attempt on + the endpoint the client already holds, once every five minutes at the cap. + Two events cut a backoff step short with a fresh series: the device + reporting its network path back (`set_network_available`) and the embedding + app coming to the foreground (`set_background(false)`), so a user who is + looking never waits out a step sized for an unattended outage. +- An outage's **third** consecutive failure escalates to a **full endpoint + rebuild** (`ClientEndpoint::rebuild`), repeated at most every + `REBUILD_ENDPOINT_MIN_INTERVAL` (30 min) for as long as the outage lasts; the + other retries nudge `Endpoint::network_change()` (rebinds dead UDP sockets) + instead. The rebuild swaps in a freshly bound endpoint — new sockets, new relay connections, fresh discovery — and closes the wedged one in the - background. This is the in-process equivalent of restarting the - client, for wedges a rebind can't fix (a relay link lost to a ping timeout - and never re-established, stale cached paths for the server). The rebuild - skips the startup per-relay probe and tolerates the online-wait failing, so - a partial outage never blocks recovery. + background. This is the in-process equivalent of restarting the client, for + wedges a rebind can't fix (a relay link lost to a ping timeout and never + re-established, stale cached paths for the server). It is the expensive step + and it repairs the *endpoint*; a rebuilt endpoint that still cannot connect + has ruled that out, which is why it is rate-limited rather than repeated on + every third failure. The rebuild skips the startup per-relay probe and + tolerates the online-wait failing, so a partial outage never blocks + recovery. A successful connection resets the budget. +- The loop publishes its progress (`ProxyClient::reconnect_status`: failed + attempts, last error, next attempt due) for status displays; the CLI panel + shows it on the phase line. - The local proxy listeners stay bound across reconnects. Off-list targets keep connecting directly; on-list requests are held for the reconnect — up to 45s (`TUNNEL_RECOVERY_HOLD`), deploy-style connection holding — and only then fail @@ -354,7 +369,8 @@ defenses. | `TUNNEL_OPEN_TIMEOUT` | 30s | `proxy/client.rs` | | `CONNECT_TIMEOUT` (server dial) | 10s | `proxy/dial.rs` | | `MAX_CONCURRENT_CONNECTIONS` | 1024 | `proxy/server.rs` | -| reconnect backoff | 1s → 60s + ≤500ms jitter | `proxy/client.rs` | +| reconnect backoff | 1s → 5 min + ≤500ms jitter | `proxy/client.rs` | +| `REBUILD_ENDPOINT_ATTEMPTS` / `REBUILD_ENDPOINT_MIN_INTERVAL` (client endpoint rebuild) | 3rd failure, then ≥30 min apart | `proxy/client.rs` | | `MAX_HANDSHAKE_SIZE` | 64 KiB | `proxy/signaling.rs` | | `MAX_CONTROL_MSG_SIZE` | 16 KiB | `proxy/signaling.rs` | | `MAX_HTTP_HEADER` | 64 KiB | `proxy/http.rs` | diff --git a/docs/systemd.md b/docs/systemd.md index 6a787b4..05293b1 100644 --- a/docs/systemd.md +++ b/docs/systemd.md @@ -55,25 +55,29 @@ loginctl enable-linger "$USER" The client already supervises itself where it matters: -- After the **first** successful connection, auto-reconnect (on by default) - retries transient drops internally with exponential backoff, indefinitely. - Reconnects that keep failing escalate every third attempt to rebuilding the - iroh endpoint from scratch — the in-process equivalent of a unit restart, - covering wedges (a dead relay link, stale path state) that only a fresh - endpoint repairs. The process does not exit, so systemd never gets involved. - Don't disable `auto_reconnect` or set `max_reconnect_attempts` under - systemd — that just replaces the client's backoff with unit restarts, which - re-bind listeners and drop held proxy requests. -- The client **exits nonzero** when the *first* connection fails (server down, - network not up yet at boot) and on permanent auth/config errors. - `Restart=on-failure` + `RestartSec` covers the boot-time window where the - network isn't ready — there's no user-manager `network-online.target` to - order against, and none is needed. - -The one wrinkle: a **permanent** error (bad node id, rejected key, malformed -config) also exits nonzero, so systemd will keep retrying it every -`RestartSec`. That's harmless but noisy — if an instance is flapping, read the -reason with `journalctl --user -u flextunnel-client@`. +- Auto-reconnect (on by default) retries every failed connection attempt and + every lost connection internally with exponential backoff (1s doubling to + 5 min), indefinitely — the first attempt included. A server that is down + when the unit starts, or a network that isn't up yet at boot, is waited + out, not exited on: the client connects when the server appears, and there + is no user-manager `network-online.target` to order against nor any need + for one. A long outage costs one bounded connect attempt every five + minutes. Reconnects that keep failing escalate to rebuilding the iroh + endpoint from scratch (after the third failure, then at most every 30 + minutes) — the in-process equivalent of a unit restart, covering wedges (a + dead relay link, stale path state) that only a fresh endpoint repairs. The + process does not exit, so systemd never gets involved. Don't disable + `auto_reconnect` or set `max_reconnect_attempts` under systemd — that just + replaces the client's backoff with unit restarts, which re-bind listeners + and drop held proxy requests. +- The client **exits nonzero** only on a permanent error: a rejected key, a + bad node id, a malformed config, a proxy port taken by another process. + `Restart=on-failure` + `RestartSec` will keep retrying that every + `RestartSec` — harmless but noisy, and it never fixes itself. If an + instance is flapping, read the reason with + `journalctl --user -u flextunnel-client@`; a client that is merely + waiting for its server is not flapping, it is running, and + `flextunnel client control` shows how far its retry loop has got. ## Interacting with a running instance diff --git a/ios/flextunnel.h b/ios/flextunnel.h index afad678..f1f8208 100644 --- a/ios/flextunnel.h +++ b/ios/flextunnel.h @@ -147,7 +147,9 @@ int flextunnel_close_listeners(const FlextunnelHandle *handle); * Backgrounded, the core's app-level heartbeat — the connection's only periodic * traffic — slows from 10s to 60s so an idle session wakes the cellular radio * once a minute instead of six times; the foreground flip snaps it back and - * sends any overdue beat immediately. Idempotent. + * sends any overdue beat immediately, and ends any reconnect backoff in + * progress (up to 5 min once a long outage has pushed it to the cap) so the + * next attempt runs at once with a fresh backoff series. Idempotent. * * Returns 1 on success and -1 for a NULL handle. */ @@ -166,8 +168,9 @@ int flextunnel_set_network_available(const FlextunnelHandle *handle, int availab /* * Liveness probe. Returns 1 while the connect/serve loop is running, 0 once it - * has ended (gave up on a fatal error: bad node id, auth failure, or an - * unreachable server on the first connect), and -1 for a NULL handle. + * has ended (gave up on a permanent error: a bad node id, a rejected key; an + * unreachable server keeps retrying with backoff, on the first attempt or + * after a drop), and -1 for a NULL handle. */ int flextunnel_health(const FlextunnelHandle *handle);