diff --git a/AGENTS.md b/AGENTS.md index 664eaf3..3af4288 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,4 +7,4 @@ - the desktop client (`flextunnel-desktop`) normally stores its config in the system keychain; set `FLEXTUNNEL_DEV_CONFIG=1` (or a file path) to store it as plaintext JSON instead, avoiding the macOS keychain access prompt on every unsigned rebuild. Development only — never set it for a real install (the auth secret key is stored unencrypted). - after rust changes that affect iOS (flextunnel-core or flextunnel-ffi, including the FFI config schema and `ios/flextunnel.h`), run `./build-ios.sh release` to rebuild `libflextunnel.xcframework` into `dist/ios/` (this script no longer writes into `../flextunnel-ios`). The iOS app links via its own Swift package (`../flextunnel-ios/Packages/Flextunnel`), which **defaults to the pinned GitHub release**, so it won't see local changes unless you build the app with `FLEXTUNNEL_LOCAL_XCFRAMEWORK=1` — that links this fresh `dist/ios` build through a committed symlink (set it for both `xcodegen generate` and `xcodebuild`, then clean-rebuild). This is an **extra step only needed when actively working on the iOS app side by side** (and only possible on macOS with Xcode + the iOS Rust targets); otherwise just skip it. - the iroh transport layer shared with tunnel-rs and ezvpn — relays and address lookup, the per-relay startup probe, relay auth tokens, relay self-hosting — is documented once in https://github.com/flexaccessdev/iroh-common-architecture. Do not duplicate it in this repo; update it there and link to it. -- that shared layer's code — `RelayConfig` and the relay probe, endpoint building and rebuild, the home-relay watchdog, the endpoint-bound auth transcript — lives in the `flexaccess-iroh` crate (`../flexaccess-iroh`, consumed by git tag). Fix it there, tag a release, and bump the tag here; never re-implement or fork a copy of it in this repo. Only flextunnel-specific pieces (ALPNs, the allowlist hook, the auth context, QUIC tuning) belong in `transport/` and `auth.rs`. +- that shared layer's code — `RelayConfig` and the relay probe, endpoint building and rebuild, the endpoint-bound auth transcript — lives in the `flexaccess-iroh` crate (`../flexaccess-iroh`, consumed by git tag). Fix it there, tag a release, and bump the tag here; never re-implement or fork a copy of it in this repo. Only flextunnel-specific pieces (ALPNs, the allowlist hook, the auth context, QUIC tuning) belong in `transport/` and `auth.rs`. diff --git a/Cargo.lock b/Cargo.lock index 76cec53..b01f52b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,8 +1919,8 @@ dependencies = [ [[package]] name = "flexaccess-iroh" -version = "0.0.3" -source = "git+https://github.com/flexaccessdev/flexaccess-iroh?tag=v0.0.3#422dacee5b38a667bd6da2b709d28d86c8d7661f" +version = "0.0.5" +source = "git+https://github.com/flexaccessdev/flexaccess-iroh?tag=v0.0.5#2dde9379e82890157351b66eba4e1f319cddf7a8" dependencies = [ "anyhow", "base64", @@ -1945,7 +1945,7 @@ dependencies = [ [[package]] name = "flextunnel-cli" -version = "0.0.75" +version = "0.0.76" dependencies = [ "anyhow", "clap", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "flextunnel-core" -version = "0.0.75" +version = "0.0.76" dependencies = [ "anyhow", "askama", @@ -1992,7 +1992,7 @@ dependencies = [ [[package]] name = "flextunnel-desktop" -version = "0.0.75" +version = "0.0.76" dependencies = [ "aes-gcm", "anyhow", @@ -2021,7 +2021,7 @@ dependencies = [ [[package]] name = "flextunnel-ffi" -version = "0.0.75" +version = "0.0.76" dependencies = [ "flextunnel-core", "iroh", diff --git a/Cargo.toml b/Cargo.toml index 74252e1..a2f8990 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ default-members = [ ] [workspace.package] -version = "0.0.75" +version = "0.0.76" edition = "2024" description = "SOCKS5/HTTP-proxy-over-QUIC split tunnel via iroh P2P" @@ -29,9 +29,9 @@ futures = "0.3" ipnetwork = "0.21" hickory-resolver = { version = "0.26", default-features = false, features = ["tokio"] } # Shared iroh transport layer (relay config + probe, endpoint building and -# rebuild, home-relay watchdog, endpoint-bound auth transcript). `mdns` turns on +# rebuild, endpoint-bound auth transcript). `mdns` turns on # local-network discovery on every endpoint (compiled out on iOS by the crate). -flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.3", features = ["mdns"] } +flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.5", features = ["mdns"] } iroh = "1.1.0" libc = "0.2" log = "0.4" diff --git a/README.md b/README.md index c385b1c..77dce38 100644 --- a/README.md +++ b/README.md @@ -619,12 +619,10 @@ Auto-reconnect is **enabled by default** (`auto_reconnect = true`); pass connecting directly; on-list requests are held for the reconnect (up to 45s) and only then fail with a network-unreachable reply. -A **server** with custom relays watches its own home-relay registration: if it -has no connected home relay for 60s it re-checks the network, and if that -has not helped by 180s it rebuilds its endpoint in place (same server id) — -the in-process equivalent of a restart, so relay-only clients (the iOS app, -anything off the LAN) are not stranded until someone restarts the service. -See [`docs/architecture.md`](docs/architecture.md#relay-watchdog-server-custom-relays). +Servers rely on iroh 1.1.x for relay reconnects and keep the same endpoint +during relay outages. The former server watchdog has been removed; see the +[relay recovery history and workaround](https://github.com/flexaccessdev/iroh-common-architecture/blob/9eacd43b80d867a8a4a76e3051237b854b4b0cd5/home-relay-watchdog.md) +if permanent loss of relay registration recurs. ## Logging diff --git a/crates/flextunnel-cli/src/main.rs b/crates/flextunnel-cli/src/main.rs index b36b82a..7d4c462 100644 --- a/crates/flextunnel-cli/src/main.rs +++ b/crates/flextunnel-cli/src/main.rs @@ -34,9 +34,8 @@ use flextunnel_core::proxy::{ }; use flextunnel_core::secret::secret_to_endpoint_id; use flextunnel_core::transport::endpoint::{ - EndpointAllowlists, RelayConfig, create_server_endpoint, server_rebuild_factory, + EndpointAllowlists, RelayConfig, create_server_endpoint, }; -use flextunnel_core::flexaccess_iroh::relay_watchdog::{self, RelayOutage}; use flextunnel_core::{auth, config, secret}; #[derive(Parser)] @@ -617,28 +616,6 @@ struct QuickServer { /// relay/connection teardown must never leave the process unkillable. const SHUTDOWN_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -/// Pause between attempts to bind a replacement endpoint after the relay -/// watchdog retired the old one and the rebuild itself failed (e.g. no route -/// to bind on). The server has no endpoint at all during this wait, so it is -/// short — there is nothing to lose by trying again soon. -const REBUILD_RETRY: Duration = Duration::from_secs(30); - -/// Cap on the watchdog's rebuild deadline once consecutive rebuilt endpoints -/// keep failing to register on any home relay. -const REBUILD_DEADLINE_MAX: Duration = Duration::from_secs(30 * 60); - -/// The watchdog's rebuild deadline for the next serve pass, given how many -/// endpoints in a row never registered on a home relay: the usual -/// [`relay_watchdog::RELAY_OUTAGE_REBUILD`] after an endpoint that did -/// register, doubling per unregistered endpoint up to [`REBUILD_DEADLINE_MAX`] -/// (180s, 6m, 12m, 24m, 30m). Rebuilding while the relay itself is down -/// gains nothing and drops every LAN client, so it is done less and less -/// often; a relay that comes back resets the escalation. -fn rebuild_deadline(unregistered_endpoints: u32) -> Duration { - let factor = 1u32 << unregistered_endpoints.min(4); - (relay_watchdog::RELAY_OUTAGE_REBUILD * factor).min(REBUILD_DEADLINE_MAX) -} - /// Build the ephemeral `ServerConfig` for `server start --quick`: a full-tunnel /// routed set (`routed_domains = ["*"]`, `routed_cidrs = ["0.0.0.0/0", "::/0"]`) /// plus a freshly generated in-memory identity, returned *alongside* the config — @@ -823,17 +800,9 @@ async fn run_server( .map(|q| std::collections::HashSet::from([q.client_id])) .unwrap_or_default(), }; - let endpoint = create_server_endpoint(&relay_config, secret_key.clone(), allowlists.clone()) + let endpoint = create_server_endpoint(&relay_config, secret_key, allowlists) .await .context("Failed to create iroh endpoint")?; - // The relay watchdog's remedy of last resort: a fresh endpoint with the - // same identity and allowlists (see the serve loop below). - let rebuild = server_rebuild_factory(relay_config.clone(), secret_key, allowlists); - // Only a custom-relay server hangs its reachability on one home-relay - // registration (n0 discovery is off, clients dial by relay hint), so the - // watchdog is armed for custom relays only. - let relay_watchdog_armed = relay_config.is_custom(); - log::info!("flextunnel server Node ID: {}", endpoint.id()); match &quick { Some(_) => log::info!( @@ -881,7 +850,7 @@ async fn run_server( // forever). `first_client` is `Some` exactly in quick mode; a normal server // parks here immediately, so the arm never fires. `notify_one` stores a // permit, so a client that connects before this future is first polled is - // not missed. Pinned outside the serve loop so it spans endpoint rebuilds. + // not missed. let mut grace = pin!(async { match &first_client { Some(notify) => { @@ -893,105 +862,15 @@ async fn run_server( None => std::future::pending::<()>().await, } }); - // One signal listener for the whole serve loop: re-registering it per pass - // could drop a signal delivered while an endpoint is being rebuilt. - let mut shutdown = pin!(app::shutdown_signal()); - - /// How one pass of the serve loop ended. - enum Pass { - /// The server is done (clean or failed): close the endpoint and return. - Exit(Result<()>), - /// The relay watchdog gave up on the endpoint. - Rebuild(RelayOutage), - } - - // Serve loop. A pass serves on the current endpoint until the server ends, - // a shutdown signal arrives, the quick-mode grace expires, or — custom - // relays only — the relay watchdog reports the endpoint has lost its home - // relay for good. That last case is the in-process equivalent of the - // process restart known to fix it: close the wedged endpoint, bind a fresh - // one with the same identity, and serve again. The `ProxyServer` (its - // registries, blocklist, status state) carries over; the old endpoint's - // connections and bridge tasks end with it. - // - // A rebuild only helps when iroh's relay bookkeeping went stale. When the - // relay itself is unreachable the fresh endpoint never registers either, - // and rebuilding it again every few minutes would keep dropping the LAN - // clients that still work. So consecutive endpoints that never saw a home - // relay lengthen the watchdog's deadline (`rebuild_deadline`); one that - // did register resets the escalation. - let mut endpoint = endpoint; - let mut unregistered_endpoints: u32 = 0; - let res = loop { - let pass = { - let run = Arc::clone(&server).run(&endpoint); - let deadline = rebuild_deadline(unregistered_endpoints); - let outage = async { - if relay_watchdog_armed { - relay_watchdog::watch_home_relay(&endpoint, deadline).await - } else { - std::future::pending().await - } - }; - tokio::select! { - res = run => Pass::Exit(res.map_err(|e| anyhow::anyhow!("Server error: {e}"))), - sig = &mut shutdown => Pass::Exit(sig.map(|()| { - log::info!("Received shutdown signal, stopping server"); - })), - _ = &mut grace => { - log::warn!("Quick mode: no client connected within 5 minutes — exiting"); - Pass::Exit(Ok(())) - } - outage = outage => Pass::Rebuild(outage), - } - }; - let outage = match pass { - Pass::Exit(res) => break res, - Pass::Rebuild(outage) => outage, - }; - - unregistered_endpoints = if outage.relay_seen { 0 } else { unregistered_endpoints + 1 }; - log::error!( - "No connected home relay for {:.0}s despite a network re-check; rebuilding the \ - endpoint from scratch (server id stays {})", - outage.duration.as_secs_f64(), - endpoint.id() - ); - if unregistered_endpoints > 0 { - log::error!( - "{unregistered_endpoints} endpoint(s) in a row never registered on any home \ - relay; the relay itself is probably unreachable. If the rebuilt endpoint does \ - not register either, the next rebuild waits {}s", - rebuild_deadline(unregistered_endpoints).as_secs() - ); + let res = tokio::select! { + res = Arc::clone(&server).run(&endpoint) => res.map_err(|e| anyhow::anyhow!("Server error: {e}")), + sig = app::shutdown_signal() => sig.map(|()| { + log::info!("Received shutdown signal, stopping server"); + }), + _ = &mut grace => { + log::warn!("Quick mode: no client connected within 5 minutes — exiting"); + Ok(()) } - close_endpoint_or_exit(&endpoint).await; - endpoint = loop { - match rebuild().await { - Ok(fresh) => break fresh, - Err(e) => { - log::error!( - "Endpoint rebuild failed: {e:#}; retrying in {}s", - REBUILD_RETRY.as_secs() - ); - // Nothing is bound while waiting here, so both exits - // below return directly: there is no endpoint to close. - tokio::select! { - _ = tokio::time::sleep(REBUILD_RETRY) => {} - sig = &mut shutdown => { - sig?; - log::info!("Received shutdown signal, stopping server"); - return Ok(()); - } - _ = &mut grace => { - log::warn!("Quick mode: no client connected within 5 minutes — exiting"); - return Ok(()); - } - } - } - } - }; - log::warn!("Endpoint rebuilt; serving again as {}", endpoint.id()); }; close_endpoint_or_exit(&endpoint).await; @@ -1025,17 +904,6 @@ mod tests { use super::*; use std::collections::HashMap; - #[test] - fn rebuild_deadline_doubles_per_unregistered_endpoint_up_to_the_cap() { - let base = relay_watchdog::RELAY_OUTAGE_REBUILD; - assert_eq!(rebuild_deadline(0), base); - assert_eq!(rebuild_deadline(1), base * 2); - assert_eq!(rebuild_deadline(2), base * 4); - assert_eq!(rebuild_deadline(3), base * 8); - assert_eq!(rebuild_deadline(4), REBUILD_DEADLINE_MAX); - assert_eq!(rebuild_deadline(50), REBUILD_DEADLINE_MAX); - } - fn forwarder(suffix: &str) -> DnsForwarder { let mut m = HashMap::new(); m.insert(suffix.to_string(), vec!["10.0.0.53".to_string()]); diff --git a/crates/flextunnel-core/src/lib.rs b/crates/flextunnel-core/src/lib.rs index 0e6e1df..335fe27 100644 --- a/crates/flextunnel-core/src/lib.rs +++ b/crates/flextunnel-core/src/lib.rs @@ -18,8 +18,7 @@ pub use iroh; // Same for the shared FlexAccess key crate (`ed25519-sec:` / `ed25519-pub:` // tokens, key files, authorized-keys parsing) that `auth` builds on, and the -// shared iroh transport crate (relay config, endpoint rebuild, relay -// watchdog) that `transport` builds on. +// shared iroh transport crate (relay config and endpoint rebuild) that `transport` builds on. pub use flexaccess_iroh; pub use flexaccess_keys; diff --git a/crates/flextunnel-core/src/proxy/bridge.rs b/crates/flextunnel-core/src/proxy/bridge.rs index 23406c1..c8cb4a8 100644 --- a/crates/flextunnel-core/src/proxy/bridge.rs +++ b/crates/flextunnel-core/src/proxy/bridge.rs @@ -106,10 +106,6 @@ impl BridgeUpstream { /// only when `endpoint` closes underneath it, failing each retry. pub async fn run(self: Arc, endpoint: Endpoint) { let name = &self.config.name; - // A previous run (on a since-closed endpoint, see the server's relay - // watchdog rebuild) may have been aborted while connected; its stale - // connection must not read as live until this run establishes its own. - *self.conn.lock().expect("bridge conn lock") = None; let mut attempt: u32 = 0; loop { if attempt > 0 { diff --git a/crates/flextunnel-core/src/proxy/server.rs b/crates/flextunnel-core/src/proxy/server.rs index f2dc022..b9f86ac 100644 --- a/crates/flextunnel-core/src/proxy/server.rs +++ b/crates/flextunnel-core/src/proxy/server.rs @@ -307,18 +307,12 @@ impl ProxyServer { } /// Accept connections until the endpoint closes or the server self-blocks. - /// - /// May be called again on a *fresh* endpoint after the previous one was - /// closed (the relay watchdog's rebuild): the registries and blocklist - /// carry over, the old endpoint's connection handlers end as its - /// connections close, and its bridge tasks are aborted when the previous - /// `run` future is dropped. pub async fn run(self: Arc, endpoint: &Endpoint) -> ProxyResult<()> { // Maintain the outbound bridge upstreams for the life of this run. The // bridging side dials out on this same server endpoint, so the TLS // identity it presents is this server's persistent id — what the target // server's allowlist matches. The tasks retry forever; owning them in a - // `JoinSet` aborts them with this future, so a rebuild never leaves + // `JoinSet` aborts them with this future, so shutdown never leaves // bridges retrying on a closed endpoint. let mut bridge_tasks = tokio::task::JoinSet::new(); for bridge in &self.bridges { diff --git a/crates/flextunnel-core/src/transport/endpoint.rs b/crates/flextunnel-core/src/transport/endpoint.rs index 4047446..268dbcd 100644 --- a/crates/flextunnel-core/src/transport/endpoint.rs +++ b/crates/flextunnel-core/src/transport/endpoint.rs @@ -118,9 +118,7 @@ fn base_builder(relay_config: &RelayConfig, publish_address: bool) -> Result EndpointFactory { - Arc::new(move || { - let relay_config = relay_config.clone(); - let secret = secret.clone(); - let allowlists = allowlists.clone(); - Box::pin(async move { - rebuild_endpoint(server_builder(&relay_config, secret, allowlists)?).await - }) - }) -} - /// A client endpoint builder. A client never publishes its address (it only /// dials out), even with a quick-mode secret bound as its identity. fn client_builder(relay_config: &RelayConfig, secret: Option) -> Result { diff --git a/docs/architecture.md b/docs/architecture.md index c9fdb7e..f5ca146 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -245,49 +245,12 @@ Implemented in `ProxyClient::run` / `handle_failure`: (`TUNNEL_RECOVERY_HOLD`), deploy-style connection holding — and only then fail with a network-unreachable reply. -## Relay watchdog (server, custom relays) - -Implemented once for every FlexAccess program in the shared -[flexaccess-iroh](https://github.com/flexaccessdev/flexaccess-iroh) crate -(`relay_watchdog`), driven here by the serve loop in the CLI's `run_server`. A custom-relay server is dialable from off the LAN only -while it is **registered on its home relay** (n0 discovery is off; clients dial -by relay hint, and a relay forwards Initials only to endpoints connected to it). -iroh has been seen to silently lose its home relay for good after a routine -relay reconnect: no dial retries, no warnings, no registration on any relay — -the server stops being reachable through the relays until the process restarts, -while LAN clients that find it over mDNS keep working and hide the outage (the -mac desktop reconnects, the iOS app times out). Observed on v1.0.3; no fix for -it is called out through v1.1.0, so the watchdog stays. - -The watchdog observes `Endpoint::home_relay_status()` and escalates like the -client's reconnect loop: - -1. no connected home relay for `RELAY_OUTAGE_NUDGE` (60s) → log a warning and - call `Endpoint::network_change()` (forces a fresh net report and relay - re-selection — enough when only the bookkeeping went stale); -2. still none at `RELAY_OUTAGE_REBUILD` (180s from the outage start) → the - serve loop closes the endpoint, binds a fresh one with the **same identity - and allowlists** (`server_rebuild_factory`: no per-relay probe, online-wait - tolerated failing), and calls `ProxyServer::run` again on it. The - `ProxyServer` — registries, blocklist, status state — carries over; the old - endpoint's connections end with it, and its bridge tasks are aborted with - the previous `run` future (they are owned by a `JoinSet` per run). A failed - rebuild is retried every `REBUILD_RETRY` (30s). - -A rebuild only helps when iroh's bookkeeping went stale; when the relay itself -is unreachable the fresh endpoint never registers either, and rebuilding again -every three minutes would keep dropping the LAN clients that still work. The -watchdog therefore reports whether the endpoint held a home relay at any point -(`RelayOutage::relay_seen`), and the serve loop doubles the rebuild deadline -for each consecutive endpoint that never did (`rebuild_deadline`: 180s, 6m, -12m, 24m, then capped at `REBUILD_DEADLINE_MAX`, 30m). An endpoint that -registers resets the escalation to the usual 180s. The 60s nudge is unaffected. - -A reconnect at any point resets the outage clock. Non-home relays are -connected on demand and dropped after a minute idle, which is normal and never -counts as an outage. With the default relays the watchdog is not armed: -reachability there rests on n0 publishing/resolution, not on one relay -registration. +## Server relay recovery + +Servers rely on iroh 1.1.x for relay reconnects and keep the same endpoint +during relay outages. The former server watchdog has been removed; see the +[relay recovery history and workaround](https://github.com/flexaccessdev/iroh-common-architecture/blob/9eacd43b80d867a8a4a76e3051237b854b4b0cd5/home-relay-watchdog.md) +if permanent loss of relay registration recurs. On every exit path both `run_server` and `run_client` call `endpoint.close().await` before the `Endpoint` drops; skipping it makes iroh tear @@ -345,10 +308,6 @@ defenses. | `HEARTBEAT_INTERVAL` | 10s | `transport/mod.rs` | | `LIVENESS_WINDOW` | 33s | `transport/mod.rs` | | `RELAY_CONNECT_TIMEOUT` (`endpoint.online()`) | 10s | flexaccess-iroh `relay` | -| `RELAY_OUTAGE_NUDGE` (server relay watchdog) | 60s | flexaccess-iroh `relay_watchdog` | -| `RELAY_OUTAGE_REBUILD` (server relay watchdog, default deadline) | 180s | flexaccess-iroh `relay_watchdog` | -| `REBUILD_DEADLINE_MAX` (server relay watchdog, escalated deadline cap) | 30m | `flextunnel-cli/src/main.rs` | -| `REBUILD_RETRY` (server endpoint rebuild) | 30s | `flextunnel-cli/src/main.rs` | | `CONNECT_TIMEOUT` (client server connect) | 30s | `proxy/client.rs` | | `HANDSHAKE_TIMEOUT` | 10s | `proxy/client.rs`, `proxy/server.rs`, `proxy/bridge.rs` | | `LOCAL_HANDSHAKE_TIMEOUT` | 10s | `proxy/client.rs` |