Add cross-store ordering and recovery to tiered storage

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();

.github/workflows/rust.yml

@@ -86,6 +86,14 @@ jobs:
  if: "matrix.platform != 'windows-latest'"
  run: |
  RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test
+ - name: Test tiered storage
+ if: matrix.check-fmt
+ run: |
+ cargo test --lib --features storage-tier io::tier_store
+ RUSTFLAGS="--cfg no_download" cargo test \
+ --features storage-tier \
+ --test integration_tests_rust \
+ builder_configures_sqlite_backup_store
  - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
  if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
  run: |

Cargo.toml

@@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [package.metadata.docs.rs]
-features = ["storage-postgres-vendored-tls"]
+features = ["storage-postgres-vendored-tls", "storage-tier"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [lib]
@@ -53,6 +53,7 @@ chain-electrum = [
 ]
 chain-bitcoind = ["dep:lightning-block-sync"]
 storage-sqlite = ["dep:rusqlite"]
+storage-tier = ["storage-sqlite"]
 storage-filesystem = ["dep:lightning-persister"]
 storage-vss = ["dep:vss-client", "dep:prost"]
 storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"]

README.md

@@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 | `chain-electrum` | Electrum chain source |
 | `chain-bitcoind` | Bitcoin Core RPC and REST chain source |
 | `storage-sqlite` | SQLite storage |
+| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores |
 | `storage-filesystem` | Filesystem storage |
 | `storage-vss` | Versioned Storage Service storage |
 | `storage-postgres` | PostgreSQL storage |
@@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide:
 
 The `default` feature set preserves the native Rust API's previous behavior. It enables all three
 chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI
-remain opt-in. Every build must enable at least one chain source feature.
+remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables
+`storage-sqlite`. Every build must enable at least one chain source feature.
 
 On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL
 development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build

src/builder.rs

@@ -11,7 +11,7 @@ use std::convert::TryInto;
 use std::default::Default;
 #[cfg(feature = "unified-payments")]
 use std::net::ToSocketAddrs;
-#[cfg(feature = "storage-filesystem")]
+#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))]
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, Once, RwLock};
 use std::time::SystemTime;
@@ -72,6 +72,8 @@ use crate::gossip::GossipSource;
 use crate::io::fs_store::open_or_migrate_fs_store;
 #[cfg(feature = "storage-sqlite")]
 use crate::io::sqlite_store::SqliteStore;
+#[cfg(feature = "storage-tier")]
+use crate::io::tier_store::{setup_index_store, TierStore};
 use crate::io::utils::{
 	read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache,
 	read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info,
@@ -173,6 +175,13 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[cfg(feature = "storage-tier")]
+#[derive(Default, Debug)]
+struct TierStoreConfig {
+	ephemeral_storage_dir_path: Option<PathBuf>,
+	backup_storage_dir_path: Option<PathBuf>,
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -326,6 +335,8 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	#[cfg(feature = "storage-tier")]
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 	probing_config: Option<ProbingConfig>,
@@ -347,6 +358,8 @@ impl NodeBuilder {
  let gossip_source_config = None;
  let liquidity_source_config = None;
  let log_writer_config = None;
+ #[cfg(feature = "storage-tier")]
+ let tier_store_config = None;
  let runtime_handle = None;
  let pathfinding_scores_sync_config = None;
  let probing_config = None;
@@ -356,6 +369,8 @@ impl NodeBuilder {
  gossip_source_config,
  liquidity_source_config,
  log_writer_config,
+ #[cfg(feature = "storage-tier")]
+ tier_store_config,
  runtime_handle,
  async_payments_role: None,
  pathfinding_scores_sync_config,
@@ -686,6 +701,41 @@ impl NodeBuilder {
  self
 	}
 
+	/// Configures a local SQLite backup store for disaster recovery.
+	///
+	/// When building with tiered storage, a SQLite store will be created at the
+	/// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database
+	/// file name. It receives a second durable copy of data written to the
+	/// primary store.
+	///
+	/// Writes and removals for primary-backed data only succeed once both the
+	/// primary and backup SQLite stores complete successfully.
+	///
+	/// If not set, durable data will be stored only in the primary store.
+	///
+	/// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into());
+ self
+	}
+
+	/// Configures the ephemeral storage directory path for non-critical, frequently-accessed data.
+	///
+	/// When set, a local SQLite store is created at this path for ephemeral data like
+	/// the network graph and scorer. Data stored here can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))]
+	pub fn set_ephemeral_storage_dir_path(
+ &mut self, ephemeral_storage_dir_path: String,
+	) -> &mut Self {
+ let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+ tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into());
+ self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	#[cfg(feature = "storage-sqlite")]
@@ -901,11 +951,18 @@ impl NodeBuilder {
 	}
 
 	/// Builds a [`Node`] instance according to the options previously configured.
+	///
+	/// The provided `kv_store` will be used as the primary storage backend. Optionally,
+	/// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer)
+	/// and a local SQLite backup store for disaster recovery can be configured via
+	/// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`].
+	///
+	/// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path
+	/// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path
 	pub fn build_with_store<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S,
 	) -> Result<Node, BuildError> {
  let logger = setup_logger(&self.log_writer_config, &self.config)?;
-
  self.build_with_store_and_logger(node_entropy, kv_store, logger)
 	}
 
@@ -930,6 +987,53 @@ impl NodeBuilder {
 	fn build_with_store_runtime_and_logger<S: PaginatedKVStore + Send + Sync + 'static>(
  &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc<Runtime>, logger: Arc<Logger>,
 	) -> Result<Node, BuildError> {
+ #[cfg(feature = "storage-tier")]
+ let store: Arc<DynStore> = {
+ let ts_config = self.tier_store_config.as_ref();
+ let primary_store = Arc::new(DynStoreWrapper(kv_store));
+ let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger));
+ if let Some(config) = ts_config {
+ if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref()
+ {
+ let index_store = runtime
+ .block_on(setup_index_store(self.config.storage_dir_path.clone().into()))
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup tier-store index: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store = SqliteStore::new(
+ ephemeral_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let ephemeral_store: Arc<DynStore> = Arc::new(DynStoreWrapper(ephemeral_store));
+ tier_store.set_index_store(index_store);
+ tier_store.set_ephemeral_store(ephemeral_store);
+ }
+
+ if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() {
+ let backup_store = SqliteStore::new(
+ backup_storage_dir_path.clone(),
+ Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()),
+ Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+ )
+ .map_err(|e| {
+ log_error!(logger, "Failed to setup backup SQLite store: {}", e);
+ BuildError::KVStoreSetupFailed
+ })?;
+ let backup_store: Arc<DynStore> = Arc::new(DynStoreWrapper(backup_store));
+ tier_store.set_backup_store(backup_store);
+ }
+ }
+ Arc::new(DynStoreWrapper(tier_store))
+ };
+ #[cfg(not(feature = "storage-tier"))]
+ let store: Arc<DynStore> = Arc::new(DynStoreWrapper(kv_store));
+
  let seed_bytes = node_entropy.to_seed_bytes();
  let config = Arc::new(self.config.clone());
 
@@ -944,7 +1048,7 @@ impl NodeBuilder {
  seed_bytes,
  runtime,
  logger,
-Arc::new(DynStoreWrapper(kv_store)),
+store,
  )
 	}
 }

src/io/mod.rs

@@ -15,6 +15,8 @@ pub mod postgres_store;
 pub mod sqlite_store;
 #[cfg(test)]
 pub(crate) mod test_utils;
+#[cfg(feature = "storage-tier")]
+pub(crate) mod tier_store;
 pub(crate) mod utils;
 #[cfg(feature = "storage-vss")]
 pub mod vss_store;

src/io/sqlite_store/mod.rs

@@ -12,12 +12,14 @@ use std::future::Future;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use lightning::io;
 use lightning::util::persist::{
 	KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse,
 };
 use lightning_types::string::PrintableString;
+use rusqlite::ffi::ErrorCode;
 use rusqlite::{named_params, Connection};
 
 use crate::io::utils::check_namespace_key_validity;
@@ -26,6 +28,15 @@ mod migrations;
 
 /// LDK Node's database file name.
 pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite";
+/// LDK Node's internal tier-store index database file name.
+#[cfg(feature = "storage-tier")]
+pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite";
+/// LDK Node's backup database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite";
+/// LDK Node's ephemeral database file name.
+#[cfg(feature = "storage-tier")]
+pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite";
 /// LDK Node's table in which we store all data.
 pub const KV_TABLE_NAME: &str = "ldk_node_data";
 
@@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3;
 // The number of entries returned per page in paginated list operations.
 const PAGE_SIZE: usize = 50;
 
+fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind {
+	match error {
+ rusqlite::Error::SqliteFailure(error, _)
+ if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) =>
+ {
+ io::ErrorKind::AlreadyExists
+ },
+ _ => io::ErrorKind::Other,
+	}
+}
+
 /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database.
 ///
 /// [SQLite]: https://sqlite.org
@@ -62,7 +84,23 @@ impl SqliteStore {
 	pub fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
 	) -> io::Result<Self> {
- let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?);
+ Self::new_internal(data_dir, db_file_name, kv_table_name, false)
+	}
+
+	/// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime.
+	#[cfg(feature = "storage-tier")]
+	pub(crate) fn new_exclusive(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+	) -> io::Result<Self> {
+ Self::new_internal(data_dir, db_file_name, kv_table_name, true)
+	}
+
+	fn new_internal(
+ data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
+	) -> io::Result<Self> {
+ let inner =
+ Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?);
 
  let next_write_version = AtomicU64::new(1);
  Ok(Self { inner, next_write_version })
@@ -230,6 +268,7 @@ struct SqliteStoreInner {
 impl SqliteStoreInner {
 	fn new(
  data_dir: PathBuf, db_file_name: Option<String>, kv_table_name: Option<String>,
+ exclusive: bool,
 	) -> io::Result<Self> {
  let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string());
  let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string());
@@ -251,6 +290,33 @@ impl SqliteStoreInner {
  io::Error::new(io::ErrorKind::Other, msg)
  })?;
 
+ if exclusive {
+ connection.busy_timeout(Duration::ZERO).map_err(|e| {
+ let msg = format!(
+ "Failed to configure exclusive database lock timeout for {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(io::ErrorKind::Other, msg)
+ })?;
+ connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| {
+ let msg = format!(
+ "Failed to exclusively lock database file {}: {}",
+ db_file_path.display(),
+ e
+ );
+ io::Error::new(exclusive_lock_error_kind(&e), msg)
+ })?;
+ }
+
  let sql = format!("SELECT user_version FROM pragma_user_version");
  let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| {
  let msg = format!("Failed to read PRAGMA user_version: {}", e);
@@ -700,6 +766,21 @@ mod tests {
  }
 	}
 
+	#[test]
+	fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() {
+ for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
+ let error =
+ rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None);
+ assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists);
+ }
+
+ let io_error = rusqlite::Error::SqliteFailure(
+ rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR),
+ None,
+ );
+ assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other);
+	}
+
 	#[tokio::test]
 	async fn read_write_remove_list_persist() {
  let mut temp_path = random_storage_path();