Skip to content

Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tnull merged 2 commits into from
Mar 26, 2026
Merged

Paginated sqlite #807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7bb5852
prefactor: move v1->v2 migration to separate function
benthecarman Feb 25, 2026
9b1da60
Add PaginatedKVStore support to SqliteStore
benthecarman Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down
277 changes: 228 additions & 49 deletions src/io/sqlite_store/migrations.rs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
assert!(from_version < to_version);
if from_version == 1 && to_version == 2 {
let tx = connection.transaction().map_err(|e| {
let msg = format!(
"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
if from_version == 1 && to_version >= 2 {
migrate_v1_to_v2(connection, kv_table_name)?;
from_version = 2;
}
if from_version == 2 && to_version >= 3 {
migrate_v2_to_v3(connection, kv_table_name)?;
}

Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
let tx = connection.transaction().map_err(|e| {
let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
})?;

// Rename 'namespace' column to 'primary_namespace'
let sql = format!(
"ALTER TABLE {}
RENAME COLUMN namespace TO primary_namespace;",
kv_table_name
);

tx.execute(&sql, []).map_err(|e| {
let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
})?;

// Add new 'secondary_namespace' column
let sql = format!(
"ALTER TABLE {}
ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);

tx.execute(&sql, []).map_err(|e| {
let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
})?;

// Update user_version
tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
|e| {
let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
io::Error::new(io::ErrorKind::Other, msg)
})?;
},
)?;

tx.commit().map_err(|e| {
let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
})?;

Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
let map_err = |e: rusqlite::Error| -> io::Error {
let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
};

// Check whether the primary key already includes secondary_namespace.
// Tables migrated from v1 have PK (primary_namespace, key) only — missing

@tnulltnullFeb 25, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have

// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);

in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?

@benthecarmanbenthecarmanFeb 25, 2026•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner

@tnulltnullMar 2, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused

This rewrites the table to make it a part of the primary key

But it doesn't?

 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?

@benthecarmanbenthecarmanMar 3, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).

This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates

@tnulltnullMar 25, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.

AFAICT you should be able to use PRAGMA table_info(table_name) and check the pk column.

@benthecarmanbenthecarmanMar 25, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!

// secondary_namespace. Tables created directly as v2 already have the correct
// PK (primary_namespace, secondary_namespace, key).
let needs_table_rewrite = {
let sql = format!("PRAGMA table_info({})", kv_table_name);
let mut stmt = connection.prepare(&sql).map_err(map_err)?;
let mut pk_cols: Vec<(i64, String)> = stmt
.query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
.map_err(map_err)?
.collect::<Result<Vec<_>, _>>()
.map_err(map_err)?
.into_iter()
.filter(|(pk, _)| *pk > 0)
.collect();
pk_cols.sort_by_key(|(pk, _)| *pk);
let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
};

let tx = connection.transaction().map_err(|e| {
let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
})?;

if needs_table_rewrite {
// Full table rewrite to fix the primary key.
let old_table = format!("{}_v2_old", kv_table_name);

let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
tx.execute(&rename_sql, []).map_err(map_err)?;

// Rename 'namespace' column to 'primary_namespace'
let sql = format!(
"ALTER TABLE {}
RENAME COLUMN namespace TO primary_namespace;",
let create_table_sql = format!(
"CREATE TABLE {} (
primary_namespace TEXT NOT NULL,
secondary_namespace TEXT DEFAULT \"\" NOT NULL,
key TEXT NOT NULL CHECK (key <> ''),
value BLOB,
sort_order INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (primary_namespace, secondary_namespace, key)
)",
kv_table_name
);
tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

// Add new 'secondary_namespace' column
let sql = format!(
"ALTER TABLE {}
ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
let drop_old_sql = format!("DROP TABLE {}", old_table);
tx.execute(&drop_old_sql, []).map_err(map_err)?;
} else {
// Primary key is already correct — just add the sort_order column and backfill.
let add_col_sql = format!(
"ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
kv_table_name
);
tx.execute(&add_col_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
io::Error::new(io::ErrorKind::Other, msg)
})?;

// Update user_version
tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
.map_err(|e| {
let msg = format!(
"Failed to upgrade user_version from {} to {}: {}",
from_version, to_version, e
);
io::Error::new(io::ErrorKind::Other, msg)
})?;
let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
tx.execute(&backfill_sql, []).map_err(map_err)?;
}

tx.commit().map_err(|e| {
let msg = format!(
"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
// Create composite index for paginated listing
let sql = format!(
"CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
kv_table_name, kv_table_name
);
tx.execute(&sql, []).map_err(map_err)?;

// Update user_version
tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
|e| {
let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
io::Error::new(io::ErrorKind::Other, msg)
})?;
}
},
)?;

tx.commit().map_err(|e| {
let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
io::Error::new(io::ErrorKind::Other, msg)
})?;

Ok(())
}

#[cfg(test)]
mod tests {
use std::fs;

use lightning::util::persist::KVStoreSync;
use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
use rusqlite::{named_params, Connection};

use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
let sql = format!(
"INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
kv_table_name
);
);
let mut stmt = connection.prepare_cached(&sql).unwrap();

stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
// Check we can continue to use the store just fine.
do_read_write_remove_list_persist(&store);
}

#[test]
fn rwrl_post_schema_2_migration() {
let old_schema_version = 2u16;

let mut temp_path = random_storage_path();
temp_path.push("rwrl_post_schema_2_migration");

let db_file_name = "test_db".to_string();
let kv_table_name = "test_table".to_string();

let test_ns = "testspace";
let test_sub = "testsub";

{
// Create a v2 database manually
fs::create_dir_all(temp_path.clone()).unwrap();
let mut db_file_path = temp_path.clone();
db_file_path.push(db_file_name.clone());

let connection = Connection::open(db_file_path.clone()).unwrap();

connection
.pragma(
Some(rusqlite::DatabaseName::Main),
"user_version",
old_schema_version,
|_| Ok(()),
)
.unwrap();

let sql = format!(
"CREATE TABLE IF NOT EXISTS {} (
primary_namespace TEXT NOT NULL,
secondary_namespace TEXT DEFAULT \"\" NOT NULL,
key TEXT NOT NULL CHECK (key <> ''),
value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
);",
kv_table_name
);
connection.execute(&sql, []).unwrap();

// Insert 3 rows in a known order
for i in 0..3 {
let key = format!("key_{}", i);
let sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
kv_table_name
);
let mut stmt = connection.prepare_cached(&sql).unwrap();
stmt.execute(named_params! {
":ns": test_ns,
":sub": test_sub,
":key": key,
":value": vec![i as u8; 8],
})
.unwrap();
}
}

// Open with new code, triggering v2→v3 migration
let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

// Verify data survived
for i in 0..3 {
let key = format!("key_{}", i);
let data = store.read(test_ns, test_sub, &key).unwrap();
assert_eq!(data, vec![i as u8; 8]);
}

// Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
let response =
PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
assert_eq!(response.keys.len(), 3);
// ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

// Verify we can write new entries and they get proper ordering
KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
let response =
PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
assert_eq!(response.keys[0], "key_new");

// Check we can continue to use the store just fine.
do_read_write_remove_list_persist(&store);
}
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
Paginated sqlite by benthecarman · Pull Request #807 · lightningdevkit/ldk-node · GitHub
Skip to content
Paginated sqlite#807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

tnull
merged 2 commits into

from

Mar 26, 2026
 Merged


Paginated sqlite
#807
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
7bb5852

prefactor: move v1->v2 migration to separate function


benthecarman Feb 25, 2026
9b1da60

Add PaginatedKVStore support to SqliteStore


benthecarman Feb 24, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
Cargo.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -91,6 +91,7 @@ proptest = "1.0.0"
regex = "1.5.6"
criterion = { version = "0.7.0", features = ["async_tokio"] }
ldk-node-062 = { package = "ldk-node", version = "=0.6.2" }
ldk-node-070 = { package = "ldk-node", version = "=0.7.0" }

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
Expand Down

277 changes: 228 additions & 49 deletions
src/io/sqlite_store/migrations.rs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,74 +9,167 @@ use lightning::io;
use rusqlite::Connection;

pub(super) fn migrate_schema(
	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
	connection: &mut Connection, kv_table_name: &str, mut from_version: u16, to_version: u16,
) -> io::Result<()> {
	assert!(from_version < to_version);
	if from_version == 1 && to_version == 2 {
 let tx = connection.transaction().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	if from_version == 1 && to_version >= 2 {
 migrate_v1_to_v2(connection, kv_table_name)?;
 from_version = 2;
	}
	if from_version == 2 && to_version >= 3 {
 migrate_v2_to_v3(connection, kv_table_name)?;
	}

	Ok(())
}

fn migrate_v1_to_v2(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Rename 'namespace' column to 'primary_namespace'
	let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Add new 'secondary_namespace' column
	let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 kv_table_name
	);

	tx.execute(&sql, []).map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 2u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 1 to 2: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v1 to v2: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

fn migrate_v2_to_v3(connection: &mut Connection, kv_table_name: &str) -> io::Result<()> {
	let map_err = |e: rusqlite::Error| -> io::Error {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	};

	// Check whether the primary key already includes secondary_namespace.
	// Tables migrated from v1 have PK (primary_namespace, key) only — missing
@tnulltnullFeb 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh, I'm confused, why is this true? We explictly have
// Add new 'secondary_namespace' columnlet sql = format!("ALTER TABLE {} ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
kv_table_name
);
in migrate_v1_to_v2? Am I missing something, or is this Claude hallucinating?
@benthecarmanbenthecarmanFeb 25, 2026•

edited

Loading

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v1 to v2 is added but its not a part of the primary key so we can't use ON CONFLICT on writes for just updating the value. Before we could use replace but now we want to keep the created_at date. This rewrites the table to make it a part of the primary key, otherwise we need to do a query inside to look up the created_at date when replacing. Could add a unique index instead but this felt cleaner
@tnulltnullMar 2, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still confused
This rewrites the table to make it a part of the primary key
But it doesn't?
 PRIMARY KEY (primary_namespace, secondary_namespace, key)

And arguably created_at really shouldn't be part of the key?
@benthecarmanbenthecarmanMar 3, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the v1->v2 migration added secondary_namespace but didn't update the primary key, so databases that started with v1 had PK (primary_namespace, key) instead of PK (primary_namespace, secondary_namespace, key).
This issue didn't cause problems because we used INSERT OR REPLACE so we'd silently overwrite rows without errors (besides potentially losing data). However now we want to use ON CONFLICT so we don't replace the created_at and just update the value. This however now breaks for databases that started in v1 because we'll get conflicts even when the secondary namespace is different because the primary namespace and key are the primary key. So instead of silently replacing data, we now actually error because we're trying to do the ON CONFLICT updates
@tnulltnullMar 25, 2026

Copy link


Copy Markdown

Collaborator
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, right. But can we check whether we already have the right primary key, before we initiate a potentially costly/laggy migration? Node that still have the v1 schema would need to have been upgraded since LDK Node v0.2. I doubt there would be (m)any nodes requiring this migration at all, given this is such an early version that almost nobody ran in production.
AFAICT you should be able to use  PRAGMA table_info(table_name) and check the pk column.
@benthecarmanbenthecarmanMar 25, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done!
	// secondary_namespace. Tables created directly as v2 already have the correct
	// PK (primary_namespace, secondary_namespace, key).
	let needs_table_rewrite = {
 let sql = format!("PRAGMA table_info({})", kv_table_name);
 let mut stmt = connection.prepare(&sql).map_err(map_err)?;
 let mut pk_cols: Vec<(i64, String)> = stmt
 .query_map([], |row| Ok((row.get::<_, i64>(5)?, row.get::<_, String>(1)?)))
 .map_err(map_err)?
 .collect::<Result<Vec<_>, _>>()
 .map_err(map_err)?
 .into_iter()
 .filter(|(pk, _)| *pk > 0)
 .collect();
 pk_cols.sort_by_key(|(pk, _)| *pk);
 let pk_names: Vec<&str> = pk_cols.iter().map(|(_, name)| name.as_str()).collect();
 pk_names != vec!["primary_namespace", "secondary_namespace", "key"]
	};

	let tx = connection.transaction().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	if needs_table_rewrite {
 // Full table rewrite to fix the primary key.
 let old_table = format!("{}_v2_old", kv_table_name);

 let rename_sql = format!("ALTER TABLE {} RENAME TO {}", kv_table_name, old_table);
 tx.execute(&rename_sql, []).map_err(map_err)?;

 // Rename 'namespace' column to 'primary_namespace'
 let sql = format!(
 "ALTER TABLE {}
 RENAME COLUMN namespace TO primary_namespace;",
 let create_table_sql = format!(
 "CREATE TABLE {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB,
 sort_order INTEGER NOT NULL DEFAULT 0,
 PRIMARY KEY (primary_namespace, secondary_namespace, key)
 )",
 kv_table_name
 );
 tx.execute(&create_table_sql, []).map_err(map_err)?;

tx.execute(&sql, []).map_err(|e| {
let msg = format!(
	"Failed to migrate table {} from user_version {} to {}: {}",
kv_table_name, from_version, to_version, e
);
	io::Error::new(io::ErrorKind::Other, msg)
})?;
// Copy data and backfill sort_order from ROWID for relative ordering
 let copy_sql = format!(
"INSERT INTO {} (primary_namespace, secondary_namespace, key, value, sort_order)
SELECT primary_namespace, secondary_namespace, key, value, ROWID FROM {}",
kv_table_name, old_table
);
tx.execute(&copy_sql, []).map_err(map_err)?;

 // Add new 'secondary_namespace' column
 let sql = format!(
 "ALTER TABLE {}
 ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
 let drop_old_sql = format!("DROP TABLE {}", old_table);
 tx.execute(&drop_old_sql, []).map_err(map_err)?;
	} else {
 // Primary key is already correct — just add the sort_order column and backfill.
 let add_col_sql = format!(
 "ALTER TABLE {} ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0",
 kv_table_name
 );
 tx.execute(&add_col_sql, []).map_err(map_err)?;

 tx.execute(&sql, []).map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;

 // Update user_version
 tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
 .map_err(|e| {
 let msg = format!(
 "Failed to upgrade user_version from {} to {}: {}",
 from_version, to_version, e
 );
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
 let backfill_sql = format!("UPDATE {} SET sort_order = ROWID", kv_table_name);
 tx.execute(&backfill_sql, []).map_err(map_err)?;
	}

 tx.commit().map_err(|e| {
 let msg = format!(
 "Failed to migrate table {} from user_version {} to {}: {}",
 kv_table_name, from_version, to_version, e
 );
	// Create composite index for paginated listing
	let sql = format!(
 "CREATE INDEX idx_{}_paginated ON {} (primary_namespace, secondary_namespace, sort_order DESC, key ASC)",
 kv_table_name, kv_table_name
	);
	tx.execute(&sql, []).map_err(map_err)?;

	// Update user_version
	tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", 3u16, |_| Ok(())).map_err(
 |e| {
 let msg = format!("Failed to upgrade user_version from 2 to 3: {}", e);
 io::Error::new(io::ErrorKind::Other, msg)
 })?;
	}
 },
	)?;

	tx.commit().map_err(|e| {
 let msg = format!("Failed to migrate table {} from v2 to v3: {}", kv_table_name, e);
 io::Error::new(io::ErrorKind::Other, msg)
	})?;

	Ok(())
}

#[cfg(test)]
mod tests {
	use std::fs;

	use lightning::util::persist::KVStoreSync;
	use lightning::util::persist::{KVStoreSync, PaginatedKVStoreSync};
	use rusqlite::{named_params, Connection};

	use crate::io::sqlite_store::SqliteStore;
Expand DownExpand Up@@ -128,7 +221,7 @@ mod tests {
 let sql = format!(
 "INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
 kv_table_name
);
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();

 stmt.execute(named_params! {
Expand DownExpand Up@@ -166,4 +259,90 @@ mod tests {
 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}

	#[test]
	fn rwrl_post_schema_2_migration() {
 let old_schema_version = 2u16;

 let mut temp_path = random_storage_path();
 temp_path.push("rwrl_post_schema_2_migration");

 let db_file_name = "test_db".to_string();
 let kv_table_name = "test_table".to_string();

 let test_ns = "testspace";
 let test_sub = "testsub";

 {
 // Create a v2 database manually
 fs::create_dir_all(temp_path.clone()).unwrap();
 let mut db_file_path = temp_path.clone();
 db_file_path.push(db_file_name.clone());

 let connection = Connection::open(db_file_path.clone()).unwrap();

 connection
 .pragma(
 Some(rusqlite::DatabaseName::Main),
 "user_version",
 old_schema_version,
 |_| Ok(()),
 )
 .unwrap();

 let sql = format!(
 "CREATE TABLE IF NOT EXISTS {} (
 primary_namespace TEXT NOT NULL,
 secondary_namespace TEXT DEFAULT \"\" NOT NULL,
 key TEXT NOT NULL CHECK (key <> ''),
 value BLOB, PRIMARY KEY ( primary_namespace, secondary_namespace, key )
 );",
 kv_table_name
 );
 connection.execute(&sql, []).unwrap();

 // Insert 3 rows in a known order
 for i in 0..3 {
 let key = format!("key_{}", i);
 let sql = format!(
 "INSERT INTO {} (primary_namespace, secondary_namespace, key, value) VALUES (:ns, :sub, :key, :value);",
 kv_table_name
 );
 let mut stmt = connection.prepare_cached(&sql).unwrap();
 stmt.execute(named_params! {
 ":ns": test_ns,
 ":sub": test_sub,
 ":key": key,
 ":value": vec![i as u8; 8],
 })
 .unwrap();
 }
 }

 // Open with new code, triggering v2→v3 migration
 let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();

 // Verify data survived
 for i in 0..3 {
 let key = format!("key_{}", i);
 let data = store.read(test_ns, test_sub, &key).unwrap();
 assert_eq!(data, vec![i as u8; 8]);
 }

 // Verify paginated listing works and returns entries in ROWID-backfilled order (newest first)
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys.len(), 3);
 // ROWIDs were 1, 2, 3 so sort_order was backfilled as 1, 2, 3; newest first
 assert_eq!(response.keys, vec!["key_2", "key_1", "key_0"]);

 // Verify we can write new entries and they get proper ordering
 KVStoreSync::write(&store, test_ns, test_sub, "key_new", vec![99u8; 8]).unwrap();
 let response =
 PaginatedKVStoreSync::list_paginated(&store, test_ns, test_sub, None).unwrap();
 assert_eq!(response.keys[0], "key_new");

 // Check we can continue to use the store just fine.
 do_read_write_remove_list_persist(&store);
	}
}
Loading
Loading