Skip to content

Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kdush wants to merge 9 commits into
base:main
Choose a base branch
from
Open

Add SQLite-backed registry with JSON migration support #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
280fce1
feat: add SQLite-backed registry
kdush Apr 11, 2026
1397f7f
feat: add SQLite backend and migration tests
kdush Apr 11, 2026
ed49b0a
docs: document storage backend and migration
kdush Apr 11, 2026
3be4233
fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json
kdush May 13, 2026
8cb4522
fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口
kdush May 13, 2026
3451b79
fix(state): 🐛 对齐 SQLite registry 评审语义
kdush May 23, 2026
e2260a0
chore(merge): 🔧 合并上游 main 解决 PR 冲突
kdush May 23, 2026
ad7d146
Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…
kdush May 30, 2026
9e4a56f
Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…
kdush Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down
51 changes: 27 additions & 24 deletions openkb/cli.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
retry without re-downloading.
"""
from openkb.agent.compiler import compile_long_doc, compile_short_doc
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)
openkb_dir = kb_dir / ".openkb"
config = load_config(openkb_dir / "config.yaml")
_setup_llm_key(kb_dir)
model: str = config.get("model", DEFAULT_CONFIG["model"])
registry = HashRegistry(openkb_dir / "hashes.json")
backend = config.get("storage_backend", "sqlite")
registry = get_registry(openkb_dir, backend=backend)

# 2. Convert document
click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
"model": model,
"language": language,
"pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
"storage_backend": DEFAULT_CONFIG["storage_backend"],
}
save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

# Write API key to KB-local .env (0600) if the user provided one
if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
scan_affected_pages,
)
from openkb.lint import fix_broken_links
from openkb.state import HashRegistry
from openkb.state import get_registry

kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
if kb_dir is None:
click.echo("No knowledge base found. Run `openkb init` first.")
return

openkb_dir = kb_dir / ".openkb"
registry = HashRegistry(openkb_dir / "hashes.json")
config = load_config(openkb_dir / "config.yaml")
backend = config.get("storage_backend", "sqlite")
registry = get_registry(openkb_dir, backend=backend)

matches = _resolve_doc_identifier(registry, identifier)
if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
"""
from openkb.lint import run_structural_lint
from openkb.agent.linter import run_knowledge_lint
from openkb.state import get_registry

openkb_dir = kb_dir / ".openkb"

# Skip lint entirely when the KB has no indexed documents
hashes_file = openkb_dir / "hashes.json"
if hashes_file.exists():
hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
else:
hashes = {}
config = load_config(openkb_dir / "config.yaml")
backend: str = config.get("storage_backend", "sqlite")
registry = get_registry(openkb_dir, backend=backend)
hashes = registry.all_entries()
if not hashes:
click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
return

config = load_config(openkb_dir / "config.yaml")
_setup_llm_key(kb_dir)
model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
"""Print all documents in the knowledge base. Usable from CLI and chat REPL."""
openkb_dir = kb_dir / ".openkb"
hashes_file = openkb_dir / "hashes.json"
if not hashes_file.exists():
click.echo("No documents indexed yet.")
return
from openkb.state import get_registry

hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
openkb_dir = kb_dir / ".openkb"
config = load_config(openkb_dir / "config.yaml")
backend = config.get("storage_backend", "sqlite")
registry = get_registry(openkb_dir, backend=backend)
hashes = registry.all_entries()
if not hashes:
click.echo("No documents indexed yet.")
return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
click.echo(f" {'raw':<20} {raw_count:<10}")

# Hash registry summary
from openkb.state import get_registry

openkb_dir = kb_dir / ".openkb"
hashes_file = openkb_dir / "hashes.json"
if hashes_file.exists():
hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
click.echo(f"\n Total indexed: {len(hashes)} document(s)")
config = load_config(openkb_dir / "config.yaml")
backend = config.get("storage_backend", "sqlite")
registry = get_registry(openkb_dir, backend=backend)
hashes = registry.all_entries()
click.echo(f"\n Total indexed: {len(hashes)} document(s)")

# Last compile time: newest compiled page across summaries/, concepts/,
# and entities/ (an entity-only compile must still bump the shown time).
Expand Down
1 change: 1 addition & 0 deletions openkb/config.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
"model": "gpt-5.4-mini",
"language": "en",
"pageindex_threshold": 20,
"storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down
7 changes: 4 additions & 3 deletions openkb/converter.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
openkb_dir = kb_dir / ".openkb"
config = load_config(openkb_dir / "config.yaml")
threshold: int = config.get("pageindex_threshold", 20)
registry = HashRegistry(openkb_dir / "hashes.json")
backend = config.get("storage_backend", "sqlite")
registry = get_registry(openkb_dir, backend=backend)

# ------------------------------------------------------------------
# 1. Hash check
# ------------------------------------------------------------------
file_hash = HashRegistry.hash_file(src)
file_hash = registry.hash_file(src)
if registry.is_known(file_hash):
logger.info("Skipping already-known file: %s", src.name)
return ConvertResult(skipped=True)
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Add SQLite-backed registry with JSON migration support by kdush · Pull Request #15 · VectifyAI/OpenKB · GitHub
Skip to content
Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

kdush
wants to merge
9
commits into

base:main
Choose a base branch

from

 Open


Add SQLite-backed registry with JSON migration support
#15
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
280fce1

feat: add SQLite-backed registry


kdush Apr 11, 2026
1397f7f

feat: add SQLite backend and migration tests


kdush Apr 11, 2026
ed49b0a

docs: document storage backend and migration


kdush Apr 11, 2026
3be4233

fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json


kdush May 13, 2026
8cb4522

fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口


kdush May 13, 2026
3451b79

fix(state): 🐛 对齐 SQLite registry 评审语义


kdush May 23, 2026
e2260a0

chore(merge): 🔧 合并上游 main 解决 PR 冲突


kdush May 23, 2026
ad7d146

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush May 30, 2026
9e4a56f

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush Jun 2, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down

51 changes: 27 additions & 24 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 retry without re-downloading.
 """
 from openkb.agent.compiler import compile_long_doc, compile_short_doc
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 logger = logging.getLogger(__name__)
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # 2. Convert document
 click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
 "model": model,
 "language": language,
 "pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
 "storage_backend": DEFAULT_CONFIG["storage_backend"],
 }
 save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

 # Write API key to KB-local .env (0600) if the user provided one
 if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 scan_affected_pages,
 )
 from openkb.lint import fix_broken_links
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
 if kb_dir is None:
 click.echo("No knowledge base found. Run `openkb init` first.")
 return

 openkb_dir = kb_dir / ".openkb"
 registry = HashRegistry(openkb_dir / "hashes.json")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 matches = _resolve_doc_identifier(registry, identifier)
 if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
 """
 from openkb.lint import run_structural_lint
 from openkb.agent.linter import run_knowledge_lint
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"

 # Skip lint entirely when the KB has no indexed documents
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 else:
 hashes = {}
 config = load_config(openkb_dir / "config.yaml")
 backend: str = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
 return

 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
 """Print all documents in the knowledge base. Usable from CLI and chat REPL."""
 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if not hashes_file.exists():
 click.echo("No documents indexed yet.")
 return
 from openkb.state import get_registry

 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("No documents indexed yet.")
 return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
 click.echo(f" {'raw':<20} {raw_count:<10}")

 # Hash registry summary
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")

 # Last compile time: newest compiled page across summaries/, concepts/,
 # and entities/ (an entity-only compile must still bump the shown time).
Expand Down

1 change: 1 addition & 0 deletions
openkb/config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 "model": "gpt-5.4-mini",
 "language": "en",
 "pageindex_threshold": 20,
 "storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down

7 changes: 4 additions & 3 deletions
openkb/converter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 threshold: int = config.get("pageindex_threshold", 20)
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # ------------------------------------------------------------------
 # 1. Hash check
 # ------------------------------------------------------------------
 file_hash = HashRegistry.hash_file(src)
 file_hash = registry.hash_file(src)
 if registry.is_known(file_hash):
 logger.info("Skipping already-known file: %s", src.name)
 return ConvertResult(skipped=True)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Add SQLite-backed registry with JSON migration support by kdush · Pull Request #15 · VectifyAI/OpenKB · GitHub
Skip to content
Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

kdush
wants to merge
9
commits into

base:main
Choose a base branch

from

 Open


Add SQLite-backed registry with JSON migration support
#15
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
280fce1

feat: add SQLite-backed registry


kdush Apr 11, 2026
1397f7f

feat: add SQLite backend and migration tests


kdush Apr 11, 2026
ed49b0a

docs: document storage backend and migration


kdush Apr 11, 2026
3be4233

fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json


kdush May 13, 2026
8cb4522

fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口


kdush May 13, 2026
3451b79

fix(state): 🐛 对齐 SQLite registry 评审语义


kdush May 23, 2026
e2260a0

chore(merge): 🔧 合并上游 main 解决 PR 冲突


kdush May 23, 2026
ad7d146

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush May 30, 2026
9e4a56f

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush Jun 2, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down

51 changes: 27 additions & 24 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 retry without re-downloading.
 """
 from openkb.agent.compiler import compile_long_doc, compile_short_doc
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 logger = logging.getLogger(__name__)
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # 2. Convert document
 click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
 "model": model,
 "language": language,
 "pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
 "storage_backend": DEFAULT_CONFIG["storage_backend"],
 }
 save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

 # Write API key to KB-local .env (0600) if the user provided one
 if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 scan_affected_pages,
 )
 from openkb.lint import fix_broken_links
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
 if kb_dir is None:
 click.echo("No knowledge base found. Run `openkb init` first.")
 return

 openkb_dir = kb_dir / ".openkb"
 registry = HashRegistry(openkb_dir / "hashes.json")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 matches = _resolve_doc_identifier(registry, identifier)
 if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
 """
 from openkb.lint import run_structural_lint
 from openkb.agent.linter import run_knowledge_lint
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"

 # Skip lint entirely when the KB has no indexed documents
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 else:
 hashes = {}
 config = load_config(openkb_dir / "config.yaml")
 backend: str = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
 return

 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
 """Print all documents in the knowledge base. Usable from CLI and chat REPL."""
 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if not hashes_file.exists():
 click.echo("No documents indexed yet.")
 return
 from openkb.state import get_registry

 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("No documents indexed yet.")
 return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
 click.echo(f" {'raw':<20} {raw_count:<10}")

 # Hash registry summary
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")

 # Last compile time: newest compiled page across summaries/, concepts/,
 # and entities/ (an entity-only compile must still bump the shown time).
Expand Down

1 change: 1 addition & 0 deletions
openkb/config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 "model": "gpt-5.4-mini",
 "language": "en",
 "pageindex_threshold": 20,
 "storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down

7 changes: 4 additions & 3 deletions
openkb/converter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 threshold: int = config.get("pageindex_threshold", 20)
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # ------------------------------------------------------------------
 # 1. Hash check
 # ------------------------------------------------------------------
 file_hash = HashRegistry.hash_file(src)
 file_hash = registry.hash_file(src)
 if registry.is_known(file_hash):
 logger.info("Skipping already-known file: %s", src.name)
 return ConvertResult(skipped=True)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Add SQLite-backed registry with JSON migration support by kdush · Pull Request #15 · VectifyAI/OpenKB · GitHub
Skip to content
Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

kdush
wants to merge
9
commits into

base:main
Choose a base branch

from

 Open


Add SQLite-backed registry with JSON migration support
#15
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
280fce1

feat: add SQLite-backed registry


kdush Apr 11, 2026
1397f7f

feat: add SQLite backend and migration tests


kdush Apr 11, 2026
ed49b0a

docs: document storage backend and migration


kdush Apr 11, 2026
3be4233

fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json


kdush May 13, 2026
8cb4522

fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口


kdush May 13, 2026
3451b79

fix(state): 🐛 对齐 SQLite registry 评审语义


kdush May 23, 2026
e2260a0

chore(merge): 🔧 合并上游 main 解决 PR 冲突


kdush May 23, 2026
ad7d146

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush May 30, 2026
9e4a56f

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush Jun 2, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down

51 changes: 27 additions & 24 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 retry without re-downloading.
 """
 from openkb.agent.compiler import compile_long_doc, compile_short_doc
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 logger = logging.getLogger(__name__)
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # 2. Convert document
 click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
 "model": model,
 "language": language,
 "pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
 "storage_backend": DEFAULT_CONFIG["storage_backend"],
 }
 save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

 # Write API key to KB-local .env (0600) if the user provided one
 if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 scan_affected_pages,
 )
 from openkb.lint import fix_broken_links
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
 if kb_dir is None:
 click.echo("No knowledge base found. Run `openkb init` first.")
 return

 openkb_dir = kb_dir / ".openkb"
 registry = HashRegistry(openkb_dir / "hashes.json")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 matches = _resolve_doc_identifier(registry, identifier)
 if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
 """
 from openkb.lint import run_structural_lint
 from openkb.agent.linter import run_knowledge_lint
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"

 # Skip lint entirely when the KB has no indexed documents
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 else:
 hashes = {}
 config = load_config(openkb_dir / "config.yaml")
 backend: str = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
 return

 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
 """Print all documents in the knowledge base. Usable from CLI and chat REPL."""
 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if not hashes_file.exists():
 click.echo("No documents indexed yet.")
 return
 from openkb.state import get_registry

 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("No documents indexed yet.")
 return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
 click.echo(f" {'raw':<20} {raw_count:<10}")

 # Hash registry summary
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")

 # Last compile time: newest compiled page across summaries/, concepts/,
 # and entities/ (an entity-only compile must still bump the shown time).
Expand Down

1 change: 1 addition & 0 deletions
openkb/config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 "model": "gpt-5.4-mini",
 "language": "en",
 "pageindex_threshold": 20,
 "storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down

7 changes: 4 additions & 3 deletions
openkb/converter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 threshold: int = config.get("pageindex_threshold", 20)
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # ------------------------------------------------------------------
 # 1. Hash check
 # ------------------------------------------------------------------
 file_hash = HashRegistry.hash_file(src)
 file_hash = registry.hash_file(src)
 if registry.is_known(file_hash):
 logger.info("Skipping already-known file: %s", src.name)
 return ConvertResult(skipped=True)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
Add SQLite-backed registry with JSON migration support by kdush · Pull Request #15 · VectifyAI/OpenKB · GitHub
Skip to content
Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

kdush
wants to merge
9
commits into

base:main
Choose a base branch

from

 Open


Add SQLite-backed registry with JSON migration support
#15
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
280fce1

feat: add SQLite-backed registry


kdush Apr 11, 2026
1397f7f

feat: add SQLite backend and migration tests


kdush Apr 11, 2026
ed49b0a

docs: document storage backend and migration


kdush Apr 11, 2026
3be4233

fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json


kdush May 13, 2026
8cb4522

fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口


kdush May 13, 2026
3451b79

fix(state): 🐛 对齐 SQLite registry 评审语义


kdush May 23, 2026
e2260a0

chore(merge): 🔧 合并上游 main 解决 PR 冲突


kdush May 23, 2026
ad7d146

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush May 30, 2026
9e4a56f

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush Jun 2, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down

51 changes: 27 additions & 24 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 retry without re-downloading.
 """
 from openkb.agent.compiler import compile_long_doc, compile_short_doc
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 logger = logging.getLogger(__name__)
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # 2. Convert document
 click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
 "model": model,
 "language": language,
 "pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
 "storage_backend": DEFAULT_CONFIG["storage_backend"],
 }
 save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

 # Write API key to KB-local .env (0600) if the user provided one
 if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 scan_affected_pages,
 )
 from openkb.lint import fix_broken_links
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
 if kb_dir is None:
 click.echo("No knowledge base found. Run `openkb init` first.")
 return

 openkb_dir = kb_dir / ".openkb"
 registry = HashRegistry(openkb_dir / "hashes.json")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 matches = _resolve_doc_identifier(registry, identifier)
 if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
 """
 from openkb.lint import run_structural_lint
 from openkb.agent.linter import run_knowledge_lint
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"

 # Skip lint entirely when the KB has no indexed documents
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 else:
 hashes = {}
 config = load_config(openkb_dir / "config.yaml")
 backend: str = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
 return

 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
 """Print all documents in the knowledge base. Usable from CLI and chat REPL."""
 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if not hashes_file.exists():
 click.echo("No documents indexed yet.")
 return
 from openkb.state import get_registry

 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("No documents indexed yet.")
 return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
 click.echo(f" {'raw':<20} {raw_count:<10}")

 # Hash registry summary
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")

 # Last compile time: newest compiled page across summaries/, concepts/,
 # and entities/ (an entity-only compile must still bump the shown time).
Expand Down

1 change: 1 addition & 0 deletions
openkb/config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 "model": "gpt-5.4-mini",
 "language": "en",
 "pageindex_threshold": 20,
 "storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down

7 changes: 4 additions & 3 deletions
openkb/converter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 threshold: int = config.get("pageindex_threshold", 20)
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # ------------------------------------------------------------------
 # 1. Hash check
 # ------------------------------------------------------------------
 file_hash = HashRegistry.hash_file(src)
 file_hash = registry.hash_file(src)
 if registry.is_known(file_hash):
 logger.info("Skipping already-known file: %s", src.name)
 return ConvertResult(skipped=True)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Add SQLite-backed registry with JSON migration support by kdush · Pull Request #15 · VectifyAI/OpenKB · GitHub
Skip to content
Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

kdush
wants to merge
9
commits into

base:main
Choose a base branch

from

 Open


Add SQLite-backed registry with JSON migration support
#15
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
280fce1

feat: add SQLite-backed registry


kdush Apr 11, 2026
1397f7f

feat: add SQLite backend and migration tests


kdush Apr 11, 2026
ed49b0a

docs: document storage backend and migration


kdush Apr 11, 2026
3be4233

fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json


kdush May 13, 2026
8cb4522

fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口


kdush May 13, 2026
3451b79

fix(state): 🐛 对齐 SQLite registry 评审语义


kdush May 23, 2026
e2260a0

chore(merge): 🔧 合并上游 main 解决 PR 冲突


kdush May 23, 2026
ad7d146

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush May 30, 2026
9e4a56f

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush Jun 2, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down

51 changes: 27 additions & 24 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 retry without re-downloading.
 """
 from openkb.agent.compiler import compile_long_doc, compile_short_doc
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 logger = logging.getLogger(__name__)
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # 2. Convert document
 click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
 "model": model,
 "language": language,
 "pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
 "storage_backend": DEFAULT_CONFIG["storage_backend"],
 }
 save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

 # Write API key to KB-local .env (0600) if the user provided one
 if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 scan_affected_pages,
 )
 from openkb.lint import fix_broken_links
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
 if kb_dir is None:
 click.echo("No knowledge base found. Run `openkb init` first.")
 return

 openkb_dir = kb_dir / ".openkb"
 registry = HashRegistry(openkb_dir / "hashes.json")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 matches = _resolve_doc_identifier(registry, identifier)
 if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
 """
 from openkb.lint import run_structural_lint
 from openkb.agent.linter import run_knowledge_lint
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"

 # Skip lint entirely when the KB has no indexed documents
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 else:
 hashes = {}
 config = load_config(openkb_dir / "config.yaml")
 backend: str = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
 return

 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
 """Print all documents in the knowledge base. Usable from CLI and chat REPL."""
 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if not hashes_file.exists():
 click.echo("No documents indexed yet.")
 return
 from openkb.state import get_registry

 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("No documents indexed yet.")
 return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
 click.echo(f" {'raw':<20} {raw_count:<10}")

 # Hash registry summary
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")

 # Last compile time: newest compiled page across summaries/, concepts/,
 # and entities/ (an entity-only compile must still bump the shown time).
Expand Down

1 change: 1 addition & 0 deletions
openkb/config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 "model": "gpt-5.4-mini",
 "language": "en",
 "pageindex_threshold": 20,
 "storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down

7 changes: 4 additions & 3 deletions
openkb/converter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 threshold: int = config.get("pageindex_threshold", 20)
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # ------------------------------------------------------------------
 # 1. Hash check
 # ------------------------------------------------------------------
 file_hash = HashRegistry.hash_file(src)
 file_hash = registry.hash_file(src)
 if registry.is_known(file_hash):
 logger.info("Skipping already-known file: %s", src.name)
 return ConvertResult(skipped=True)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
})();
Add SQLite-backed registry with JSON migration support by kdush · Pull Request #15 · VectifyAI/OpenKB · GitHub
Skip to content
Add SQLite-backed registry with JSON migration support#15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

kdush
wants to merge
9
commits into

base:main
Choose a base branch

from

 Open


Add SQLite-backed registry with JSON migration support
#15
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
280fce1

feat: add SQLite-backed registry


kdush Apr 11, 2026
1397f7f

feat: add SQLite backend and migration tests


kdush Apr 11, 2026
ed49b0a

docs: document storage backend and migration


kdush Apr 11, 2026
3be4233

fix(cli): run_lint 改用 get_registry() 读取注册表，不再直接访问 hashes.json


kdush May 13, 2026
8cb4522

fix(state): JSON→SQLite 迁移改为原子/可重试，并预兼容 PR #30 接口


kdush May 13, 2026
3451b79

fix(state): 🐛 对齐 SQLite registry 评审语义


kdush May 23, 2026
e2260a0

chore(merge): 🔧 合并上游 main 解决 PR 冲突


kdush May 23, 2026
ad7d146

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush May 30, 2026
9e4a56f

Merge remote-tracking branch 'origin/main' into feat/sqlite-storage-b…


kdush Jun 2, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -269,10 +269,22 @@ Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
model: gpt-5.4 # LLM model (any LiteLLM-supported provider)
language: en # Wiki output language
pageindex_threshold: 20 # PDF pages threshold for PageIndex
storage_backend: sqlite # Storage backend: sqlite (default) or json
```

`entity_types` (optional): a YAML list overriding the entity-type vocabulary used for entity pages; omit it to use the default `person`, `organization`, `place`, `product`, `work`, `event`, `other`.

### Storage Backend

OpenKB supports two storage backends for the file hash registry:

| Backend | Description | Use Case |
|---------|-------------|----------|
| `sqlite` | SQLite database (default) | Better concurrency, scalability, recommended for production |
| `json` | JSON file | Simple, human-readable, for small installations |

Migration from JSON to SQLite happens automatically when you switch to `sqlite` backend and a `hashes.json` file exists. The JSON file is preserved but no longer used.

Model names use `provider/model` LiteLLM [format](https://docs.litellm.ai/docs/providers) (OpenAI models can omit the prefix):

| Provider | Model example |
Expand Down

51 changes: 27 additions & 24 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -277,14 +277,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 retry without re-downloading.
 """
 from openkb.agent.compiler import compile_long_doc, compile_short_doc
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 logger = logging.getLogger(__name__)
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # 2. Convert document
 click.echo(f"Adding: {file_path.name}")
Expand DownExpand Up@@ -551,9 +552,10 @@ def init(model, language):
 "model": model,
 "language": language,
 "pageindex_threshold": DEFAULT_CONFIG["pageindex_threshold"],
 "storage_backend": DEFAULT_CONFIG["storage_backend"],
 }
 save_config(openkb_dir / "config.yaml", config)
(openkb_dir / "hashes.json").write_text(json.dumps({}), encoding="utf-8")
# SQLite DB 会在首次访问时由 get_registry() 自动创建，无需预创建

 # Write API key to KB-local .env (0600) if the user provided one
 if api_key:
Expand DownExpand Up@@ -805,15 +807,17 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 scan_affected_pages,
 )
 from openkb.lint import fix_broken_links
 from openkb.state import HashRegistry
 from openkb.state import get_registry

 kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
 if kb_dir is None:
 click.echo("No knowledge base found. Run `openkb init` first.")
 return

 openkb_dir = kb_dir / ".openkb"
 registry = HashRegistry(openkb_dir / "hashes.json")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 matches = _resolve_doc_identifier(registry, identifier)
 if not matches:
Expand DownExpand Up@@ -1365,20 +1369,16 @@ async def run_lint(kb_dir: Path) -> Path | None:
 """
 from openkb.lint import run_structural_lint
 from openkb.agent.linter import run_knowledge_lint
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"

 # Skip lint entirely when the KB has no indexed documents
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 else:
 hashes = {}
 config = load_config(openkb_dir / "config.yaml")
 backend: str = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.")
 return

 config = load_config(openkb_dir / "config.yaml")
 _setup_llm_key(kb_dir)
 model: str = config.get("model", DEFAULT_CONFIG["model"])

Expand DownExpand Up@@ -1431,13 +1431,13 @@ def lint(ctx, fix):

def print_list(kb_dir: Path) -> None:
 """Print all documents in the knowledge base. Usable from CLI and chat REPL."""
 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if not hashes_file.exists():
 click.echo("No documents indexed yet.")
 return
 from openkb.state import get_registry

 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 if not hashes:
 click.echo("No documents indexed yet.")
 return
Expand DownExpand Up@@ -1531,11 +1531,14 @@ def print_status(kb_dir: Path) -> None:
 click.echo(f" {'raw':<20} {raw_count:<10}")

 # Hash registry summary
 from openkb.state import get_registry

 openkb_dir = kb_dir / ".openkb"
 hashes_file = openkb_dir / "hashes.json"
 if hashes_file.exists():
 hashes = json.loads(hashes_file.read_text(encoding="utf-8"))
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")
 config = load_config(openkb_dir / "config.yaml")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)
 hashes = registry.all_entries()
 click.echo(f"\n Total indexed: {len(hashes)} document(s)")

 # Last compile time: newest compiled page across summaries/, concepts/,
 # and entities/ (an entity-only compile must still bump the shown time).
Expand Down

1 change: 1 addition & 0 deletions
openkb/config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,7 @@
 "model": "gpt-5.4-mini",
 "language": "en",
 "pageindex_threshold": 20,
 "storage_backend": "sqlite",
}

# Default entity-type vocabulary. Overridable per-KB via the optional
Expand Down

7 changes: 4 additions & 3 deletions
openkb/converter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@

from openkb.config import load_config
from openkb.images import copy_relative_images, extract_base64_images, convert_pdf_with_images
from openkb.state import HashRegistry
from openkb.state import get_registry

logger = logging.getLogger(__name__)

Expand DownExpand Up@@ -50,12 +50,13 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
 openkb_dir = kb_dir / ".openkb"
 config = load_config(openkb_dir / "config.yaml")
 threshold: int = config.get("pageindex_threshold", 20)
 registry = HashRegistry(openkb_dir / "hashes.json")
 backend = config.get("storage_backend", "sqlite")
 registry = get_registry(openkb_dir, backend=backend)

 # ------------------------------------------------------------------
 # 1. Hash check
 # ------------------------------------------------------------------
 file_hash = HashRegistry.hash_file(src)
 file_hash = registry.hash_file(src)
 if registry.is_known(file_hash):
 logger.info("Skipping already-known file: %s", src.name)
 return ConvertResult(skipped=True)
Expand Down
Loading