Skip to content

Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
specode wants to merge 1 commit into
base:main
Choose a base branch
from
Open

Add pluggable ingest bundle pipeline #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions examples/ingest-plugin/README.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
pipeline: bundle
importers:
enabled:
- example_text
normalizers:
enabled:
- example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.
50 changes: 50 additions & 0 deletions examples/ingest-plugin/openkb_example_ingest.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
name = "example_text"

def can_handle(self, target: str, context) -> bool:
del context
return target.startswith("example-text:")

def import_source(self, target: str, context) -> IngestInput:
path = context.staging_dir / "raw" / "example.txt"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
return IngestInput(
target=target,
path=path,
source_uri=target,
media_type="text/x-example",
metadata={"display_name": "example.txt"},
)


class ExampleTextNormalizer:
name = "example_text"

def supports(self, input_: IngestInput, context) -> bool:
del context
return input_.media_type == "text/x-example"

def normalize(self, input_: IngestInput, context) -> DocumentBundle:
del context
if input_.path is None:
raise ValueError("Example normalizer requires a local path.")
source_uri = input_.source_uri or input_.target
text = input_.path.read_text(encoding="utf-8")
return DocumentBundle(
id=source_uri,
title="Example Text",
source_uri=source_uri,
blocks=[TextBlock(text)],
metadata={
"display_name": input_.metadata.get("display_name", input_.path.name),
"source_path": input_.path.as_posix(),
"source_identity": source_uri,
},
provenance=[ProvenanceRecord(source_uri=source_uri)],
)
11 changes: 11 additions & 0 deletions examples/ingest-plugin/pyproject.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"
63 changes: 53 additions & 10 deletions openkb/cli.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
".png",
".jpg",
".jpeg",
".gif",
".webp",
".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
"long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
help="Import an already-indexed PageIndex Cloud document by its doc-id "
"(no local file). Mutually exclusive with PATH.",
)
@click.option(
"--ingest-pipeline",
"ingest_pipeline",
type=click.Choice(["legacy", "bundle"]),
default=None,
help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
"""Add a document or directory of documents at PATH to the knowledge base.

PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
return

# URL ingest: download into raw/ first, then call add_single_file explicitly.
# Keep staged conversion enabled so converted source artifacts do not touch
# the live KB before the mutation snapshot exists. The tri-state outcome
# still lets us clean up the just-downloaded raw file on dedup.
from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

config = load_config(kb_dir / ".openkb" / "config.yaml")
pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
supported_extensions = (
BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
)

# Legacy URL ingest downloads into raw/ first, then calls add_single_file.
# Bundle URL ingest keeps downloads inside the bundle staging directory and
# commits through the same mutation boundary as local bundle files.
from openkb.url_ingest import looks_like_url, fetch_url_to_raw

if looks_like_url(path):
if pipeline == "bundle":
add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
return
fetched = fetch_url_to_raw(path, kb_dir)
if fetched is None:
return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
files = [
f
for f in sorted(target.rglob("*"))
if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
if f.is_file() and f.suffix.lower() in supported_extensions
]
if not files:
click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
click.echo(f"Found {total} supported file(s) in {path}.")
for i, f in enumerate(files, 1):
click.echo(f"\n[{i}/{total}] ", nl=False)
add_single_file(f, kb_dir)
if pipeline == "bundle":
add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
else:
add_single_file(f, kb_dir)
else:
if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
if target.suffix.lower() not in supported_extensions:
click.echo(
f"Unsupported file type: {target.suffix}. "
f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
f"Supported: {', '.join(sorted(supported_extensions))}"
)
return
add_single_file(target, kb_dir)
if pipeline == "bundle":
add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
else:
add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
)
)

bundle_path = None
if meta.get("bundle_path"):
candidate = kb_dir / meta["bundle_path"]
if candidate.exists():
bundle_path = candidate
actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

# Scan concept pages to predict which will be edited vs. deleted.
# Only frontmatter ``sources:`` membership drives the plan — body-only
# references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
source_json.unlink(missing_ok=True)
if images_dir.is_dir():
shutil.rmtree(images_dir, ignore_errors=True)
if bundle_path is not None:
bundle_path.unlink(missing_ok=True)

concept_result = remove_doc_from_concept_pages(
wiki_dir,
Expand Down
7 changes: 7 additions & 0 deletions openkb/ingest/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Add pluggable ingest bundle pipeline by specode · Pull Request #179 · VectifyAI/OpenKB · GitHub
Skip to content
Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

specode
wants to merge
1
commit into

base:main
Choose a base branch

from

 Open


Add pluggable ingest bundle pipeline
#179
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


27 changes: 27 additions & 0 deletions
examples/ingest-plugin/README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
 pipeline: bundle
 importers:
 enabled:
 - example_text
 normalizers:
 enabled:
 - example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.

50 changes: 50 additions & 0 deletions
examples/ingest-plugin/openkb_example_ingest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
 name = "example_text"

 def can_handle(self, target: str, context) -> bool:
 del context
 return target.startswith("example-text:")

 def import_source(self, target: str, context) -> IngestInput:
 path = context.staging_dir / "raw" / "example.txt"
 path.parent.mkdir(parents=True, exist_ok=True)
 path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
 return IngestInput(
 target=target,
 path=path,
 source_uri=target,
 media_type="text/x-example",
 metadata={"display_name": "example.txt"},
 )


class ExampleTextNormalizer:
 name = "example_text"

 def supports(self, input_: IngestInput, context) -> bool:
 del context
 return input_.media_type == "text/x-example"

 def normalize(self, input_: IngestInput, context) -> DocumentBundle:
 del context
 if input_.path is None:
 raise ValueError("Example normalizer requires a local path.")
 source_uri = input_.source_uri or input_.target
 text = input_.path.read_text(encoding="utf-8")
 return DocumentBundle(
 id=source_uri,
 title="Example Text",
 source_uri=source_uri,
 blocks=[TextBlock(text)],
 metadata={
 "display_name": input_.metadata.get("display_name", input_.path.name),
 "source_path": input_.path.as_posix(),
 "source_identity": source_uri,
 },
 provenance=[ProvenanceRecord(source_uri=source_uri)],
 )

11 changes: 11 additions & 0 deletions
examples/ingest-plugin/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"

63 changes: 53 additions & 10 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
 ".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
 ".png",
 ".jpg",
 ".jpeg",
 ".gif",
 ".webp",
 ".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
 "long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
 help="Import an already-indexed PageIndex Cloud document by its doc-id "
 "(no local file). Mutually exclusive with PATH.",
)
@click.option(
 "--ingest-pipeline",
 "ingest_pipeline",
 type=click.Choice(["legacy", "bundle"]),
 default=None,
 help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
 """Add a document or directory of documents at PATH to the knowledge base.

 PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
 return

 # URL ingest: download into raw/ first, then call add_single_file explicitly.
 # Keep staged conversion enabled so converted source artifacts do not touch
 # the live KB before the mutation snapshot exists. The tri-state outcome
 # still lets us clean up the just-downloaded raw file on dedup.
 from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

 config = load_config(kb_dir / ".openkb" / "config.yaml")
 pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
 supported_extensions = (
 BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
 )

 # Legacy URL ingest downloads into raw/ first, then calls add_single_file.
 # Bundle URL ingest keeps downloads inside the bundle staging directory and
 # commits through the same mutation boundary as local bundle files.
 from openkb.url_ingest import looks_like_url, fetch_url_to_raw

 if looks_like_url(path):
 if pipeline == "bundle":
 add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
 return
 fetched = fetch_url_to_raw(path, kb_dir)
 if fetched is None:
 return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
 files = [
 f
 for f in sorted(target.rglob("*"))
 if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
 if f.is_file() and f.suffix.lower() in supported_extensions
 ]
 if not files:
 click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo(f"Found {total} supported file(s) in {path}.")
 for i, f in enumerate(files, 1):
 click.echo(f"\n[{i}/{total}] ", nl=False)
 add_single_file(f, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(f, kb_dir)
 else:
 if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
 if target.suffix.lower() not in supported_extensions:
 click.echo(
 f"Unsupported file type: {target.suffix}. "
 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
 f"Supported: {', '.join(sorted(supported_extensions))}"
 )
 return
 add_single_file(target, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 )
 )

 bundle_path = None
 if meta.get("bundle_path"):
 candidate = kb_dir / meta["bundle_path"]
 if candidate.exists():
 bundle_path = candidate
 actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

 # Scan concept pages to predict which will be edited vs. deleted.
 # Only frontmatter ``sources:`` membership drives the plan — body-only
 # references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 source_json.unlink(missing_ok=True)
 if images_dir.is_dir():
 shutil.rmtree(images_dir, ignore_errors=True)
 if bundle_path is not None:
 bundle_path.unlink(missing_ok=True)

 concept_result = remove_doc_from_concept_pages(
 wiki_dir,
Expand Down

7 changes: 7 additions & 0 deletions
openkb/ingest/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Add pluggable ingest bundle pipeline by specode · Pull Request #179 · VectifyAI/OpenKB · GitHub
Skip to content
Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

specode
wants to merge
1
commit into

base:main
Choose a base branch

from

 Open


Add pluggable ingest bundle pipeline
#179
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


27 changes: 27 additions & 0 deletions
examples/ingest-plugin/README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
 pipeline: bundle
 importers:
 enabled:
 - example_text
 normalizers:
 enabled:
 - example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.

50 changes: 50 additions & 0 deletions
examples/ingest-plugin/openkb_example_ingest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
 name = "example_text"

 def can_handle(self, target: str, context) -> bool:
 del context
 return target.startswith("example-text:")

 def import_source(self, target: str, context) -> IngestInput:
 path = context.staging_dir / "raw" / "example.txt"
 path.parent.mkdir(parents=True, exist_ok=True)
 path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
 return IngestInput(
 target=target,
 path=path,
 source_uri=target,
 media_type="text/x-example",
 metadata={"display_name": "example.txt"},
 )


class ExampleTextNormalizer:
 name = "example_text"

 def supports(self, input_: IngestInput, context) -> bool:
 del context
 return input_.media_type == "text/x-example"

 def normalize(self, input_: IngestInput, context) -> DocumentBundle:
 del context
 if input_.path is None:
 raise ValueError("Example normalizer requires a local path.")
 source_uri = input_.source_uri or input_.target
 text = input_.path.read_text(encoding="utf-8")
 return DocumentBundle(
 id=source_uri,
 title="Example Text",
 source_uri=source_uri,
 blocks=[TextBlock(text)],
 metadata={
 "display_name": input_.metadata.get("display_name", input_.path.name),
 "source_path": input_.path.as_posix(),
 "source_identity": source_uri,
 },
 provenance=[ProvenanceRecord(source_uri=source_uri)],
 )

11 changes: 11 additions & 0 deletions
examples/ingest-plugin/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"

63 changes: 53 additions & 10 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
 ".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
 ".png",
 ".jpg",
 ".jpeg",
 ".gif",
 ".webp",
 ".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
 "long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
 help="Import an already-indexed PageIndex Cloud document by its doc-id "
 "(no local file). Mutually exclusive with PATH.",
)
@click.option(
 "--ingest-pipeline",
 "ingest_pipeline",
 type=click.Choice(["legacy", "bundle"]),
 default=None,
 help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
 """Add a document or directory of documents at PATH to the knowledge base.

 PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
 return

 # URL ingest: download into raw/ first, then call add_single_file explicitly.
 # Keep staged conversion enabled so converted source artifacts do not touch
 # the live KB before the mutation snapshot exists. The tri-state outcome
 # still lets us clean up the just-downloaded raw file on dedup.
 from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

 config = load_config(kb_dir / ".openkb" / "config.yaml")
 pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
 supported_extensions = (
 BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
 )

 # Legacy URL ingest downloads into raw/ first, then calls add_single_file.
 # Bundle URL ingest keeps downloads inside the bundle staging directory and
 # commits through the same mutation boundary as local bundle files.
 from openkb.url_ingest import looks_like_url, fetch_url_to_raw

 if looks_like_url(path):
 if pipeline == "bundle":
 add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
 return
 fetched = fetch_url_to_raw(path, kb_dir)
 if fetched is None:
 return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
 files = [
 f
 for f in sorted(target.rglob("*"))
 if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
 if f.is_file() and f.suffix.lower() in supported_extensions
 ]
 if not files:
 click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo(f"Found {total} supported file(s) in {path}.")
 for i, f in enumerate(files, 1):
 click.echo(f"\n[{i}/{total}] ", nl=False)
 add_single_file(f, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(f, kb_dir)
 else:
 if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
 if target.suffix.lower() not in supported_extensions:
 click.echo(
 f"Unsupported file type: {target.suffix}. "
 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
 f"Supported: {', '.join(sorted(supported_extensions))}"
 )
 return
 add_single_file(target, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 )
 )

 bundle_path = None
 if meta.get("bundle_path"):
 candidate = kb_dir / meta["bundle_path"]
 if candidate.exists():
 bundle_path = candidate
 actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

 # Scan concept pages to predict which will be edited vs. deleted.
 # Only frontmatter ``sources:`` membership drives the plan — body-only
 # references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 source_json.unlink(missing_ok=True)
 if images_dir.is_dir():
 shutil.rmtree(images_dir, ignore_errors=True)
 if bundle_path is not None:
 bundle_path.unlink(missing_ok=True)

 concept_result = remove_doc_from_concept_pages(
 wiki_dir,
Expand Down

7 changes: 7 additions & 0 deletions
openkb/ingest/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Add pluggable ingest bundle pipeline by specode · Pull Request #179 · VectifyAI/OpenKB · GitHub
Skip to content
Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

specode
wants to merge
1
commit into

base:main
Choose a base branch

from

 Open


Add pluggable ingest bundle pipeline
#179
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


27 changes: 27 additions & 0 deletions
examples/ingest-plugin/README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
 pipeline: bundle
 importers:
 enabled:
 - example_text
 normalizers:
 enabled:
 - example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.

50 changes: 50 additions & 0 deletions
examples/ingest-plugin/openkb_example_ingest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
 name = "example_text"

 def can_handle(self, target: str, context) -> bool:
 del context
 return target.startswith("example-text:")

 def import_source(self, target: str, context) -> IngestInput:
 path = context.staging_dir / "raw" / "example.txt"
 path.parent.mkdir(parents=True, exist_ok=True)
 path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
 return IngestInput(
 target=target,
 path=path,
 source_uri=target,
 media_type="text/x-example",
 metadata={"display_name": "example.txt"},
 )


class ExampleTextNormalizer:
 name = "example_text"

 def supports(self, input_: IngestInput, context) -> bool:
 del context
 return input_.media_type == "text/x-example"

 def normalize(self, input_: IngestInput, context) -> DocumentBundle:
 del context
 if input_.path is None:
 raise ValueError("Example normalizer requires a local path.")
 source_uri = input_.source_uri or input_.target
 text = input_.path.read_text(encoding="utf-8")
 return DocumentBundle(
 id=source_uri,
 title="Example Text",
 source_uri=source_uri,
 blocks=[TextBlock(text)],
 metadata={
 "display_name": input_.metadata.get("display_name", input_.path.name),
 "source_path": input_.path.as_posix(),
 "source_identity": source_uri,
 },
 provenance=[ProvenanceRecord(source_uri=source_uri)],
 )

11 changes: 11 additions & 0 deletions
examples/ingest-plugin/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"

63 changes: 53 additions & 10 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
 ".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
 ".png",
 ".jpg",
 ".jpeg",
 ".gif",
 ".webp",
 ".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
 "long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
 help="Import an already-indexed PageIndex Cloud document by its doc-id "
 "(no local file). Mutually exclusive with PATH.",
)
@click.option(
 "--ingest-pipeline",
 "ingest_pipeline",
 type=click.Choice(["legacy", "bundle"]),
 default=None,
 help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
 """Add a document or directory of documents at PATH to the knowledge base.

 PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
 return

 # URL ingest: download into raw/ first, then call add_single_file explicitly.
 # Keep staged conversion enabled so converted source artifacts do not touch
 # the live KB before the mutation snapshot exists. The tri-state outcome
 # still lets us clean up the just-downloaded raw file on dedup.
 from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

 config = load_config(kb_dir / ".openkb" / "config.yaml")
 pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
 supported_extensions = (
 BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
 )

 # Legacy URL ingest downloads into raw/ first, then calls add_single_file.
 # Bundle URL ingest keeps downloads inside the bundle staging directory and
 # commits through the same mutation boundary as local bundle files.
 from openkb.url_ingest import looks_like_url, fetch_url_to_raw

 if looks_like_url(path):
 if pipeline == "bundle":
 add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
 return
 fetched = fetch_url_to_raw(path, kb_dir)
 if fetched is None:
 return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
 files = [
 f
 for f in sorted(target.rglob("*"))
 if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
 if f.is_file() and f.suffix.lower() in supported_extensions
 ]
 if not files:
 click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo(f"Found {total} supported file(s) in {path}.")
 for i, f in enumerate(files, 1):
 click.echo(f"\n[{i}/{total}] ", nl=False)
 add_single_file(f, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(f, kb_dir)
 else:
 if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
 if target.suffix.lower() not in supported_extensions:
 click.echo(
 f"Unsupported file type: {target.suffix}. "
 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
 f"Supported: {', '.join(sorted(supported_extensions))}"
 )
 return
 add_single_file(target, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 )
 )

 bundle_path = None
 if meta.get("bundle_path"):
 candidate = kb_dir / meta["bundle_path"]
 if candidate.exists():
 bundle_path = candidate
 actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

 # Scan concept pages to predict which will be edited vs. deleted.
 # Only frontmatter ``sources:`` membership drives the plan — body-only
 # references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 source_json.unlink(missing_ok=True)
 if images_dir.is_dir():
 shutil.rmtree(images_dir, ignore_errors=True)
 if bundle_path is not None:
 bundle_path.unlink(missing_ok=True)

 concept_result = remove_doc_from_concept_pages(
 wiki_dir,
Expand Down

7 changes: 7 additions & 0 deletions
openkb/ingest/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
Add pluggable ingest bundle pipeline by specode · Pull Request #179 · VectifyAI/OpenKB · GitHub
Skip to content
Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

specode
wants to merge
1
commit into

base:main
Choose a base branch

from

 Open


Add pluggable ingest bundle pipeline
#179
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


27 changes: 27 additions & 0 deletions
examples/ingest-plugin/README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
 pipeline: bundle
 importers:
 enabled:
 - example_text
 normalizers:
 enabled:
 - example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.

50 changes: 50 additions & 0 deletions
examples/ingest-plugin/openkb_example_ingest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
 name = "example_text"

 def can_handle(self, target: str, context) -> bool:
 del context
 return target.startswith("example-text:")

 def import_source(self, target: str, context) -> IngestInput:
 path = context.staging_dir / "raw" / "example.txt"
 path.parent.mkdir(parents=True, exist_ok=True)
 path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
 return IngestInput(
 target=target,
 path=path,
 source_uri=target,
 media_type="text/x-example",
 metadata={"display_name": "example.txt"},
 )


class ExampleTextNormalizer:
 name = "example_text"

 def supports(self, input_: IngestInput, context) -> bool:
 del context
 return input_.media_type == "text/x-example"

 def normalize(self, input_: IngestInput, context) -> DocumentBundle:
 del context
 if input_.path is None:
 raise ValueError("Example normalizer requires a local path.")
 source_uri = input_.source_uri or input_.target
 text = input_.path.read_text(encoding="utf-8")
 return DocumentBundle(
 id=source_uri,
 title="Example Text",
 source_uri=source_uri,
 blocks=[TextBlock(text)],
 metadata={
 "display_name": input_.metadata.get("display_name", input_.path.name),
 "source_path": input_.path.as_posix(),
 "source_identity": source_uri,
 },
 provenance=[ProvenanceRecord(source_uri=source_uri)],
 )

11 changes: 11 additions & 0 deletions
examples/ingest-plugin/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"

63 changes: 53 additions & 10 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
 ".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
 ".png",
 ".jpg",
 ".jpeg",
 ".gif",
 ".webp",
 ".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
 "long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
 help="Import an already-indexed PageIndex Cloud document by its doc-id "
 "(no local file). Mutually exclusive with PATH.",
)
@click.option(
 "--ingest-pipeline",
 "ingest_pipeline",
 type=click.Choice(["legacy", "bundle"]),
 default=None,
 help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
 """Add a document or directory of documents at PATH to the knowledge base.

 PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
 return

 # URL ingest: download into raw/ first, then call add_single_file explicitly.
 # Keep staged conversion enabled so converted source artifacts do not touch
 # the live KB before the mutation snapshot exists. The tri-state outcome
 # still lets us clean up the just-downloaded raw file on dedup.
 from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

 config = load_config(kb_dir / ".openkb" / "config.yaml")
 pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
 supported_extensions = (
 BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
 )

 # Legacy URL ingest downloads into raw/ first, then calls add_single_file.
 # Bundle URL ingest keeps downloads inside the bundle staging directory and
 # commits through the same mutation boundary as local bundle files.
 from openkb.url_ingest import looks_like_url, fetch_url_to_raw

 if looks_like_url(path):
 if pipeline == "bundle":
 add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
 return
 fetched = fetch_url_to_raw(path, kb_dir)
 if fetched is None:
 return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
 files = [
 f
 for f in sorted(target.rglob("*"))
 if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
 if f.is_file() and f.suffix.lower() in supported_extensions
 ]
 if not files:
 click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo(f"Found {total} supported file(s) in {path}.")
 for i, f in enumerate(files, 1):
 click.echo(f"\n[{i}/{total}] ", nl=False)
 add_single_file(f, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(f, kb_dir)
 else:
 if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
 if target.suffix.lower() not in supported_extensions:
 click.echo(
 f"Unsupported file type: {target.suffix}. "
 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
 f"Supported: {', '.join(sorted(supported_extensions))}"
 )
 return
 add_single_file(target, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 )
 )

 bundle_path = None
 if meta.get("bundle_path"):
 candidate = kb_dir / meta["bundle_path"]
 if candidate.exists():
 bundle_path = candidate
 actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

 # Scan concept pages to predict which will be edited vs. deleted.
 # Only frontmatter ``sources:`` membership drives the plan — body-only
 # references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 source_json.unlink(missing_ok=True)
 if images_dir.is_dir():
 shutil.rmtree(images_dir, ignore_errors=True)
 if bundle_path is not None:
 bundle_path.unlink(missing_ok=True)

 concept_result = remove_doc_from_concept_pages(
 wiki_dir,
Expand Down

7 changes: 7 additions & 0 deletions
openkb/ingest/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Add pluggable ingest bundle pipeline by specode · Pull Request #179 · VectifyAI/OpenKB · GitHub
Skip to content
Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

specode
wants to merge
1
commit into

base:main
Choose a base branch

from

 Open


Add pluggable ingest bundle pipeline
#179
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


27 changes: 27 additions & 0 deletions
examples/ingest-plugin/README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
 pipeline: bundle
 importers:
 enabled:
 - example_text
 normalizers:
 enabled:
 - example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.

50 changes: 50 additions & 0 deletions
examples/ingest-plugin/openkb_example_ingest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
 name = "example_text"

 def can_handle(self, target: str, context) -> bool:
 del context
 return target.startswith("example-text:")

 def import_source(self, target: str, context) -> IngestInput:
 path = context.staging_dir / "raw" / "example.txt"
 path.parent.mkdir(parents=True, exist_ok=True)
 path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
 return IngestInput(
 target=target,
 path=path,
 source_uri=target,
 media_type="text/x-example",
 metadata={"display_name": "example.txt"},
 )


class ExampleTextNormalizer:
 name = "example_text"

 def supports(self, input_: IngestInput, context) -> bool:
 del context
 return input_.media_type == "text/x-example"

 def normalize(self, input_: IngestInput, context) -> DocumentBundle:
 del context
 if input_.path is None:
 raise ValueError("Example normalizer requires a local path.")
 source_uri = input_.source_uri or input_.target
 text = input_.path.read_text(encoding="utf-8")
 return DocumentBundle(
 id=source_uri,
 title="Example Text",
 source_uri=source_uri,
 blocks=[TextBlock(text)],
 metadata={
 "display_name": input_.metadata.get("display_name", input_.path.name),
 "source_path": input_.path.as_posix(),
 "source_identity": source_uri,
 },
 provenance=[ProvenanceRecord(source_uri=source_uri)],
 )

11 changes: 11 additions & 0 deletions
examples/ingest-plugin/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"

63 changes: 53 additions & 10 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
 ".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
 ".png",
 ".jpg",
 ".jpeg",
 ".gif",
 ".webp",
 ".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
 "long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
 help="Import an already-indexed PageIndex Cloud document by its doc-id "
 "(no local file). Mutually exclusive with PATH.",
)
@click.option(
 "--ingest-pipeline",
 "ingest_pipeline",
 type=click.Choice(["legacy", "bundle"]),
 default=None,
 help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
 """Add a document or directory of documents at PATH to the knowledge base.

 PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
 return

 # URL ingest: download into raw/ first, then call add_single_file explicitly.
 # Keep staged conversion enabled so converted source artifacts do not touch
 # the live KB before the mutation snapshot exists. The tri-state outcome
 # still lets us clean up the just-downloaded raw file on dedup.
 from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

 config = load_config(kb_dir / ".openkb" / "config.yaml")
 pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
 supported_extensions = (
 BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
 )

 # Legacy URL ingest downloads into raw/ first, then calls add_single_file.
 # Bundle URL ingest keeps downloads inside the bundle staging directory and
 # commits through the same mutation boundary as local bundle files.
 from openkb.url_ingest import looks_like_url, fetch_url_to_raw

 if looks_like_url(path):
 if pipeline == "bundle":
 add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
 return
 fetched = fetch_url_to_raw(path, kb_dir)
 if fetched is None:
 return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
 files = [
 f
 for f in sorted(target.rglob("*"))
 if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
 if f.is_file() and f.suffix.lower() in supported_extensions
 ]
 if not files:
 click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo(f"Found {total} supported file(s) in {path}.")
 for i, f in enumerate(files, 1):
 click.echo(f"\n[{i}/{total}] ", nl=False)
 add_single_file(f, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(f, kb_dir)
 else:
 if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
 if target.suffix.lower() not in supported_extensions:
 click.echo(
 f"Unsupported file type: {target.suffix}. "
 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
 f"Supported: {', '.join(sorted(supported_extensions))}"
 )
 return
 add_single_file(target, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 )
 )

 bundle_path = None
 if meta.get("bundle_path"):
 candidate = kb_dir / meta["bundle_path"]
 if candidate.exists():
 bundle_path = candidate
 actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

 # Scan concept pages to predict which will be edited vs. deleted.
 # Only frontmatter ``sources:`` membership drives the plan — body-only
 # references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 source_json.unlink(missing_ok=True)
 if images_dir.is_dir():
 shutil.rmtree(images_dir, ignore_errors=True)
 if bundle_path is not None:
 bundle_path.unlink(missing_ok=True)

 concept_result = remove_doc_from_concept_pages(
 wiki_dir,
Expand Down

7 changes: 7 additions & 0 deletions
openkb/ingest/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
})();
Add pluggable ingest bundle pipeline by specode · Pull Request #179 · VectifyAI/OpenKB · GitHub
Skip to content
Add pluggable ingest bundle pipeline#179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

specode
wants to merge
1
commit into

base:main
Choose a base branch

from

 Open


Add pluggable ingest bundle pipeline
#179
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


27 changes: 27 additions & 0 deletions
examples/ingest-plugin/README.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
# OpenKB Ingest Plugin Example

This directory shows the minimal shape of an external package that contributes
bundle ingest components through Python entry points.

Install a package like this in the same environment as OpenKB, then enable its
components in `.openkb/config.yaml`:

```yaml
ingest:
 pipeline: bundle
 importers:
 enabled:
 - example_text
 normalizers:
 enabled:
 - example_text
```

The entry point groups are:

- `openkb.ingest.importers`
- `openkb.ingest.normalizers`
- `openkb.ingest.enrichers`

Each entry point should resolve to a class or factory returning an object that
matches the corresponding OpenKB ingest protocol.

50 changes: 50 additions & 0 deletions
examples/ingest-plugin/openkb_example_ingest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
from __future__ import annotations

from openkb.ingest.models import DocumentBundle, IngestInput, ProvenanceRecord, TextBlock


class ExampleTextImporter:
 name = "example_text"

 def can_handle(self, target: str, context) -> bool:
 del context
 return target.startswith("example-text:")

 def import_source(self, target: str, context) -> IngestInput:
 path = context.staging_dir / "raw" / "example.txt"
 path.parent.mkdir(parents=True, exist_ok=True)
 path.write_text(target.removeprefix("example-text:").strip() + "\n", encoding="utf-8")
 return IngestInput(
 target=target,
 path=path,
 source_uri=target,
 media_type="text/x-example",
 metadata={"display_name": "example.txt"},
 )


class ExampleTextNormalizer:
 name = "example_text"

 def supports(self, input_: IngestInput, context) -> bool:
 del context
 return input_.media_type == "text/x-example"

 def normalize(self, input_: IngestInput, context) -> DocumentBundle:
 del context
 if input_.path is None:
 raise ValueError("Example normalizer requires a local path.")
 source_uri = input_.source_uri or input_.target
 text = input_.path.read_text(encoding="utf-8")
 return DocumentBundle(
 id=source_uri,
 title="Example Text",
 source_uri=source_uri,
 blocks=[TextBlock(text)],
 metadata={
 "display_name": input_.metadata.get("display_name", input_.path.name),
 "source_path": input_.path.as_posix(),
 "source_identity": source_uri,
 },
 provenance=[ProvenanceRecord(source_uri=source_uri)],
 )

11 changes: 11 additions & 0 deletions
examples/ingest-plugin/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
[project]
name = "openkb-example-ingest-plugin"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = ["openkb"]

[project.entry-points."openkb.ingest.importers"]
example_text = "openkb_example_ingest:ExampleTextImporter"

[project.entry-points."openkb.ingest.normalizers"]
example_text = "openkb_example_ingest:ExampleTextNormalizer"

63 changes: 53 additions & 10 deletions
openkb/cli.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -240,6 +240,17 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None:
 ".csv",
}

BUNDLE_ONLY_EXTENSIONS = {
 ".png",
 ".jpg",
 ".jpeg",
 ".gif",
 ".webp",
 ".bmp",
}

BUNDLE_SUPPORTED_EXTENSIONS = SUPPORTED_EXTENSIONS | BUNDLE_ONLY_EXTENSIONS

# Map raw doc types to display types
_TYPE_DISPLAY_MAP = {
 "long_pdf": "pageindex",
Expand DownExpand Up@@ -987,9 +998,16 @@ def init(model, language):
 help="Import an already-indexed PageIndex Cloud document by its doc-id "
 "(no local file). Mutually exclusive with PATH.",
)
@click.option(
 "--ingest-pipeline",
 "ingest_pipeline",
 type=click.Choice(["legacy", "bundle"]),
 default=None,
 help="Select the add ingest pipeline. Defaults to .openkb/config.yaml ingest.pipeline, then legacy.",
)
@click.pass_context
@_with_kb_lock(exclusive=True)
def add(ctx, path, from_pageindex_cloud):
def add(ctx, path, from_pageindex_cloud, ingest_pipeline):
 """Add a document or directory of documents at PATH to the knowledge base.

 PATH may be a local file, a local directory (which is walked
Expand DownExpand Up@@ -1021,13 +1039,23 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
 return

 # URL ingest: download into raw/ first, then call add_single_file explicitly.
 # Keep staged conversion enabled so converted source artifacts do not touch
 # the live KB before the mutation snapshot exists. The tri-state outcome
 # still lets us clean up the just-downloaded raw file on dedup.
 from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

 config = load_config(kb_dir / ".openkb" / "config.yaml")
 pipeline = resolve_ingest_pipeline(config, ingest_pipeline)
 supported_extensions = (
 BUNDLE_SUPPORTED_EXTENSIONS if pipeline == "bundle" else SUPPORTED_EXTENSIONS
 )

 # Legacy URL ingest downloads into raw/ first, then calls add_single_file.
 # Bundle URL ingest keeps downloads inside the bundle staging directory and
 # commits through the same mutation boundary as local bundle files.
 from openkb.url_ingest import looks_like_url, fetch_url_to_raw

 if looks_like_url(path):
 if pipeline == "bundle":
 add_bundle_target(path, kb_dir, legacy_fallback=add_single_file)
 return
 fetched = fetch_url_to_raw(path, kb_dir)
 if fetched is None:
 return
Expand All@@ -1049,7 +1077,7 @@ def add(ctx, path, from_pageindex_cloud):
 files = [
 f
 for f in sorted(target.rglob("*"))
 if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
 if f.is_file() and f.suffix.lower() in supported_extensions
 ]
 if not files:
 click.echo(f"No supported files found in {path}.")
Expand All@@ -1058,15 +1086,21 @@ def add(ctx, path, from_pageindex_cloud):
 click.echo(f"Found {total} supported file(s) in {path}.")
 for i, f in enumerate(files, 1):
 click.echo(f"\n[{i}/{total}] ", nl=False)
 add_single_file(f, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(f, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(f, kb_dir)
 else:
 if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
 if target.suffix.lower() not in supported_extensions:
 click.echo(
 f"Unsupported file type: {target.suffix}. "
 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
 f"Supported: {', '.join(sorted(supported_extensions))}"
 )
 return
 add_single_file(target, kb_dir)
 if pipeline == "bundle":
 add_bundle_target(target, kb_dir, legacy_fallback=add_single_file)
 else:
 add_single_file(target, kb_dir)


def _stream_to_tty() -> bool:
Expand DownExpand Up@@ -1309,6 +1343,13 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 )
 )

 bundle_path = None
 if meta.get("bundle_path"):
 candidate = kb_dir / meta["bundle_path"]
 if candidate.exists():
 bundle_path = candidate
 actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

 # Scan concept pages to predict which will be edited vs. deleted.
 # Only frontmatter ``sources:`` membership drives the plan — body-only
 # references (e.g. a stray ``See also:`` line a user added by hand
Expand DownExpand Up@@ -1424,6 +1465,8 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
 source_json.unlink(missing_ok=True)
 if images_dir.is_dir():
 shutil.rmtree(images_dir, ignore_errors=True)
 if bundle_path is not None:
 bundle_path.unlink(missing_ok=True)

 concept_result = remove_doc_from_concept_pages(
 wiki_dir,
Expand Down

7 changes: 7 additions & 0 deletions
openkb/ingest/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
"""Pluggable ingest pipeline for OpenKB."""

from __future__ import annotations

from openkb.ingest.add import add_bundle_target, resolve_ingest_pipeline

__all__ = ["add_bundle_target", "resolve_ingest_pipeline"]
Loading