feat(cli): add 'add-all' command and auto_delete_added_files config

openkb/cli.py

@@ -452,6 +452,59 @@ def add_single_file(
  return _add_single_file_locked(file_path, kb_dir, stage=stage, bundle=bundle)
 
 
+def _delete_if_auto_cleanup_enabled(
+ file_path: Path, status: Literal["added", "skipped", "failed"], config: dict
+) -> bool:
+ """Delete file if auto_delete_added_files is enabled and ingestion succeeded/skipped.
+
+ Deletes on both "added" (successful ingestion) and "skipped" (duplicate already
+ in KB) to keep raw/ directory clean. Preserves files on "failed" to allow retries.
+
+ Args:
+ file_path: Path to the file to potentially delete.
+ status: Result status from add_single_file ("added", "skipped", or "failed").
+ config: Configuration dict (typically from resolve_effective_config).
+
+ Returns:
+ True if file was deleted, False otherwise.
+ """
+ if status in ("added", "skipped") and config.get("auto_delete_added_files", False):
+ try:
+ file_path.unlink(missing_ok=True)
+ return True
+ except Exception as exc:
+ logger.warning(f"Failed to delete {file_path.name}: {exc}")
+ return False
+ return False
+
+
+def _cleanup_empty_directories(start_dir: Path) -> int:
+ """Recursively delete empty directories under start_dir.
+
+ Walks from deepest subdirectories up, deleting directories that become
+ empty after file cleanup.
+
+ Args:
+ start_dir: Root directory to clean up (e.g., kb_dir / "raw").
+
+ Returns:
+ Number of directories deleted.
+ """
+ deleted_count = 0
+ try:
+ for directory in sorted(start_dir.rglob("*"), key=lambda p: len(p.parts), reverse=True):
+ if directory.is_dir() and directory != start_dir:
+ try:
+ if not list(directory.iterdir()):
+ directory.rmdir()
+ deleted_count += 1
+ except OSError:
+ pass
+ except Exception as exc:
+ logger.warning(f"Error during directory cleanup: {exc}")
+ return deleted_count
+
+
 def _add_single_file_locked(
  file_path: Path, kb_dir: Path, *, stage: bool = True, bundle=None
 ) -> Literal["added", "skipped", "failed"]:
@@ -1086,6 +1139,10 @@ def add(ctx, path, from_pageindex_cloud):
  Alternatively, pass --from-pageindex-cloud <DOC_ID> to import a document
  that is already indexed in PageIndex Cloud, with no local file. Requires
  the PAGEINDEX_API_KEY environment variable.
+
+ If ``auto_delete_added_files`` is enabled in config.yaml, files are
+ automatically deleted after ingestion (both on successful addition and
+ on skip/duplicate).
  """
  kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
  if kb_dir is None:
@@ -1106,6 +1163,8 @@ def add(ctx, path, from_pageindex_cloud):
  click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
  return
 
+ config = resolve_effective_config(kb_dir)[0]
+
  # URL ingest: download into raw/ first, then call add_single_file explicitly.
  # Keep staged conversion enabled so converted source artifacts do not touch
  # the live KB before the mutation snapshot exists. The tri-state outcome
@@ -1123,6 +1182,8 @@ def add(ctx, path, from_pageindex_cloud):
  # indexing has already succeeded but compilation didn't.
  if outcome == "skipped":
  fetched.unlink(missing_ok=True)
+ else:
+ _delete_if_auto_cleanup_enabled(fetched, outcome, config)
  return
 
  target = Path(path)
@@ -1143,15 +1204,77 @@ def add(ctx, path, from_pageindex_cloud):
  click.echo(f"Found {total} supported file(s) in {path}.")
  for i, f in enumerate(files, 1):
  click.echo(f"\n[{i}/{total}] ", nl=False)
- add_single_file(f, kb_dir)
+ outcome = add_single_file(f, kb_dir)
+ _delete_if_auto_cleanup_enabled(f, outcome, config)
  else:
  if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
  click.echo(
  f"Unsupported file type: {target.suffix}. "
  f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
  )
  return
- add_single_file(target, kb_dir)
+ outcome = add_single_file(target, kb_dir)
+ _delete_if_auto_cleanup_enabled(target, outcome, config)
+
+
+@cli.command()
+@click.pass_context
+@_with_kb_lock(exclusive=True)
+def add_all(ctx):
+ """Process all files in the ``raw/`` directory and add them to the knowledge base.
+
+ This command walks the ``raw/`` directory recursively for all supported
+ document types and ingests them into the KB. If ``auto_delete_added_files``
+ is enabled in config.yaml, files are automatically deleted after ingestion
+ (both on successful addition and on skip/duplicate), and empty subdirectories
+ are cleaned up.
+
+ Returns a summary of the operation (added, skipped, failed, deleted counts).
+ """
+ kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
+ if kb_dir is None:
+ click.echo("No knowledge base found. Run `openkb init` first.")
+ return
+
+ raw_dir = kb_dir / "raw"
+ if not raw_dir.is_dir():
+ click.echo(f"No raw/ directory found at {raw_dir}")
+ return
+
+ files = [
+ f
+ for f in sorted(raw_dir.rglob("*"))
+ if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
+ ]
+ if not files:
+ click.echo("No supported files found in raw/ directory.")
+ return
+
+ config = resolve_effective_config(kb_dir)[0]
+ total = len(files)
+ added = skipped = failed = deleted = dirs_deleted = 0
+
+ click.echo(f"Processing {total} file(s) from raw/ directory...")
+ for i, f in enumerate(files, 1):
+ click.echo(f"\n[{i}/{total}] ", nl=False)
+ outcome = add_single_file(f, kb_dir)
+ if outcome == "added":
+ added += 1
+ elif outcome == "skipped":
+ skipped += 1
+ else:
+ failed += 1
+ if _delete_if_auto_cleanup_enabled(f, outcome, config):
+ deleted += 1
+
+ # Clean up empty subdirectories if auto-cleanup is enabled
+ if config.get("auto_delete_added_files", False):
+ dirs_deleted = _cleanup_empty_directories(raw_dir)
+
+ summary = f"Added: {added}, Skipped: {skipped}, Failed: {failed}, Deleted: {deleted}"
+ if dirs_deleted > 0:
+ summary += f", Empty dirs cleaned: {dirs_deleted}"
+ click.echo(f"\n\nSummary: {summary}")
 
 
 def _stream_to_tty() -> bool:

openkb/config.py

@@ -36,6 +36,7 @@
  # global/KB list overrides it wholesale; resolve_entity_types cleans the
  # effective value on read.
  "entity_types": list(DEFAULT_ENTITY_TYPES),
+ "auto_delete_added_files": False,
 }
 
 GLOBAL_CONFIG_DIR = Path.home() / ".config" / "openkb"