') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); feat(cli): add 'add-all' command and auto_delete_added_files config by sebastianbraun25 · Pull Request #232 · VectifyAI/OpenKB · GitHub
Skip to content

feat(cli): add 'add-all' command and auto_delete_added_files config - #232

Open
sebastianbraun25 wants to merge 3 commits into
VectifyAI:mainfrom
sebastianbraun25:feat/issue-231-add-all-command
Open

feat(cli): add 'add-all' command and auto_delete_added_files config#232
sebastianbraun25 wants to merge 3 commits into
VectifyAI:mainfrom
sebastianbraun25:feat/issue-231-add-all-command

Conversation

@sebastianbraun25

@sebastianbraun25sebastianbraun25 commented Aug 27, 2026•
edited
Loading

Copy link
Copy Markdown

Note

This PR was created in collaboration between a human and AI: implementation, tests, and
PR text were created by an AI assistant under the guidance and review of the human author.

Problem

The openkb add command supports adding directories recursively, but users who frequently download documents to /raw must manually invoke openkb add raw/ and then manually delete processed files. There is no batch processing command for the /raw staging directory, and no configuration option to auto-delete successfully ingested files and duplicates.

Solution / Changes

New CLI command:

  • openkb add-all processes all files in the KB's raw/ directory recursively
  • Ingests all supported file types (PDF, Markdown, DOCX, PPTX, XLSX, XLS, HTML, TXT, CSV)
  • Returns summary of operation: Added, Skipped, Failed, Deleted counts

New configuration parameter:

  • auto_delete_added_files (boolean, default: false)
  • When enabled, both add and add-all automatically delete files after processing
  • Deletes on both "added" (successful ingestion) and "skipped" (duplicate/already in KB)
  • Preserves "failed" files to allow user retries
  • Applies to all ingest methods: direct files, directories, and URLs

Implementation:

  • Added _delete_if_auto_cleanup_enabled() helper function to handle cleanup logic
  • Updated both add and add-all commands to use the helper function
  • Updated docstrings to document cleanup behavior on both success and skip cases

Issues

@sebastianbraun25
sebastianbraun25force-pushed the feat/issue-231-add-all-command branch from eecc0bc to 680c04bCompareAugust 28, 2026 14:14
@sebastianbraun25

sebastianbraun25 commented Aug 28, 2026

Copy link
Copy Markdown
Author

Rewrote this branch's history to remove unrelated commits.

The branch had accidentally picked up commits from the (now-withdrawn, see
#229/#230) retry-timeout work as ancestors — likely from being built on top
of a local integration branch at the time rather than directly off
main. That made this PR's diff show unrelated changes to
openkb/agent/compiler.py and a new tests/test_compiler_retry.py
that have nothing to do with add-all/auto-delete.

Rebuilt cleanly from current main with only the two commits that
actually belong to this feature (add-all command + the
auto-delete-on-skipped follow-up fix). Diff now only touches
openkb/cli.py and openkb/config.py as expected. No functional
changes to the add-all feature itself — force-pushed the same branch name so
this PR stays open with its history/comments intact.

@sebastianbraun25
sebastianbraun25force-pushed the feat/issue-231-add-all-command branch from 0885647 to 153149cCompareAugust 31, 2026 08:46
Sebastian Braun added 3 commits August 31, 2026 14:27
Sebastian Braun
feat(cli): add 'add-all' command and auto_delete_added_files config o…
e7a7910
…ption
- New 'openkb add-all' command processes all files in raw/ directory
- New config parameter 'auto_delete_added_files' (default: false)
- When enabled, both 'add' and 'add-all' automatically delete successfully ingested files
- Updated help texts to document the new cleanup behavior
- Config applies to all ingest methods: direct files, directories, and URLs
Sebastian Braun
fix(cli): auto-delete also on 'skipped' status to keep raw/ clean
fa0c3fe
Duplicates (skipped files) should also be auto-deleted when auto_delete_added_files
is enabled, so raw/ stays clean. Only 'failed' status files are preserved to allow
retries. Updated docstrings and helper function logic accordingly.
Sebastian Braun
feat(cli): add empty subdirectory cleanup to add-all command
a57bcd9
When auto_delete_added_files is enabled, the add-all command now recursively
cleans up empty subdirectories in raw/ after file deletion. Directories are
deleted from deepest to shallowest to ensure proper cleanup. Added summary
output showing count of cleaned directories.
@sebastianbraun25
sebastianbraun25force-pushed the feat/issue-231-add-all-command branch from 153149c to a57bcd9CompareAugust 31, 2026 12:31
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

No reviews

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

feat(cli): add-all command and auto_delete_added_files config

1 participant

@sebastianbraun25