Skip to content

fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rejojer merged 5 commits into from
Apr 10, 2026
Merged

fix: compiler concept update bugs #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8818ada
fix: update existing concept briefs in index.md instead of skipping
rejojer Apr 10, 2026
aabcf5f
fix: preserve non-ASCII characters in concept name slugs
rejojer Apr 10, 2026
9df6e6c
fix: always replace concept body on update, not only when source is new
rejojer Apr 10, 2026
ef235d2
Fix concept index updates by section
rejojer Apr 10, 2026
ed0d6ba
Fix exact concept index row matching
rejojer Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 81 additions & 27 deletions openkb/agent/compiler.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
"""Return the [start, end) bounds for a Markdown H2 section."""
for i, line in enumerate(lines):
if line == heading:
start = i + 1
end = len(lines)
for j in range(start, len(lines)):
if lines[j].startswith("## "):
end = j
break
return start, end
return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
"""Check whether an index entry already exists inside the named section."""
bounds = _get_section_bounds(lines, heading)
if bounds is None:
return False

start, end = bounds
entry_prefix = f"- {link}"
return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
"""Replace the first matching entry within a specific section."""
bounds = _get_section_bounds(lines, heading)
if bounds is None:
return False

start, end = bounds
entry_prefix = f"- {link}"
for i in range(start, end):
if lines[i].startswith(entry_prefix):
lines[i] = entry
return True
return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
"""Insert a new entry at the top of a specific section."""
bounds = _get_section_bounds(lines, heading)
if bounds is None:
return False

start, _ = bounds
lines.insert(start, entry)
return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
(summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
"""Sanitize a concept name for safe use as a filename."""
name = unicodedata.normalize("NFKC", name)
sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
existing = fm + body
else:
existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
# Strip frontmatter from LLM content to avoid duplicate blocks
clean = content
if clean.startswith("---"):
end = clean.find("---", 3)
if end != -1:
clean = clean[end + 3:].lstrip("\n")
# Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
if existing.startswith("---"):
end = existing.find("---", 3)
if end != -1:
existing = existing[:end + 3] + "\n\n" + clean
else:
existing = clean
# Strip frontmatter from LLM content to avoid duplicate blocks
clean = content
if clean.startswith("---"):
end = clean.find("---", 3)
if end != -1:
clean = clean[end + 3:].lstrip("\n")
# Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
if existing.startswith("---"):
end = existing.find("---", 3)
if end != -1:
existing = existing[:end + 3] + "\n\n" + clean
else:
existing = clean
else:
existing = clean
if brief and existing.startswith("---"):
end = existing.find("---", 3)
if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
text += f"\n\n## Related Documents\n- {link}\n"
path.write_text(text, encoding="utf-8")


def _update_index(
wiki_dir: Path, doc_name: str, concept_names: list[str],
doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
"""Append document and concept entries to index.md.

When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
are written as ``- [[link]] (type) — brief text``. Existing entries are
detected by the link part only and skipped to avoid duplicates.
are written as ``- [[link]] (type) — brief text``. Existing entries are
detected within their own section by exact entry prefix and skipped to
avoid duplicates.
``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
query agent knows how to access detailed content.
"""
Expand All@@ -482,26 +535,27 @@ def _update_index(
encoding="utf-8",
)

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

doc_link = f"[[summaries/{doc_name}]]"
if doc_link not in text:
if not _section_contains_link(lines, "## Documents", doc_link):
doc_entry = f"- {doc_link} ({doc_type})"
if doc_brief:
doc_entry += f" — {doc_brief}"
if "## Documents" in text:
text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
_insert_section_entry(lines, "## Documents", doc_entry)

for name in concept_names:
concept_link = f"[[concepts/{name}]]"
if concept_link not in text:
concept_entry = f"- {concept_link}"
concept_entry = f"- {concept_link}"
if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
if _section_contains_link(lines, "## Concepts", concept_link):
if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
if "## Concepts" in text:
text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

index_path.write_text(text, encoding="utf-8")
index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down
102 changes: 102 additions & 0 deletions tests/test_compiler.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
compile_short_doc,
_compile_concepts,
_parse_json,
_sanitize_concept_name,
_write_summary,
_write_concept,
_update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
_parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
def test_ascii_passthrough(self):
assert _sanitize_concept_name("hello-world") == "hello-world"

def test_spaces_replaced(self):
assert _sanitize_concept_name("hello world") == "hello-world"

def test_chinese(self):
result = _sanitize_concept_name("注意力机制")
assert result == "注意力机制"

def test_japanese(self):
result = _sanitize_concept_name("トランスフォーマー")
assert result == "トランスフォーマー"

def test_french_accents(self):
result = _sanitize_concept_name("réseau neuronal")
assert "r" in result
assert result != "r-seau-neuronal" # accented chars preserved, not stripped

def test_distinct_chinese_names_no_collision(self):
a = _sanitize_concept_name("注意力机制")
b = _sanitize_concept_name("变压器模型")
assert a != b

def test_empty_fallback(self):
assert _sanitize_concept_name("!!!") == "unnamed-concept"

def test_nfkc_normalization(self):
# U+FF21 (fullwidth A) should normalize to regular A
assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
def test_writes_with_frontmatter(self, tmp_path):
wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
assert "[[concepts/attention]] — Focus mechanism" in text
assert "[[concepts/transformer]] — NN architecture" in text

def test_updates_only_exact_concept_row(self, tmp_path):
wiki = tmp_path / "wiki"
wiki.mkdir()
(wiki / "index.md").write_text(
"# Index\n\n## Documents\n\n## Concepts\n"
"- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
"- [[concepts/attention]] — Old brief\n\n## Explorations\n",
encoding="utf-8",
)
_update_index(
wiki,
"my-doc",
["attention"],
concept_briefs={"attention": "New brief"},
)
text = (wiki / "index.md").read_text()
assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
assert "- [[concepts/attention]] — New brief" in text
assert text.count("[[concepts/attention]] — New brief") == 1

def test_no_duplicates(self, tmp_path):
wiki = tmp_path / "wiki"
wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
assert "[[summaries/my-doc]]" in text
assert "[[concepts/attention]]" in text

def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
wiki = tmp_path / "wiki"
wiki.mkdir()
(wiki / "index.md").write_text(
"# Index\n\n"
"## Documents\n"
"- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
"## Concepts\n"
"- [[concepts/attention]] — Old brief\n\n"
"## Explorations\n",
encoding="utf-8",
)

_update_index(
wiki,
"my-doc",
["attention"],
concept_briefs={"attention": "New brief"},
)

text = (wiki / "index.md").read_text()
assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
assert "- [[concepts/attention]] — New brief" in text
assert "- [[concepts/attention]] — Old brief" not in text

def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
wiki = tmp_path / "wiki"
wiki.mkdir()
(wiki / "index.md").write_text(
"# Index\n\n"
"## Documents\n"
"- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
"## Concepts\n\n"
"## Explorations\n",
encoding="utf-8",
)

_update_index(
wiki,
"my-doc",
["attention"],
concept_briefs={"attention": "New brief"},
)

text = (wiki / "index.md").read_text()
assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
def test_empty_wiki(self, tmp_path):
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
fix: compiler concept update bugs by rejojer · Pull Request #11 · VectifyAI/OpenKB · GitHub
Skip to content
fix: compiler concept update bugs#11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

rejojer
merged 5 commits into

from

Apr 10, 2026
 Merged


fix: compiler concept update bugs
#11
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
8818ada

fix: update existing concept briefs in index.md instead of skipping


rejojer Apr 10, 2026
aabcf5f

fix: preserve non-ASCII characters in concept name slugs


rejojer Apr 10, 2026
9df6e6c

fix: always replace concept body on update, not only when source is new


rejojer Apr 10, 2026
ef235d2

Fix concept index updates by section


rejojer Apr 10, 2026
ed0d6ba

Fix exact concept index row matching


rejojer Apr 10, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


108 changes: 81 additions & 27 deletions
openkb/agent/compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
import sys
import threading
import time
import unicodedata
from pathlib import Path

import litellm
Expand DownExpand Up@@ -283,6 +284,57 @@ def _read_concept_briefs(wiki_dir: Path) -> str:
 return "\n".join(lines) or "(none yet)"


def _get_section_bounds(lines: list[str], heading: str) -> tuple[int, int] | None:
 """Return the [start, end) bounds for a Markdown H2 section."""
 for i, line in enumerate(lines):
 if line == heading:
 start = i + 1
 end = len(lines)
 for j in range(start, len(lines)):
 if lines[j].startswith("## "):
 end = j
 break
 return start, end
 return None


def _section_contains_link(lines: list[str], heading: str, link: str) -> bool:
 """Check whether an index entry already exists inside the named section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 return any(line.startswith(entry_prefix) for line in lines[start:end])


def _replace_section_entry(lines: list[str], heading: str, link: str, entry: str) -> bool:
 """Replace the first matching entry within a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, end = bounds
 entry_prefix = f"- {link}"
 for i in range(start, end):
 if lines[i].startswith(entry_prefix):
 lines[i] = entry
 return True
 return False


def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
 """Insert a new entry at the top of a specific section."""
 bounds = _get_section_bounds(lines, heading)
 if bounds is None:
 return False

 start, _ = bounds
 lines.insert(start, entry)
 return True



def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 doc_type: str = "short") -> None:
Expand All@@ -302,11 +354,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
 (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")


_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
_SAFE_NAME_RE = re.compile(r'[^\w\-]')


def _sanitize_concept_name(name: str) -> str:
 """Sanitize a concept name for safe use as a filename."""
 name = unicodedata.normalize("NFKC", name)
 sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
 return sanitized or "unnamed-concept"

Expand DownExpand Up@@ -336,21 +389,21 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
 existing = fm + body
 else:
 existing = f"---\nsources: [{source_file}]\n---\n\n" + existing
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 # Strip frontmatter from LLM content to avoid duplicate blocks
 clean = content
 if clean.startswith("---"):
 end = clean.find("---", 3)
 if end != -1:
 clean = clean[end + 3:].lstrip("\n")
 # Replace body with LLM rewrite (prompt asks for full rewrite, not delta)
 if existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
 existing = existing[:end + 3] + "\n\n" + clean
 else:
 existing = clean
 else:
 existing = clean
 if brief and existing.startswith("---"):
 end = existing.find("---", 3)
 if end != -1:
Expand DownExpand Up@@ -458,7 +511,6 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
 text += f"\n\n## Related Documents\n- {link}\n"
 path.write_text(text, encoding="utf-8")


def _update_index(
 wiki_dir: Path, doc_name: str, concept_names: list[str],
 doc_brief: str = "", concept_briefs: dict[str, str] | None = None,
Expand All@@ -467,8 +519,9 @@ def _update_index(
 """Append document and concept entries to index.md.

 When ``doc_brief`` or entries in ``concept_briefs`` are provided, entries
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected by the link part only and skipped to avoid duplicates.
 are written as ``- [[link]] (type) — brief text``. Existing entries are
 detected within their own section by exact entry prefix and skipped to
 avoid duplicates.
 ``doc_type`` is ``"short"`` or ``"pageindex"`` — shown in the entry so the
 query agent knows how to access detailed content.
 """
Expand All@@ -482,26 +535,27 @@ def _update_index(
 encoding="utf-8",
 )

text = index_path.read_text(encoding="utf-8")
lines = index_path.read_text(encoding="utf-8").split("\n")

 doc_link = f"[[summaries/{doc_name}]]"
 if doc_link not in text:
 if not _section_contains_link(lines, "## Documents", doc_link):
 doc_entry = f"- {doc_link} ({doc_type})"
 if doc_brief:
 doc_entry += f" — {doc_brief}"
 if "## Documents" in text:
 text = text.replace("## Documents\n", f"## Documents\n{doc_entry}\n", 1)
 _insert_section_entry(lines, "## Documents", doc_entry)

 for name in concept_names:
 concept_link = f"[[concepts/{name}]]"
 if concept_link not in text:
 concept_entry = f"- {concept_link}"
 concept_entry = f"- {concept_link}"
 if name in concept_briefs:
 concept_entry += f" — {concept_briefs[name]}"
 if _section_contains_link(lines, "## Concepts", concept_link):
 if name in concept_briefs:
concept_entry += f" — {concept_briefs[name]}"
 if "## Concepts" in text:
 text = text.replace("## Concepts\n", f"## Concepts\n{concept_entry}\n", 1)
_replace_section_entry(lines, "## Concepts", concept_link, concept_entry)
else:
_insert_section_entry(lines, "## Concepts", concept_entry)

 index_path.write_text(text, encoding="utf-8")
 index_path.write_text("\n".join(lines), encoding="utf-8")


# ---------------------------------------------------------------------------
Expand Down

102 changes: 102 additions & 0 deletions
tests/test_compiler.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,6 +12,7 @@
 compile_short_doc,
 _compile_concepts,
 _parse_json,
 _sanitize_concept_name,
 _write_summary,
 _write_concept,
 _update_index,
Expand DownExpand Up@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
 _parse_json("Just plain markdown text without JSON")


class TestSanitizeConceptName:
 def test_ascii_passthrough(self):
 assert _sanitize_concept_name("hello-world") == "hello-world"

 def test_spaces_replaced(self):
 assert _sanitize_concept_name("hello world") == "hello-world"

 def test_chinese(self):
 result = _sanitize_concept_name("注意力机制")
 assert result == "注意力机制"

 def test_japanese(self):
 result = _sanitize_concept_name("トランスフォーマー")
 assert result == "トランスフォーマー"

 def test_french_accents(self):
 result = _sanitize_concept_name("réseau neuronal")
 assert "r" in result
 assert result != "r-seau-neuronal" # accented chars preserved, not stripped

 def test_distinct_chinese_names_no_collision(self):
 a = _sanitize_concept_name("注意力机制")
 b = _sanitize_concept_name("变压器模型")
 assert a != b

 def test_empty_fallback(self):
 assert _sanitize_concept_name("!!!") == "unnamed-concept"

 def test_nfkc_normalization(self):
 # U+FF21 (fullwidth A) should normalize to regular A
 assert _sanitize_concept_name("\uff21\uff22") == "AB"


class TestWriteSummary:
 def test_writes_with_frontmatter(self, tmp_path):
 wiki = tmp_path / "wiki"
Expand DownExpand Up@@ -164,6 +198,26 @@ def test_appends_entries_with_briefs(self, tmp_path):
 assert "[[concepts/attention]] — Focus mechanism" in text
 assert "[[concepts/transformer]] — NN architecture" in text

 def test_updates_only_exact_concept_row(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n## Documents\n\n## Concepts\n"
 "- [[concepts/transformer]] — Uses [[concepts/attention]] internally\n"
 "- [[concepts/attention]] — Old brief\n\n## Explorations\n",
 encoding="utf-8",
 )
 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )
 text = (wiki / "index.md").read_text()
 assert "- [[concepts/transformer]] — Uses [[concepts/attention]] internally" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert text.count("[[concepts/attention]] — New brief") == 1

 def test_no_duplicates(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
Expand All@@ -187,6 +241,54 @@ def test_backwards_compat_no_briefs(self, tmp_path):
 assert "[[summaries/my-doc]]" in text
 assert "[[concepts/attention]]" in text

 def test_updates_concept_brief_only_inside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n"
 "- [[concepts/attention]] — Old brief\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text
 assert "- [[concepts/attention]] — Old brief" not in text

 def test_adds_concept_entry_when_link_exists_outside_concepts_section(self, tmp_path):
 wiki = tmp_path / "wiki"
 wiki.mkdir()
 (wiki / "index.md").write_text(
 "# Index\n\n"
 "## Documents\n"
 "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here\n\n"
 "## Concepts\n\n"
 "## Explorations\n",
 encoding="utf-8",
 )

 _update_index(
 wiki,
 "my-doc",
 ["attention"],
 concept_briefs={"attention": "New brief"},
 )

 text = (wiki / "index.md").read_text()
 assert "- [[summaries/my-doc]] (short) — Mentions [[concepts/attention]] here" in text
 assert "- [[concepts/attention]] — New brief" in text


class TestReadWikiContext:
 def test_empty_wiki(self, tmp_path):
Expand Down