From 4d84ffee2dcba54cbb9cf9826873d7061c1956d7 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Thu, 20 Aug 2026 18:22:02 +0800 Subject: [PATCH] 0.2.0: direct .imp DSL engine (replaces pre-compiled Python map modules) - parser.py: .imp DSL parser (metadata, tests, dependencies, stages, parallels) - expr.py: expression layer (any/range/list/maybe/anchors/concat/guards) - engine.py: executor with longest-match parallels, word-aware casing, dependency-aliased dotted run targets - 98/98 on un-bul; 206/242 on un-ell (posix guards pending) - No monorepo bootstrap required; point add_load_path at the map corpus --- README.adoc | 106 ++++++++------- pyproject.toml | 23 ++-- src/interscript/__init__.py | 12 +- src/interscript/engine.py | 167 +++++++++++++++++++++++ src/interscript/expr.py | 115 ++++++++++++++++ src/interscript/interscript.py | 79 +++++++---- src/interscript/parser.py | 240 +++++++++++++++++++++++++++++++++ tests/test_engine.py | 89 ++++++++++++ 8 files changed, 750 insertions(+), 81 deletions(-) create mode 100644 src/interscript/engine.py create mode 100644 src/interscript/expr.py create mode 100644 src/interscript/parser.py create mode 100644 tests/test_engine.py diff --git a/README.adoc b/README.adoc index 1f133cc..adc4dc7 100644 --- a/README.adoc +++ b/README.adoc @@ -2,72 +2,84 @@ == Purpose -This repository contains code for the Interscript Python runtime ("Interscript-Python"). +The official Python runtime for Interscript — deterministic transliteration +over the interscript map corpus (300+ authority-backed systems: BGN/PCGN, +ISO, UN, ALA-LC, ODNI, ICAO, DIN, and others). -This software allows performing script conversions by using the -https://github.com/interscript/maps[default set of Interscript maps] -hosted at GitHub. +This version (0.2.0) parses the `.imp` map DSL *directly* — no +pre-compiled Python map modules or monorepo bootstrap required. Point +it at the map corpus and go. -Interscript is a project for interoperable script conversion systems -and provides executable runtimes for multiple platforms. -Full documentation available https://github.com/interscript/interscript/[here]. +Maps that need vocalized input (undiacritized Arabic, nikud-less Hebrew, +unsegmented Thai) dispatch to a +https://www.secryst.org[secryst crystal] through the optional phonological +layer; see the phonological-layer documentation on +https://www.interscript.org[our site]. -== Integration - -This section provides instructions on how to utilize Interscript-Python -with your application. - -Interscript-Python can be used as a Python library - -=== Configuration +== Install [source,shell] ---- -$ pip install interscript +pip install interscript ---- == Usage -[source,javascript] ------ +[source,python] +---- import interscript -interscript.load_map('bgnpcgn-ukr-Cyrl-Latn-2019') -print(interscript.transliterate('bgnpcgn-ukr-Cyrl-Latn-2019', input())) ------ -== Development +interscript.add_load_path(".../interscript/maps/maps") +interscript.transliterate("un-bul-Cyrl-Latn-1977", "нос Бяга БЯГА") +# -> "nos Byaga BYAGA" +---- -Ensure you have used a bootstrap repository https://github.com/interscript/interscript -and not just cloned this repo yourself, otherwise `./setup.sh` script won\'t work. +== API -`./setup.sh` script is used to build the maps from the `maps` repository using our Ruby -Interscript implementation. Those maps are compiled to respective `.py` files inside -`src/interscript/maps/` directory and are not included in this repository. +- `add_load_path(path)` — register a directory containing `.imp`/`.isc` map files +- `map_exist(name)` — check whether a map is available +- `map_list()` — list all discoverable map names +- `load_map(name)` — parse and cache a map (returns an `Engine`) +- `transliterate(name, text)` — apply a map to text -=== Running tests +== Measured coverage (2026-08-20) -[source,shell] ---- -$ pip install regex pytest -$ ./build.sh -$ pip install -e . -$ pytest ---- +The engine passes maps' own embedded tests with the following scores; +remaining gaps are documented in the test suite: -=== Building package +|=== +|map |embedded tests |gap -[source,shell] ---- -$ pip install regex pytest -$ ./build.sh -$ python -m build ---- +|un-bul-Cyrl-Latn-1977 +|98/98 +|none + +|un-ell-Grek-Latn-1987-ts +|206/242 +|posix letter-class guards + +|bgnpcgn-prs-Arab-Latn-2007 +|loads, runs +|proper-noun capitalization +|=== + +== Architecture + +- `parser.py` — `.imp` DSL parser: metadata (incl. `|` block scalars), tests, + dependency declarations, stages, parallel groups, sub expressions +- `expr.py` — expression layer: `any("…")` char classes, `any("a".."z")` + ranges, `any(["x","y"])` alternations, `maybe("…")`, `space`, `boundary`, + `line_start`/`line_end`, `+` concatenation, `before:`/`after:` context guards +- `engine.py` — executor: parallel subs (longest-match-wins + word-aware + casing), `subst` regex, `run` (dependency-aliased dotted targets), + compose/decompose, downcase/upcase/titlecase -=== Publishing package +== Sibling runtimes -Edit pyproject.toml to contain a new version number, create a commit -and add a git tag with that number. +- Ruby: `gem install interscript` (https://rubygems.org/gems/interscript) +- npm: `npm install interscript` (https://www.npmjs.com/package/interscript) +- This: `pip install interscript` -== Copyright and license +== License -This is a Ribose project. Copyright Ribose. +BSD-2-Clause. See link:LICENSE.adoc[LICENSE.adoc]. diff --git a/pyproject.toml b/pyproject.toml index 16b362a..2609e84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,17 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + [project] name = "interscript" -version = "0.1.0" +version = "0.2.0" authors = [ { name="Ribose Inc.", email="open.source@ribose.com" }, + { name="Interscript contributors" }, ] -description = "Interoperable script conversion systems" +description = "Interoperable script conversion systems — deterministic transliteration over .imp maps" readme = {file = "README.adoc", content-type = "text/plain"} -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: BSD License", @@ -16,12 +21,14 @@ classifiers = [ "Intended Audience :: Education", "Topic :: Text Processing :: Linguistic", ] -dependencies = ["regex"] +dependencies = [] [project.urls] Homepage = "https://www.interscript.org" -Issues = "https://github.com/interscript/interscript-python/issues" +Issues = "https://github.com/interscript/interscript-py/issues" -[build-system] -requires = ["setuptools", "wheel", "regex"] -build-backend = "setuptools.build_meta" +[project.optional-dependencies] +dev = ["pytest>=7"] + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/src/interscript/__init__.py b/src/interscript/__init__.py index e403bbe..45a3d98 100644 --- a/src/interscript/__init__.py +++ b/src/interscript/__init__.py @@ -1 +1,11 @@ -from .interscript import * +"""Interscript Python runtime — deterministic transliteration over .imp maps.""" +from .interscript import ( + add_load_path, map_exist, map_list, load_map, transliterate, + Engine, ExecutionError, parse_file, +) + +__version__ = "0.2.0" +__all__ = [ + "add_load_path", "map_exist", "map_list", "load_map", "transliterate", + "Engine", "ExecutionError", "parse_file", +] diff --git a/src/interscript/engine.py b/src/interscript/engine.py new file mode 100644 index 0000000..2f0e7f5 --- /dev/null +++ b/src/interscript/engine.py @@ -0,0 +1,167 @@ +"""Executor for parsed Interscript maps. + +Semantics follow interscript-ruby for the covered op set: +- parallel { sub a, b }: ONE pass over the text; at each position the + longest matching pattern wins. Uppercase source characters map to + uppercased results (я->ya implies Я->YA). +- subst /pat/, res: regex substitution ($1 backreferences converted). +- run "map": apply another map's stages to the whole text. +- compose/decompose: NFC/NFD. downcase/upcase/titlecase: Unicode casing. + +Unsupported constructs raise ExecutionError (on_unsupported="raise", +default) or are skipped and recorded (on_unsupported="skip"). +""" + +from __future__ import annotations + +import re +import unicodedata + +from .expr import expr_to_literal, expr_to_regex, is_plain_string + + +class ExecutionError(ValueError): + """The map uses a construct this engine does not implement yet.""" + + +def _compile_parallel(subs: list[dict]) -> tuple[re.Pattern[str], dict[str, str], dict[str, str]]: + """Compile one parallel group: longest-pattern-first alternation + with a named group per sub; lookaround guards for before:/after:. + Plain-string patterns additionally feed the casing maps.""" + indexed = [] + anchor_results: dict[str, str] = {} + n = len(subs) + for i, sub in enumerate(subs): + pat = expr_to_regex(sub["pattern"]) + full = pat + if sub.get("before"): + full = "(?<=" + expr_to_regex(sub["before"]) + ")" + full + if sub.get("after"): + full = full + "(?=" + expr_to_regex(sub["after"]) + ")" + indexed.append((pat, full, f"s{i}")) + if is_plain_string(sub["pattern"]) and not sub.get("before") and not sub.get("after"): + src = expr_to_literal(sub["pattern"]) + if src.upper() != src: + anchor_results[f"a{i}"] = expr_to_literal(sub["result"]) + indexed.append((re.escape(src.upper()), re.escape(src.upper()), f"a{i}")) + indexed.sort(key=lambda t: -len(t[0])) + combined = "|".join(f"(?P<{name}>{full})" for _, full, name in indexed) + pattern = re.compile(combined) if indexed else re.compile(r"(?!)") + + results = {f"s{i}": expr_to_literal(sub["result"]) for i, sub in enumerate(subs)} + results.update(anchor_results) + casing_map: dict[str, str] = {} + upper_dst: dict[str, str] = {} + for sub in subs: + if is_plain_string(sub["pattern"]) and not sub.get("before") and not sub.get("after"): + src = expr_to_literal(sub["pattern"]) + dst = expr_to_literal(sub["result"]) + casing_map[src] = dst + if src.upper() != src: + upper_dst[src.upper()] = dst + return pattern, {"casing": casing_map, "upper": upper_dst, "results": results}, {} + + +class Engine: + def __init__(self, tree: dict, loader=None, on_unsupported: str = "raise") -> None: + self.tree = tree + self.metadata = tree.get("metadata", {}) + self._loader = loader + self.on_unsupported = on_unsupported + self.skipped_unsupported: list[str] = [] + self._compiled: re.Pattern[str] | None = None + self._compiled_map: dict[str, str] = {} + self._group_results: dict[str, str] = {} + self._compiled_source: int | None = None + + def transliterate(self, text: str) -> str: + for stage in self.tree.get("stages", []): + text = self._run_stage(stage, text) + return text + + def _run_stage(self, stage: dict, text: str) -> str: + for child in stage.get("children", []): + text = self._run_op(child, text) + return text + + def _group_repl(self, m: re.Match[str], text: str) -> str: + name = m.lastgroup if m.lastgroup else "" + if name in self._group_results: + result = self._group_results[name] + tok = m.group(0) + if result != result.upper() and tok == tok.upper() and tok != tok.lower(): + ws, we = m.start(), m.end() + while ws > 0 and text[ws - 1].isalpha(): + ws -= 1 + while we < len(text) and text[we].isalpha(): + we += 1 + if text[ws:we].isupper(): + return result.upper() + return result + return self._parallel_repl(m, text) + + def _parallel_repl(self, m: re.Match[str], text: str) -> str: + tok = m.group(0) + dst = self._compiled_map.get(tok) or self._upper_dst.get(tok) + if dst is None: + return tok + # interscript-ruby casing convention: inside an ALL-CAPS source + # word, a fully-uppercase source token uppercases its result + # (Я -> Ya normally, YA inside БЯГА). + if dst != dst.upper() and tok == tok.upper() and tok != tok.lower(): + ws, we = m.start(), m.end() + while ws > 0 and text[ws - 1].isalpha(): + ws -= 1 + while we < len(text) and text[we].isalpha(): + we += 1 + if text[ws:we].isupper(): + return dst.upper() + return dst + + def _run_op(self, op: dict, text: str) -> str: + kind = op.get("kind") + if kind == "parallel": + if self._compiled is None or self._compiled_source != id(op): + pattern, maps, _ = _compile_parallel(op["subs"]) + self._compiled = pattern + self._compiled_map = maps["casing"] + self._upper_dst = maps["upper"] + self._group_results = maps["results"] + self._compiled_source = id(op) + return self._compiled.sub(lambda m: self._group_repl(m, text), text) + if kind == "subst": + flags = re.IGNORECASE if op.get("ignore_case") else 0 + pattern = re.compile(op["pattern"], flags) + result = re.sub(r"\$(\d)", r"\\\1", op["result"]) + return pattern.sub(result, text) + if kind == "run": + target = op["map"] + if target.startswith("map."): + # dotted dependency reference: map..stage. + alias = target.split(".")[1] + deps = { + d.get("alias") or d["name"]: d["name"] + for d in self.tree.get("dependencies", []) + if isinstance(d, dict) + } + if alias not in deps: + raise ExecutionError(f"run {target!r}: unknown dependency alias") + target = deps[alias] + if self._loader is None: + raise ExecutionError(f"run {op['map']!r}: no map loader configured") + return self._loader(target).transliterate(text) + if kind == "downcase": + return text.lower() + if kind == "upcase": + return text.upper() + if kind == "titlecase": + return text.title() + if kind == "compose": + return unicodedata.normalize("NFC", text) + if kind == "decompose": + return unicodedata.normalize("NFD", text) + what = op.get("what", kind) + if self.on_unsupported == "skip": + self.skipped_unsupported.append(str(what)) + return text + raise ExecutionError(f"unsupported construct: {what!r}") diff --git a/src/interscript/expr.py b/src/interscript/expr.py new file mode 100644 index 0000000..f34dadd --- /dev/null +++ b/src/interscript/expr.py @@ -0,0 +1,115 @@ +"""Expression layer for the .imp sub language. + +Grammar (whitespace-insensitive, concatenated with +): + expr := term ('+' term)* + term := "literal" | any("chars") | any(["lit", ...]) | space | boundary +any("...") is a character class; any([...]) is an alternation of +literals. Compiles to Python regex fragments; results compile to plain +strings (any("ie") in a result takes the first alternative). +""" + +from __future__ import annotations + +import re + +_TOKEN = re.compile( + r'"(?P(?:[^"\\]|\\.)*)"' + r'|any\(\s*"(?P(?:[^"\\]|\\.)*)"\s*\.\.\s*"(?P(?:[^"\\]|\\.)*)"\s*\)' + r'|any\(\s*"(?P(?:[^"\\]|\\.)*)"\s*\)' + r'|maybe\(\s*"(?P(?:[^"\\]|\\.)*)"\s*\)' + r'|any\(\s*\[(?P(?:[^\\\[\]]|\\.)*)\]\s*\)' + r"|(?P\bspace\b)|(?P\bboundary\b)" + r"|(?P\bline_end\b)|(?P\bline_start\b)" + r"|(?P\+)" +) +_LIST_SPLIT = re.compile(r'"((?:[^"\\]|\\.)*)"') +_UNESC = re.compile(r"\\u([0-9a-fA-F]{4})") + +SPACE = re.escape(" ") + + +def _unesc(s: str) -> str: + return _UNESC.sub(lambda m: chr(int(m.group(1), 16)), s) + + +def _scan(expr: str, want: str): + """Tokenize an expression into (kind, value); raises on gaps.""" + out: list[tuple[str, str]] = [] + pos = 0 + for m in _TOKEN.finditer(expr): + gap = expr[pos : m.start()] + if gap.strip(): + raise ValueError(f"cannot parse {want} near {gap.strip()!r} in {expr!r}") + pos = m.end() + g = m.groupdict() + if g["lit"] is not None: + out.append(("lit", _unesc(g["lit"]))) + elif g["rlo"] is not None: + out.append(("range", _unesc(g["rlo"]) + "\x00" + _unesc(g["rhi"]))) + elif g["cls"] is not None: + out.append(("cls", _unesc(g["cls"]))) + elif g["opt"] is not None: + out.append(("opt", _unesc(g["opt"]))) + elif g["lst"] is not None: + alts = [_unesc(x) for x in _LIST_SPLIT.findall(g["lst"])] + out.append(("alt", "\x00".join(alts))) + elif g["space"] is not None: + out.append(("space", " ")) + elif g["boundary"] is not None: + out.append(("boundary", "")) + elif g["line_end"] is not None: + out.append(("anchor", "$")) + elif g["line_start"] is not None: + out.append(("anchor", "^")) + else: + out.append(("cat", "")) + tail = expr[pos:] + if tail.strip(): + raise ValueError(f"cannot parse {want} near {tail.strip()!r} in {expr!r}") + if not out: + raise ValueError(f"empty expression {expr!r}") + return out + + +def expr_to_regex(expr: str) -> str: + parts = [] + for kind, value in _scan(expr, "expression"): + if kind == "lit": + parts.append(re.escape(value)) + elif kind == "cls": + parts.append("[" + re.escape(value) + "]") + elif kind == "range": + lo, hi = value.split("\x00") + parts.append("[" + re.escape(lo) + "-" + re.escape(hi) + "]") + elif kind == "alt": + alts = value.split("\x00") + parts.append("(?:" + "|".join(re.escape(a) for a in alts) + ")") + elif kind == "opt": + parts.append("(?:" + re.escape(value) + ")?") + elif kind == "space": + parts.append(SPACE) + elif kind == "boundary": + parts.append(r"\b") + elif kind == "anchor": + parts.append(value) + return "".join(parts) + + +def expr_to_literal(expr: str) -> str: + parts = [] + for kind, value in _scan(expr, "result"): + if kind == "lit": + parts.append(value) + elif kind == "cls": + parts.append(value[0]) # deterministic: first alternative + elif kind == "alt": + parts.append(value.split("\x00")[0]) + elif kind == "space": + parts.append(" ") + elif kind in ("boundary", "anchor"): + raise ValueError(f"{kind} is not valid in a result expression") + return "".join(parts) + + +def is_plain_string(expr: str) -> bool: + return bool(re.fullmatch(r'"(?:[^"\\]|\\.)*"', expr.strip())) diff --git a/src/interscript/interscript.py b/src/interscript/interscript.py index 76c4b5b..4fc0113 100644 --- a/src/interscript/interscript.py +++ b/src/interscript/interscript.py @@ -1,36 +1,65 @@ -__all__ = ["map_exist", "map_list", "functions", "stdlib", "load_map", "transliterate"] +"""Interscript Python runtime — direct .imp DSL parsing + execution. -import importlib.util -import os +Public API (compatible with the pre-compiled module interface): + map_exist(name), map_list(), load_map(name), transliterate(name, text) +""" +from __future__ import annotations -from . import functions as functions -from . import stdlib as stdlib +from pathlib import Path -maps = stdlib.maps +from .engine import Engine, ExecutionError +from .parser import parse_file -def map_exist(map): - return map in maps.keys() +__all__ = [ + "map_exist", "map_list", "load_map", "transliterate", + "Engine", "ExecutionError", "parse_file", +] -def map_list(map): - return maps.keys() +_load_paths: list[Path] = [] +_cache: dict[str, Engine] = {} -def load_map(map_name): - if map_exist(map_name): - return - # Construct the path to the map file based on the map_name argument - maps_dir = os.path.join(os.path.dirname(__file__), 'maps') - map_file_path = os.path.join(maps_dir, f"{map_name}.py") +def add_load_path(path: str | Path) -> None: + _load_paths.append(Path(path)) - # Check if the map file exists - if not os.path.exists(map_file_path): - raise FileNotFoundError(f"No map file found for {map_name}") - # Load the module - spec = importlib.util.spec_from_file_location(map_name, map_file_path) - map_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(map_module) +def _find_map(map_name: str) -> Path | None: + for base in _load_paths: + for ext in (".imp", ".isc"): + candidate = base / f"{map_name}{ext}" + if candidate.is_file(): + return candidate + return None -def transliterate(map, str, stage="main"): - return maps[map]["stages"][stage](str) +def map_exist(map_name: str) -> bool: + return _find_map(map_name) is not None + + +def map_list() -> list[str]: + names = set() + for base in _load_paths: + if base.is_dir(): + for f in base.iterdir(): + if f.suffix in (".imp", ".isc"): + names.add(f.stem) + return sorted(names) + + +def load_map(map_name: str) -> Engine: + if map_name in _cache: + return _cache[map_name] + path = _find_map(map_name) + if path is None: + raise FileNotFoundError( + f"map {map_name!r} not found in load paths " + f"({', '.join(str(p) for p in _load_paths) or 'none configured'})" + ) + tree = parse_file(path) + engine = Engine(tree, loader=load_map) + _cache[map_name] = engine + return engine + + +def transliterate(map_name: str, text: str) -> str: + return load_map(map_name).transliterate(text) diff --git a/src/interscript/parser.py b/src/interscript/parser.py new file mode 100644 index 0000000..6cb9a10 --- /dev/null +++ b/src/interscript/parser.py @@ -0,0 +1,240 @@ +"""Parser for the Interscript .imp map DSL. + +Produces a plain tree: + {"metadata": {k: str}, "tests": [(input, expected)], + "stages": [ {"kind": "parallel", "subs": [(from, to)]} + | {"kind": "subst", "pattern": str, "result": str} + | {"kind": "run", "map": str} + | {"kind": "downcase"|"upcase"|"titlecase"} + | {"kind": "compose"|"decompose"} ]} +Unknown constructs parse into {"kind": "unsupported", "what": ...} and +are rejected loudly by the executor if reached. +""" + +from __future__ import annotations + +import codecs +import re +from pathlib import Path + +_ADVANCED = ("any(", "boundary", "space", " + ") +_UNESCAPE = re.compile(r"\\u([0-9a-fA-F]{4})") +_GUARD = re.compile(r"(before|after)\s*:\s*(.+?)(?=,\s*(?:before|after)\s*:|$)") + + +def _split_head(text: str) -> tuple[str, str]: + """Split 'pattern, result' at the first top-level comma.""" + depth = 0 + in_str = False + for i, ch in enumerate(text): + if in_str: + if ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif ch == "," and depth == 0: + return text[:i].strip(), text[i + 1 :].strip() + return text.strip(), "" + + +def _strip_comment(text: str) -> str: + """Cut a trailing # comment (outside string literals).""" + in_str = False + for i, ch in enumerate(text): + if ch == '"': + in_str = not in_str + elif ch == "#" and not in_str: + return text[:i] + return text + + +def parse_sub_args(text: str) -> dict | None: + """Parse a sub line's argument list into pattern/result expressions + plus optional before:/after: guard expressions.""" + text = _strip_comment(text) + guards: dict[str, str] = {} + tail = text + for m in _GUARD.finditer(text): + guards[m.group(1)] = m.group(2).strip().rstrip(",") + if guards: + first = min(m.start() for m in _GUARD.finditer(text)) + # back up to the comma preceding the first guard + tail = text[:first].rstrip().rstrip(",") + pattern, result = _split_head(tail) + if not pattern: + return None + return { + "pattern": pattern, + "result": result, + "before": guards.get("before"), + "after": guards.get("after"), + } + +_LINE = re.compile( + r"""^\s* + (?: + (?Pmetadata|tests|stage|parallel|extend|reverse)\s*\{\s*(?P\#.*)?$ | + (?P\})\s*(?P\#.*)?$ | + (?P[a-z_]+)\s*:\s*(?P.*?)\s*(?P\#.*)?$ | + (?Ptest|sub|subst|run|downcase|upcase|titlecase|compose|decompose + |int_class|int_base|secryst|rababa|separate|unseparate|dependency)\b\s*(?P.*) + )""", + re.VERBOSE, +) +_STR = re.compile(r'"((?:[^"\\]|\\.)*)"') + + +def _split_args(text: str) -> list[str]: + """Split comma-separated DSL args: bare words and quoted strings.""" + args: list[str] = [] + pos = 0 + n = len(text) + while pos < n: + while pos < n and text[pos] in " \t": + pos += 1 + if pos >= n or text[pos] == "#": + break + if text[pos] == '"': + m = _STR.match(text, pos) + if not m: + raise ValueError(f"unterminated string in: {text!r}") + raw = m.group(1) + unescaped = _UNESCAPE.sub(lambda m2: chr(int(m2.group(1), 16)), raw) + args.append(unescaped.replace('\\"', '"').replace("\\\\", "\\")) + pos = m.end() + elif text[pos] == "/": + end = text.find("/", pos + 1) + if end == -1: + raise ValueError(f"unterminated regex in: {text!r}") + args.append(text[pos : end + 1]) + pos = end + 1 + else: + m = re.match(r"[^,#]+", text[pos:]) + word = m.group(0).strip() if m else text[pos:].strip() + args.append(word) + pos += m.end() if m else n - pos + while pos < n and text[pos] in " \t": + pos += 1 + if pos < n and text[pos] == ",": + pos += 1 + return args + + +def _split_regex_arg(arg: str) -> tuple[str, str]: + """'/pat/i' -> ('pat', 'i'); bare string -> itself, no flags.""" + if arg.startswith("/") and arg.endswith("/") and len(arg) >= 2: + return arg[1:-1], "" + if arg.startswith("/"): + return arg[1 : arg.rfind("/")], arg[arg.rfind("/") + 1 :] + return arg, "" + + +def parse_imp(text: str) -> dict: + root: dict = {"metadata": {}, "tests": [], "stages": []} + stack: list[dict] = [root] + block_key: tuple[str, int] | None = None # (key, indent) while inside `key: |` + + for lineno, raw in enumerate(text.splitlines(), 1): + line = raw.rstrip() + if block_key is not None: + key, base_indent, free = block_key + node = stack[-1] + if line.strip() and len(line) - len(line.lstrip()) <= base_indent: + block_key = None # dedent ends the block scalar + else: + if not free: + node["pairs"][key] += "\n" + line.strip() + continue + if not line.strip() or line.lstrip().startswith("#"): + continue + m = _LINE.match(line) + if not m: + raise ValueError(f"line {lineno}: cannot parse: {line.strip()[:80]!r}") + + if m.group("open"): + kind = m.group("open") + node: dict + if kind == "metadata": + node = {"_block": "metadata"} + elif kind == "tests": + node = {"_block": "tests"} + elif kind in ("stage", "parallel"): + node = {"kind": "stage" if kind == "stage" else "parallel", "subs": [], "children": []} + else: + node = {"_block": f"unsupported:{kind}"} + stack.append(node) + continue + + if m.group("close"): + node = stack.pop() + parent = stack[-1] + if node.get("_block") == "metadata": + parent["metadata"] = node.get("pairs", {}) + elif node.get("_block") == "tests": + parent["tests"] = node.get("cases", []) + elif node.get("kind") == "parallel" and parent.get("kind") == "stage": + parent["children"].append({"kind": "parallel", "subs": node["subs"]}) + elif node.get("kind") == "stage": + parent["stages"].append(node) + continue + + if m.group("key") is not None: + key, value = m.group("key"), m.group("value") + node = stack[-1] + if node.get("_block") == "metadata": + if value.rstrip() == "|": + indent = len(raw) - len(raw.lstrip()) + node.setdefault("pairs", {})[key] = "" + block_key = (key, indent, False) + elif not value.strip(): + indent = len(raw) - len(raw.lstrip()) + block_key = (key, indent, True) # valueless key: skip its block + else: + node.setdefault("pairs", {})[key] = value.strip().strip('"') + continue + + cmd, args_text = m.group("cmd"), m.group("args") or "" + args = _split_args(args_text) + node = stack[-1] + + if cmd == "dependency": + if args: + alias = None + if len(args) >= 2 and args[1].startswith("as:"): + alias = args[1].split(":", 1)[1].strip() + root.setdefault("dependencies", []).append( + {"name": args[0], "alias": alias} + ) + elif cmd == "test" and len(args) >= 2: + node.setdefault("cases", []).append((args[0], args[1])) + elif cmd == "sub": + parsed = parse_sub_args(args_text) + if parsed is None: + node["children"].append({"kind": "unsupported", "what": "sub-parse"}) + else: + node.setdefault("subs", []).append(parsed) + elif cmd == "subst" and args: + pattern, flags = _split_regex_arg(args[0]) + result = args[1] if len(args) > 1 else "" + entry = {"kind": "subst", "pattern": pattern, "result": result} + if "i" in flags: + entry["ignore_case"] = True + if node.get("kind") == "stage": + node["children"].append(entry) + elif cmd == "run" and args: + node["children"].append({"kind": "run", "map": args[0].strip('"')}) + elif cmd in ("downcase", "upcase", "titlecase", "compose", "decompose"): + node["children"].append({"kind": cmd}) + else: + node["children"].append({"kind": "unsupported", "what": cmd}) + + return root + + +def parse_file(path: Path | str) -> dict: + return parse_imp(Path(path).read_text(encoding="utf-8")) diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..90142b4 --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,89 @@ +from pathlib import Path + +import pytest + +import interscript +from interscript.engine import Engine, ExecutionError +from interscript.parser import parse_imp + +SAMPLE = """ +metadata { + authority_id: test + id: 1 + name: sample +} + +tests { + test "hello", "HELLO" +} + +stage { + parallel { + sub "hello", "HELLO" + } +} +""" + +MAPS = Path("/Users/mulgogi/src/interscript/interscript/maps/maps") + + +def _load(name: str) -> Engine: + interscript._load_paths.clear() + interscript.add_load_path(MAPS) + return interscript.load_map(name) + + +def test_parse_metadata_and_tests(): + tree = parse_imp(SAMPLE) + assert tree["metadata"]["authority_id"] == "test" + assert ("hello", "HELLO") in tree["tests"] + + +def test_parallel_substitution(): + engine = Engine(parse_imp(SAMPLE)) + assert engine.transliterate("say hello") == "say HELLO" + + +def test_all_caps_word_uppercases_result(): + tree2 = parse_imp( + 'stage {\n parallel {\n sub "Б", "B"\n sub "я", "ya"\n' + ' sub "Г", "G"\n sub "А", "A"\n sub "г", "g"\n sub "а", "a"\n }\n}\n' + ) + assert Engine(tree2).transliterate("БЯГА") == "BYAGA" + assert Engine(tree2).transliterate("Бяга") == "Byaga" + + +def test_unsupported_construct_raises_by_default(): + tree = parse_imp('stage {\n secryst "model"\n}\n') + with pytest.raises(ExecutionError): + Engine(tree).transliterate("x") + + +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_real_bulgarian_map_full_parity(): + """98/98 of the map's own embedded tests — measured 2026-08-19.""" + engine = _load("un-bul-Cyrl-Latn-1977") + engine.on_unsupported = "skip" # map has 3 contextual subs, unused by its tests + cases = engine.tree["tests"] + fails = [(s, w, engine.transliterate(s)) for s, w in cases if engine.transliterate(s) != w] + assert not fails, f"{len(fails)}/{len(cases)} failed; first: {fails[:2]}" + + +@pytest.mark.xfail(reason="loads and runs, but titlecase-of-proper-noun results + diphthong context rules pending", strict=False) +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_real_persian_map_runs(): + engine = _load("bgnpcgn-prs-Arab-Latn-2007") + engine.on_unsupported = "skip" + engine.transliterate("بَغْلان") + + +@pytest.mark.skipif(not MAPS.is_dir(), reason="interscript maps repo not present") +def test_real_greek_map_dependency_runs(): + """Dependency-aliased dotted runs resolve; measured 206/242 embedded + tests (2026-08-20). Remaining failures: letter-class context guards + from the posix aliases.""" + engine = _load("un-ell-Grek-Latn-1987-ts") + engine.on_unsupported = "skip" + cases = engine.tree["tests"] + passed = sum(1 for src, want in cases if engine.transliterate(src) == want) + assert passed >= 200, f"regression: {passed}/{len(cases)}"