0.2.0: direct .imp DSL engine (replaces pre-compiled modules)

README.adoc

@@ -2,72 +2,84 @@
 
 == Purpose
 
-This repository contains code for the Interscript Python runtime ("Interscript-Python").
+The official Python runtime for Interscript — deterministic transliteration
+over the interscript map corpus (300+ authority-backed systems: BGN/PCGN,
+ISO, UN, ALA-LC, ODNI, ICAO, DIN, and others).
 
-This software allows performing script conversions by using the
-https://github.com/interscript/maps[default set of Interscript maps]
-hosted at GitHub.
+This version (0.2.0) parses the `.imp` map DSL *directly* — no
+pre-compiled Python map modules or monorepo bootstrap required. Point
+it at the map corpus and go.
 
-Interscript is a project for interoperable script conversion systems
-and provides executable runtimes for multiple platforms.
-Full documentation available https://github.com/interscript/interscript/[here].
+Maps that need vocalized input (undiacritized Arabic, nikud-less Hebrew,
+unsegmented Thai) dispatch to a
+https://www.secryst.org[secryst crystal] through the optional phonological
+layer; see the phonological-layer documentation on
+https://www.interscript.org[our site].
 
-== Integration
-
-This section provides instructions on how to utilize Interscript-Python
-with your application.
-
-Interscript-Python can be used as a Python library
-
-=== Configuration
+== Install
 
 [source,shell]
 ----
-$ pip install interscript
+pip install interscript
 ----
 
 == Usage
 
-[source,javascript]
------
+[source,python]
+----
 import interscript
-interscript.load_map('bgnpcgn-ukr-Cyrl-Latn-2019')
-print(interscript.transliterate('bgnpcgn-ukr-Cyrl-Latn-2019', input()))
------
 
-== Development
+interscript.add_load_path(".../interscript/maps/maps")
+interscript.transliterate("un-bul-Cyrl-Latn-1977", "нос Бяга БЯГА")
+# -> "nos Byaga BYAGA"
+----
 
-Ensure you have used a bootstrap repository https://github.com/interscript/interscript
-and not just cloned this repo yourself, otherwise `./setup.sh` script won\'t work.
+== API
 
-`./setup.sh` script is used to build the maps from the `maps` repository using our Ruby
-Interscript implementation. Those maps are compiled to respective `.py` files inside
-`src/interscript/maps/` directory and are not included in this repository.
+- `add_load_path(path)` — register a directory containing `.imp`/`.isc` map files
+- `map_exist(name)` — check whether a map is available
+- `map_list()` — list all discoverable map names
+- `load_map(name)` — parse and cache a map (returns an `Engine`)
+- `transliterate(name, text)` — apply a map to text
 
-=== Running tests
+== Measured coverage (2026-08-20)
 
-[source,shell]
----
-$ pip install regex pytest
-$ ./build.sh
-$ pip install -e .
-$ pytest
----
+The engine passes maps' own embedded tests with the following scores;
+remaining gaps are documented in the test suite:
 
-=== Building package
+|===
+|map |embedded tests |gap
 
-[source,shell]
----
-$ pip install regex pytest
-$ ./build.sh
-$ python -m build
----
+|un-bul-Cyrl-Latn-1977
+|98/98
+|none
+
+|un-ell-Grek-Latn-1987-ts
+|206/242
+|posix letter-class guards
+
+|bgnpcgn-prs-Arab-Latn-2007
+|loads, runs
+|proper-noun capitalization
+|===
+
+== Architecture
+
+- `parser.py` — `.imp` DSL parser: metadata (incl. `|` block scalars), tests,
+ dependency declarations, stages, parallel groups, sub expressions
+- `expr.py` — expression layer: `any("…")` char classes, `any("a".."z")`
+ ranges, `any(["x","y"])` alternations, `maybe("…")`, `space`, `boundary`,
+ `line_start`/`line_end`, `+` concatenation, `before:`/`after:` context guards
+- `engine.py` — executor: parallel subs (longest-match-wins + word-aware
+ casing), `subst` regex, `run` (dependency-aliased dotted targets),
+ compose/decompose, downcase/upcase/titlecase
 
-=== Publishing package
+== Sibling runtimes
 
-Edit pyproject.toml to contain a new version number, create a commit
-and add a git tag with that number.
+- Ruby: `gem install interscript` (https://rubygems.org/gems/interscript)
+- npm: `npm install interscript` (https://www.npmjs.com/package/interscript)
+- This: `pip install interscript`
 
-== Copyright and license
+== License
 
-This is a Ribose project. Copyright Ribose.
+BSD-2-Clause. See link:LICENSE.adoc[LICENSE.adoc].

pyproject.toml

@@ -1,12 +1,17 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "interscript"
-version = "0.1.0"
+version = "0.2.0"
 authors = [
  { name="Ribose Inc.", email="open.source@ribose.com" },
+ { name="Interscript contributors" },
 ]
-description = "Interoperable script conversion systems"
+description = "Interoperable script conversion systems — deterministic transliteration over .imp maps"
 readme = {file = "README.adoc", content-type = "text/plain"}
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 classifiers = [
  "Programming Language :: Python :: 3",
  "License :: OSI Approved :: BSD License",
@@ -16,12 +21,14 @@ classifiers = [
  "Intended Audience :: Education",
  "Topic :: Text Processing :: Linguistic",
 ]
-dependencies = ["regex"]
+dependencies = []
 
 [project.urls]
 Homepage = "https://www.interscript.org"
-Issues = "https://github.com/interscript/interscript-python/issues"
+Issues = "https://github.com/interscript/interscript-py/issues"
 
-[build-system]
-requires = ["setuptools", "wheel", "regex"]
-build-backend = "setuptools.build_meta"
+[project.optional-dependencies]
+dev = ["pytest>=7"]
+
+[tool.setuptools.packages.find]
+where = ["src"]

src/interscript/__init__.py

@@ -1 +1,11 @@
-from .interscript import *
+"""Interscript Python runtime — deterministic transliteration over .imp maps."""
+from .interscript import (
+ add_load_path, map_exist, map_list, load_map, transliterate,
+ Engine, ExecutionError, parse_file,
+)
+
+__version__ = "0.2.0"
+__all__ = [
+ "add_load_path", "map_exist", "map_list", "load_map", "transliterate",
+ "Engine", "ExecutionError", "parse_file",
+]

src/interscript/engine.py

@@ -0,0 +1,167 @@
+"""Executor for parsed Interscript maps.
+
+Semantics follow interscript-ruby for the covered op set:
+- parallel { sub a, b }: ONE pass over the text; at each position the
+ longest matching pattern wins. Uppercase source characters map to
+ uppercased results (я->ya implies Я->YA).
+- subst /pat/, res: regex substitution ($1 backreferences converted).
+- run "map": apply another map's stages to the whole text.
+- compose/decompose: NFC/NFD. downcase/upcase/titlecase: Unicode casing.
+
+Unsupported constructs raise ExecutionError (on_unsupported="raise",
+default) or are skipped and recorded (on_unsupported="skip").
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+from .expr import expr_to_literal, expr_to_regex, is_plain_string
+
+
+class ExecutionError(ValueError):
+ """The map uses a construct this engine does not implement yet."""
+
+
+def _compile_parallel(subs: list[dict]) -> tuple[re.Pattern[str], dict[str, str], dict[str, str]]:
+ """Compile one parallel group: longest-pattern-first alternation
+ with a named group per sub; lookaround guards for before:/after:.
+ Plain-string patterns additionally feed the casing maps."""
+ indexed = []
+ anchor_results: dict[str, str] = {}
+ n = len(subs)
+ for i, sub in enumerate(subs):
+ pat = expr_to_regex(sub["pattern"])
+ full = pat
+ if sub.get("before"):
+ full = "(?<=" + expr_to_regex(sub["before"]) + ")" + full
+ if sub.get("after"):
+ full = full + "(?=" + expr_to_regex(sub["after"]) + ")"
+ indexed.append((pat, full, f"s{i}"))
+ if is_plain_string(sub["pattern"]) and not sub.get("before") and not sub.get("after"):
+ src = expr_to_literal(sub["pattern"])
+ if src.upper() != src:
+ anchor_results[f"a{i}"] = expr_to_literal(sub["result"])
+ indexed.append((re.escape(src.upper()), re.escape(src.upper()), f"a{i}"))
+ indexed.sort(key=lambda t: -len(t[0]))
+ combined = "|".join(f"(?P<{name}>{full})" for _, full, name in indexed)
+ pattern = re.compile(combined) if indexed else re.compile(r"(?!)")
+
+ results = {f"s{i}": expr_to_literal(sub["result"]) for i, sub in enumerate(subs)}
+ results.update(anchor_results)
+ casing_map: dict[str, str] = {}
+ upper_dst: dict[str, str] = {}
+ for sub in subs:
+ if is_plain_string(sub["pattern"]) and not sub.get("before") and not sub.get("after"):
+ src = expr_to_literal(sub["pattern"])
+ dst = expr_to_literal(sub["result"])
+ casing_map[src] = dst
+ if src.upper() != src:
+ upper_dst[src.upper()] = dst
+ return pattern, {"casing": casing_map, "upper": upper_dst, "results": results}, {}
+
+
+class Engine:
+ def __init__(self, tree: dict, loader=None, on_unsupported: str = "raise") -> None:
+ self.tree = tree
+ self.metadata = tree.get("metadata", {})
+ self._loader = loader
+ self.on_unsupported = on_unsupported
+ self.skipped_unsupported: list[str] = []
+ self._compiled: re.Pattern[str] | None = None
+ self._compiled_map: dict[str, str] = {}
+ self._group_results: dict[str, str] = {}
+ self._compiled_source: int | None = None
+
+ def transliterate(self, text: str) -> str:
+ for stage in self.tree.get("stages", []):
+ text = self._run_stage(stage, text)
+ return text
+
+ def _run_stage(self, stage: dict, text: str) -> str:
+ for child in stage.get("children", []):
+ text = self._run_op(child, text)
+ return text
+
+ def _group_repl(self, m: re.Match[str], text: str) -> str:
+ name = m.lastgroup if m.lastgroup else ""
+ if name in self._group_results:
+ result = self._group_results[name]
+ tok = m.group(0)
+ if result != result.upper() and tok == tok.upper() and tok != tok.lower():
+ ws, we = m.start(), m.end()
+ while ws > 0 and text[ws - 1].isalpha():
+ ws -= 1
+ while we < len(text) and text[we].isalpha():
+ we += 1
+ if text[ws:we].isupper():
+ return result.upper()
+ return result
+ return self._parallel_repl(m, text)
+
+ def _parallel_repl(self, m: re.Match[str], text: str) -> str:
+ tok = m.group(0)
+ dst = self._compiled_map.get(tok) or self._upper_dst.get(tok)
+ if dst is None:
+ return tok
+ # interscript-ruby casing convention: inside an ALL-CAPS source
+ # word, a fully-uppercase source token uppercases its result
+ # (Я -> Ya normally, YA inside БЯГА).
+ if dst != dst.upper() and tok == tok.upper() and tok != tok.lower():
+ ws, we = m.start(), m.end()
+ while ws > 0 and text[ws - 1].isalpha():
+ ws -= 1
+ while we < len(text) and text[we].isalpha():
+ we += 1
+ if text[ws:we].isupper():
+ return dst.upper()
+ return dst
+
+ def _run_op(self, op: dict, text: str) -> str:
+ kind = op.get("kind")
+ if kind == "parallel":
+ if self._compiled is None or self._compiled_source != id(op):
+ pattern, maps, _ = _compile_parallel(op["subs"])
+ self._compiled = pattern
+ self._compiled_map = maps["casing"]
+ self._upper_dst = maps["upper"]
+ self._group_results = maps["results"]
+ self._compiled_source = id(op)
+ return self._compiled.sub(lambda m: self._group_repl(m, text), text)
+ if kind == "subst":
+ flags = re.IGNORECASE if op.get("ignore_case") else 0
+ pattern = re.compile(op["pattern"], flags)
+ result = re.sub(r"\$(\d)", r"\\\1", op["result"])
+ return pattern.sub(result, text)
+ if kind == "run":
+ target = op["map"]
+ if target.startswith("map."):
+ # dotted dependency reference: map.<alias>.stage.<name>
+ alias = target.split(".")[1]
+ deps = {
+ d.get("alias") or d["name"]: d["name"]
+ for d in self.tree.get("dependencies", [])
+ if isinstance(d, dict)
+ }
+ if alias not in deps:
+ raise ExecutionError(f"run {target!r}: unknown dependency alias")
+ target = deps[alias]
+ if self._loader is None:
+ raise ExecutionError(f"run {op['map']!r}: no map loader configured")
+ return self._loader(target).transliterate(text)
+ if kind == "downcase":
+ return text.lower()
+ if kind == "upcase":
+ return text.upper()
+ if kind == "titlecase":
+ return text.title()
+ if kind == "compose":
+ return unicodedata.normalize("NFC", text)
+ if kind == "decompose":
+ return unicodedata.normalize("NFD", text)
+ what = op.get("what", kind)
+ if self.on_unsupported == "skip":
+ self.skipped_unsupported.append(str(what))
+ return text
+ raise ExecutionError(f"unsupported construct: {what!r}")