diff --git a/.bumpy/extract-region-splice-force.md b/.bumpy/extract-region-splice-force.md new file mode 100644 index 0000000..1a3c5cb --- /dev/null +++ b/.bumpy/extract-region-splice-force.md @@ -0,0 +1,21 @@ +--- +"mdcode-ts": minor +--- + +**Behaviour change:** `extract` no longer overwrites pre-existing files that a block describes in +full. Such targets are skipped with a warning; pass `--force` for the old behaviour. When anything is +skipped the count is reported even under `--quiet` and the CLI exits with status 2, so a pipeline +cannot mistake "wrote nothing" for success. + +`extract` now splices `region=` blocks in place instead of rewriting the whole file, so surrounding +code and untouched regions survive, and aliased `file=` paths pointing at one file resolve to a +single write. Region markers are written and matched in the block language's own comment syntax, so +shell, SQL, CSS and block-comment markers splice instead of being duplicated. + +A target is left byte-identical rather than spliced when it is a symlink, is not valid UTF-8, has a +region it never closes or names inconsistently, declares a region twice, or when the blocks for one +file mix `region=` with whole-file blocks. Writes go through a temp file and `rename`, preserving the +target's permission bits, so an interrupted write cannot truncate a source file. + +Install and import docs now name the published package `mdcode-ts`; they previously pointed at a +name that resolved to the upstream fork. diff --git a/CLAUDE.md b/CLAUDE.md index 0397909..c732643 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,24 +6,23 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co TypeScript port of [szkiba/mdcode](https://github.com/szkiba/mdcode) - a Markdown code block authoring tool for extracting, updating, and managing code blocks within markdown documents. -This is a **pnpm workspace monorepo** with three packages: -- `packages/mdcode` - Main library and CLI -- `packages/usage` - E2E integration tests -- `packages/example` - Example usage (no tests) +This is a **pnpm workspace monorepo** with two packages: +- `packages/mdcode` - Main library and CLI, published as `mdcode-ts` +- `packages/usage` - Integration tests that consume the built package ## Essential Commands ### Testing (ALWAYS RUN BOTH) ```bash -# Run ALL tests (mdcode-ts + usage packages = 138 total tests) +# Run ALL tests (mdcode-ts + usage packages ) pnpm test # Watch mode during development pnpm --filter mdcode-ts test:watch # Individual packages -pnpm --filter mdcode-ts test # 51 unit tests -pnpm --filter usage test # 87 E2E tests +pnpm --filter mdcode-ts test # unit tests +pnpm --filter usage test # E2E tests ``` **NOTE**: `pnpm test` is `pnpm -r test` — it already covers both packages. `pnpm test:all` also exists but just re-runs `usage` a second time. @@ -91,7 +90,7 @@ Type signature: `(options: {tag, meta, code}) => string | Promise` Use `defineTransform()` helper for type safety: ```typescript -import { defineTransform } from 'mdcode'; +import { defineTransform } from 'mdcode-ts'; export default defineTransform(({tag, code}) => { if (tag === 'sql') return code.toUpperCase(); @@ -123,10 +122,11 @@ Original design used `unified` + `remark-parse`, but switched to custom state ma - No AST overhead for simple code block extraction ### Test Organization -- **Root tests/** - Parser and transformer unit tests (not used currently) -- **packages/mdcode/src/*.test.ts** - Co-located unit tests (16 tests) -- **packages/usage/tests/** - E2E workflow tests (5 tests) -- Import path from root: `../packages/mdcode/src/...` +- **packages/mdcode/src/\*\*/\*.test.ts** - Co-located unit tests (parser, region, commands/extract, commands/update) +- **packages/mdcode/tests/examples/** - Fixture-driven tests over the worked examples +- **packages/usage/tests/** - Integration tests; `cli-integration.test.ts` spawns the **built** `dist/main.js`, so run `pnpm build` first + +See `TESTING.md` for the full layout. ### File Imports Must Use .ts Extension TypeScript config uses `allowImportingTsExtensions: true`: @@ -139,6 +139,6 @@ import { parse } from './parser'; ``` ## Before Committing -1. `pnpm test` - Ensure ALL 138 tests pass +1. `pnpm test` - Ensure ALL tests pass 2. `pnpm build` - Ensure build succeeds 3. `pnpm -r lint:ts` - Type check all packages diff --git a/TESTING.md b/TESTING.md index 323e7c2..c09541f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,52 +2,64 @@ ## Running All Tests -### Full Test Suite (ALWAYS RUN THIS) ```bash -pnpm -w run test:all +pnpm test ``` -This runs tests for all packages in the workspace: -- Main mdcode package (16 unit tests) -- Usage package (5 E2E tests) -Total: **21 tests** -**Note**: `pnpm test` only runs the mdcode unit tests. Use `pnpm -w run test:all` to run everything. +`pnpm test` is `pnpm -r test`: it already runs **both** packages — `mdcode-ts` unit tests and the +`usage` integration tests. (`pnpm test:all` also exists, but it just runs the `usage` package a +second time.) + +Test counts are deliberately not recorded here; they rot. Run the suite to see them. ### Individual Package Tests + ```bash -# Main mdcode package (unit tests) -pnpm --filter @gcm/mdcode test +# mdcode-ts unit tests +pnpm --filter mdcode-ts test -# Usage package (E2E tests) -pnpm --filter @gcm/mdcode-usage test -cd packages/usage && pnpm test # Alternative +# Watch mode +pnpm --filter mdcode-ts test:watch + +# usage integration tests +pnpm --filter usage test ``` ## Test Structure +Two packages, `packages/mdcode` (published as `mdcode-ts`) and `packages/usage`. + ### Test Locations -- **Root tests**: `/tests/` - Core functionality tests - - `parser.test.ts` - Markdown parsing and walking (8 tests) - - `transform.test.ts` - Transformer functionality (8 tests) -- **E2E tests**: `packages/usage/tests/e2e.test.ts` - End-to-end integration (5 tests) -- **Example package**: `packages/example/` - No tests, just runnable examples + +- **Co-located unit tests** — `packages/mdcode/src/**/*.test.ts` + - `region.test.ts` — marker matching, region splicing, and its refusal cases + - `parser.test.ts` — info-string and fenced-block parsing + - `commands/extract.test.ts` — in-place splicing, `--force`, and every refusal path + - `commands/update.test.ts` — filling blocks from source regions +- **Fixture-driven tests** — `packages/mdcode/tests/examples/integration.test.ts`, against the + worked examples under `packages/mdcode/tests/examples/` +- **Integration tests** — `packages/usage/tests/` + - `cli-integration.test.ts` spawns the **built** CLI at `packages/mdcode/dist/main.js` + - the rest exercise the public library API as an external consumer would ### Important Notes -- Import paths from root tests must use `../packages/mdcode/src/...` -- **ALWAYS test all packages** - run `pnpm test` from workspace root -- E2E tests validate extract/update workflows with real files + +- `packages/usage/tests/cli-integration.test.ts` runs `dist/`, not `src/`. **Run `pnpm build` before + it** or you will be testing the previous build. +- Region fixtures live in `packages/mdcode/tests/testdata/region/` and are compared byte-for-byte, so + trailing newlines matter. +- Tests that assert on warnings use `mock.method(console, "error", …)` with `mock.restoreAll()` in a + `finally`, so a failing assertion cannot leak the stub into sibling tests. ## Before Committing -1. **`pnpm -w run test:all`** - Ensure ALL 21 tests pass (mdcode + usage packages) -2. **`pnpm build`** - Ensure build succeeds -3. **Test examples** - Manually run at least one example: - ```bash - pnpm --filter @gcm/mdcode-example example:list - pnpm --filter @gcm/mdcode-example example:extract - ``` + +1. **`pnpm test`** — all packages pass +2. **`pnpm build`** — build succeeds, and refreshes `dist/` for the CLI tests +3. **`pnpm -r lint`** — type check and ESLint are clean ## Adding New Tests -- **Parser/core**: Add to `/tests/parser.test.ts` -- **Transform**: Add to `/tests/transform.test.ts` -- **E2E workflows**: Add to `packages/usage/tests/e2e.test.ts` -- **New commands**: Add unit tests to root `/tests/` or create new test file + +- **Region/marker behaviour** → `packages/mdcode/src/region.test.ts` +- **A command's behaviour** → `packages/mdcode/src/commands/.test.ts` +- **CLI flags, exit codes, stderr** → `packages/usage/tests/cli-integration.test.ts` +- **Public API as a consumer sees it** → `packages/usage/tests/library-usage.test.ts` diff --git a/examples/CLI_EXAMPLES.md b/examples/CLI_EXAMPLES.md index 38c37f5..a76c206 100644 --- a/examples/CLI_EXAMPLES.md +++ b/examples/CLI_EXAMPLES.md @@ -22,7 +22,7 @@ Comprehensive guide to using mdcode from the command line. ### Global Install (npm) ```bash -npm install -g mdcode +npm install -g mdcode-ts ``` After installation, you can run `mdcode` from anywhere: @@ -36,7 +36,7 @@ mdcode list README.md ### Global Install (pnpm) ```bash -pnpm install -g mdcode +pnpm install -g mdcode-ts ``` Usage is identical to npm installation: @@ -51,16 +51,16 @@ mdcode --help No installation required - run directly: ```bash -pnpm dlx mdcode list README.md -pnpm dlx mdcode extract --lang js docs/*.md -pnpm dlx mdcode update --transform ./my-transformer.js README.md +pnpm dlx mdcode-ts list README.md +pnpm dlx mdcode-ts extract --lang js docs/*.md +pnpm dlx mdcode-ts update --transform ./my-transformer.js README.md ``` ### Run Without Installing (npx) ```bash -npx mdcode list README.md -npx mdcode --help +npx mdcode-ts list README.md +npx mdcode-ts --help ``` --- @@ -212,6 +212,27 @@ mdcode list --json -l python -m type=example docs/ Extract code blocks to files based on their `file` metadata. +### Existing Files Are Not Overwritten + +`extract` never destroys work you did not ask it to touch: + +- Blocks with `region=` are **spliced in place** — surrounding code and untouched regions survive. +- Blocks without `region=` describe a whole file. If that file already exists it is **skipped with a + warning**; pass `--force` to overwrite it. +- A target is also skipped, and left byte-identical, when it is a symlink, is not valid UTF-8, has a + region the file never closes, or when the blocks for one file mix `region=` with whole-file blocks. + +When anything is skipped, `extract` reports the count even under `--quiet` and exits with a non-zero +status, so a pipeline cannot mistake "wrote nothing" for success. + +```bash +# Skipped with a warning if the target already exists +mdcode extract README.md + +# Overwrite whole-file targets +mdcode extract --force README.md +``` + ### Basic Usage ```bash @@ -291,6 +312,22 @@ curl https://example.com/docs.md | mdcode extract -q Update markdown code blocks from source files or transform them with custom functions. +### `file=` Is Trusted, By Design + +`update` reads whatever path a block's `file=` names, including paths that leave the markdown's own +directory — `file=../src/app.js` from a `docs/` folder is normal and supported. The path is an +explicit instruction from whoever wrote the markdown, so it is honoured as written and is **not** +confined to `--base-path`. + +The consequence is that `update` will inline the contents of any file the process can read, and those +contents land in the markdown. Treat markdown from an untrusted source the way you would treat a +script from an untrusted source: review it before running `update` over it, and do not run `update` +on contributor-supplied markdown in an environment holding secrets. + +(`extract`, which *writes*, is confined to `--dir` and refuses paths that escape it. The asymmetry is +deliberate: reading a path you named is what you asked for, whereas writing outside the output +directory never is.) + ### Update from Source Files (Default Mode) Updates code blocks by reading from files specified in the `file` metadata attribute: @@ -699,7 +736,7 @@ mdcode update -l sql -f queries.sql --stdout README.md ## Comparison with Original mdcode -This TypeScript implementation is a **drop-in replacement** for the original Go-based [szkiba/mdcode](https://github.com/szkiba/mdcode). It maintains 100% CLI compatibility. +This TypeScript implementation is a near **drop-in replacement** for the original Go-based [szkiba/mdcode](https://github.com/szkiba/mdcode), with one deliberate difference: `extract` refuses to overwrite pre-existing whole-file targets unless `--force` is given, and exits non-zero when it skips anything. See [Existing Files Are Not Overwritten](#existing-files-are-not-overwritten). ### Feature Parity @@ -750,7 +787,7 @@ These features are **not** in the original but are available in this implementat 2. **Library API** - Use mdcode programmatically in Node.js/TypeScript projects ```javascript - import mdcode from 'mdcode'; + import mdcode from 'mdcode-ts'; const result = await mdcode('README.md', transformer); ``` @@ -779,9 +816,9 @@ sudo apt remove mdcode # Linux ```bash # Global install -npm install -g mdcode +npm install -g mdcode-ts # or -pnpm install -g mdcode +pnpm install -g mdcode-ts ``` ### Step 3: Verify Installation @@ -812,7 +849,8 @@ mdcode list -l js README.md - ✅ Metadata parsing is the same - ✅ Region extraction works the same - ✅ Stdin/stdout behavior is identical -- ✅ Exit codes match original behavior +- ⚠️ `extract` skips existing whole-file targets instead of overwriting them — add `--force` to keep + the original behaviour, and expect exit code 2 when files are skipped ### Scripts and Automation @@ -841,7 +879,7 @@ Once migrated, you can optionally explore the bonus features: mdcode update --transform ./my-transformer.js README.md # Use as a library in your Node.js projects -npm install mdcode +npm install mdcode-ts ``` ### Getting Help @@ -851,7 +889,7 @@ If you encounter any issues: 1. Check the help output: `mdcode --help` 2. Run with verbose errors (stderr will show details) 3. Compare output with original using `--json` flag -4. Open an issue at: https://github.com/adrianbrowning/mdcode/issues +4. Open an issue at: https://github.com/adrianbrowning/mdcode-ts/issues --- @@ -860,8 +898,10 @@ If you encounter any issues: ### Workflow: Extract, Modify, Update ```bash -# 1. Extract code blocks to files -mdcode extract -d ./src README.md +# 1. Extract code blocks to files. +# --force is needed on re-runs: whole-file targets that already exist are +# skipped by default. Region blocks splice in place and never need it. +mdcode extract --force -d ./src README.md # 2. Edit the extracted files vim ./src/app.js @@ -918,7 +958,9 @@ mdcode extract -m type=example -d ./docs/examples README.md set -e echo "Extracting code blocks..." -mdcode extract -q -d ./temp README.md +# --force so a re-run overwrites the previous run's scratch files rather than +# skipping them and exiting non-zero. +mdcode extract -q --force -d ./temp README.md echo "Running linter..." mdcode run -l js "eslint {file}" README.md @@ -948,8 +990,8 @@ mdcode run -k -l js "node {file}" README.md ### 2. Combining with Other Tools ```bash -# Format code blocks with prettier -mdcode extract -l js -d temp README.md && \ +# Format code blocks with prettier (--force so re-runs refresh temp/) +mdcode extract -l js --force -d temp README.md && \ prettier --write temp/**/*.js && \ mdcode update README.md @@ -1012,14 +1054,14 @@ mdcode provides a powerful CLI for working with code blocks in Markdown files: Install globally and start using it today: ```bash -npm install -g mdcode +npm install -g mdcode-ts mdcode list README.md ``` Or try it without installing: ```bash -pnpm dlx mdcode list README.md +pnpm dlx mdcode-ts list README.md ``` -For more information, visit: https://github.com/adrianbrowning/mdcode +For more information, visit: https://github.com/adrianbrowning/mdcode-ts diff --git a/packages/mdcode/README.md b/packages/mdcode/README.md index 2e2ef06..ec7d8d5 100644 --- a/packages/mdcode/README.md +++ b/packages/mdcode/README.md @@ -2,7 +2,7 @@ A TypeScript port of [szkiba/mdcode](https://github.com/szkiba/mdcode) - a Markdown code block authoring tool for extracting, updating, and managing code blocks within markdown documents. -[![npm version](https://badge.fury.io/js/@mdcode%2Fmdcode.svg)](https://www.npmjs.com/package/@mdcode/mdcode) +[![npm version](https://badge.fury.io/js/mdcode-ts.svg)](https://www.npmjs.com/package/mdcode-ts) ## Drop-in Replacement @@ -82,10 +82,10 @@ Install globally to use the `mdcode` command anywhere: ```bash file=block-3.sh # Using npm -npm install -g @mdcode/mdcode +npm install -g mdcode-ts # Using pnpm -pnpm install -g @mdcode/mdcode +pnpm install -g mdcode-ts ``` After installation, you can run `mdcode` from anywhere: @@ -102,12 +102,12 @@ No installation required - run directly: ```bash file=block-5.sh # Using pnpm dlx -pnpm dlx @mdcode/mdcode list README.md -pnpm dlx @mdcode/mdcode extract --lang js docs/*.md +pnpm dlx mdcode-ts list README.md +pnpm dlx mdcode-ts extract --lang js docs/*.md # Using npx -npx @mdcode/mdcode list README.md -npx @mdcode/mdcode --help +npx mdcode-ts list README.md +npx mdcode-ts --help ``` ### Project Installation @@ -116,10 +116,10 @@ Install as a project dependency to use in scripts or via `pnpm exec`: ```bash file=block-6.sh # Using pnpm -pnpm add -D @mdcode/mdcode +pnpm add -D mdcode-ts # Using npm -npm install --save-dev @mdcode/mdcode +npm install --save-dev mdcode-ts ``` After installation, run via `pnpm exec`: @@ -293,6 +293,19 @@ mdcode list -l js -f "*.test.js" docs/ Extract code blocks to files based on their `file` metadata. +Extract is non-destructive. When the target file already exists: + +- **All blocks for that file declare `region=`** → each region body is spliced in place. Surrounding + code, and any regions in the file that the markdown doesn't declare, are preserved. +- **A declared region has no matching `#region` marker in the file** → the region is appended at the + end of the file, wrapped in markers written with the block language's comment syntax (`//`, `#`, + ``). Existing markers are matched in any of that language's comment styles, so `/* #region + name */` in a JS file is spliced rather than duplicated. +- **Any block for that file has no `region=`** → the file is skipped with a warning, since writing it + would replace the whole file. Use `--force` to overwrite. + +Files that don't exist yet are always created. + ### Basic Usage ```bash file=block-22.sh @@ -405,6 +418,21 @@ mdcode extract --ignore-anonymous -l js -d ./src docs/API.md **Note:** The flags `--update-source` and `--ignore-anonymous` are mutually exclusive. Using both will result in an error. +### Force Overwrite + +Blocks without `region=` describe a whole file, so extracting one over an existing file replaces it. +Those files are skipped by default; `--force` overwrites them: + +```bash file=block-force.sh +# Skipped with a warning if src/demo.ts already exists +mdcode extract README.md + +# Overwrite it +mdcode extract --force README.md +``` + +`--force` has no effect on region blocks — those always splice in place. + ### Stdin Behavior with Update Source When using stdin with `--update-source`, the updated markdown is written to stdout: @@ -789,6 +817,7 @@ Additional flags by command: - `-q, --quiet` - Suppress status messages - `--update-source` - Add file metadata to anonymous code blocks and update source - `--ignore-anonymous` - Skip blocks without file metadata (mutually exclusive with --update-source) +- `--force` - Overwrite existing files whose blocks have no `region=` (skipped by default) **update:** - `-d, --dir ` - Working directory for file resolution @@ -812,7 +841,7 @@ Additional flags by command: You can use mdcode programmatically in your Node.js or TypeScript projects: ```bash file=block-54.sh -pnpm add @mdcode/mdcode +pnpm add mdcode-ts ``` ### Simple API (Default Export) @@ -820,7 +849,7 @@ pnpm add @mdcode/mdcode The simplest way to use mdcode is with the default export: ```typescript file=block-55.ts -import mdcode from '@mdcode/mdcode'; +import mdcode from 'mdcode-ts'; // Transform a markdown file const result = await mdcode('/path/to/file.md', ({tag, meta, code}) => { @@ -866,13 +895,13 @@ import { type Block, type TransformerFunction, type FilterOptions, -} from '@mdcode/mdcode'; +} from 'mdcode-ts'; ``` ### Parse and Extract Code Blocks ````typescript file=block-58.ts -import { parse } from '@mdcode/mdcode'; +import { parse } from 'mdcode-ts'; const markdown = ` # Example @@ -900,7 +929,7 @@ const jsBlocks = parse({ ### Transform Code Blocks ````typescript file=block-60.ts -import { update, defineTransform } from '@mdcode/mdcode'; +import { update, defineTransform } from 'mdcode-ts'; const markdown = ` ```sql @@ -935,7 +964,7 @@ console.log(result); // Transformed markdown ### Async Transformers ```typescript file=block-62.ts -import { update, defineTransform } from '@mdcode/mdcode'; +import { update, defineTransform } from 'mdcode-ts'; const transformer = defineTransform(async ({tag, meta, code}) => { // Fetch from API, read files, etc. @@ -949,7 +978,7 @@ const result = await update({ source: markdown, transformer }); ### Custom Walker for Advanced Processing ```typescript file=block-63.md -import { walk, type Block } from '@mdcode/mdcode'; +import { walk, type Block } from 'mdcode-ts'; const result = await walk({ source: markdown, @@ -1156,7 +1185,7 @@ These features are **not** in the original but are available in this implementat 2. **Library API** - Use mdcode programmatically in Node.js/TypeScript projects ```javascript file=block-72.sh - import mdcode from '@mdcode/mdcode'; + import mdcode from 'mdcode-ts'; const result = await mdcode('README.md', transformer); ``` diff --git a/packages/mdcode/package.json b/packages/mdcode/package.json index e5e7480..25915a9 100644 --- a/packages/mdcode/package.json +++ b/packages/mdcode/package.json @@ -11,8 +11,8 @@ "bp": "pnpm version prerelease --no-git-tag-version && pnpm --filter mdcode-ts build && pnpm pack", "build": "zshy", "dev": "zshy --watch", - "test": "node --test-reporter=spec --test {tests,src}/**/*.test.ts", - "test:watch": "node -w --test-reporter=spec --test {tests,src}/**/*.test.ts", + "test": "node --test-reporter=spec --test './{tests,src}/**/*.test.ts'", + "test:watch": "node --watch --test-reporter=spec --test './{tests,src}/**/*.test.ts'", "prepublishOnly": "pnpm build", "lint:ts": "tsc --noEmit", "lint": "tsc --noEmit ; eslint \"src/**/*.{j,t}s{,x}\" --cache --max-warnings=0", diff --git a/packages/mdcode/src/cli.ts b/packages/mdcode/src/cli.ts index 952025e..e3eeb36 100644 --- a/packages/mdcode/src/cli.ts +++ b/packages/mdcode/src/cli.ts @@ -31,6 +31,18 @@ async function readInput(filePath?: string): Promise { return Buffer.concat(chunks).toString("utf-8"); } +/** Flags accepted by `mdcode extract`, so a renamed flag is a compile error. */ +type ExtractCliOptions = { + lang?: string; + file?: string; + meta?: Record; + dir: string; + quiet?: boolean; + updateSource?: boolean; + ignoreAnonymous?: boolean; + force?: boolean; +}; + /** * Parse filter options from command-line flags */ @@ -113,7 +125,8 @@ export async function Execute( .option("-q, --quiet", "Suppress status messages") .option("--update-source", "Add file metadata to anonymous code blocks") .option("--ignore-anonymous", "Skip blocks without file metadata") - .action(async (file, options) => { + .option("--force", "Overwrite existing files whose blocks have no region=") + .action(async (file: string | undefined, options: ExtractCliOptions) => { try { // Validation if (options.updateSource && options.ignoreAnonymous) { @@ -132,7 +145,7 @@ export async function Execute( quiet: options.quiet, updateSource: options.updateSource, ignoreAnonymous: options.ignoreAnonymous, - sourcePath: file, + force: options.force, }); // Handle --update-source behavior @@ -149,6 +162,14 @@ export async function Execute( stdout.write(result.updatedSource); } } + + // A refusal to write must be distinguishable from success by CI and by + // scripts, so it reports even under --quiet and fails the process. + if (result.skippedFiles.length > 0) { + stderr.write(styleText("yellow", `⚠ Skipped ${result.skippedFiles.length} file(s); nothing was written for them\n`)); + // eslint-disable-next-line no-process-exit + process.exit(2); + } } catch (error: unknown) { if(error instanceof Error) stderr.write(`Error: ${error.message}\n`); diff --git a/packages/mdcode/src/commands/extract.test.ts b/packages/mdcode/src/commands/extract.test.ts new file mode 100644 index 0000000..328a4d6 --- /dev/null +++ b/packages/mdcode/src/commands/extract.test.ts @@ -0,0 +1,526 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ +import assert from "node:assert/strict"; +import { chmod, lstat, mkdir, mkdtemp, readdir, readFile, readlink, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, describe, mock, test } from "node:test"; + +import { extract } from "./extract.ts"; + +const tempDirs: Array = []; + +after(async () => { + await Promise.all(tempDirs.map(async d => rm(d, { recursive: true, force: true }))); +}); + +async function tempDir(): Promise { + const dir = await mkdtemp(join(tmpdir(), "mdcode-extract-")); + tempDirs.push(dir); + return dir; +} + +async function writeSource(dir: string, relPath: string, content: string): Promise { + const full = join(dir, relPath); + await mkdir(join(full, ".."), { recursive: true }); + await writeFile(full, content, "utf-8"); + return full; +} + +describe("extract: in-place region splice", () => { + test("preserves surrounding code and regions absent from the markdown", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "src/demo.ts", [ + `import assert from "node:assert/strict";`, + "", + "// #region alpha", + "const alpha = 1;", + "// #endregion alpha", + "", + "// #region beta", + "const beta = 2;", + "// #endregion beta", + "", + "assert.equal(alpha + beta, 3);", + "", + ].join("\n")); + + const source = [ + "```typescript file=./src/demo.ts region=alpha", + "const alpha = 42;", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + const result = await readFile(target, "utf-8"); + + assert.match(result, /import assert from "node:assert\/strict";/, "import must survive"); + assert.match(result, /assert\.equal\(alpha \+ beta, 3\);/, "assertion must survive"); + assert.match(result, /#region beta[\s\S]*const beta = 2;[\s\S]*#endregion beta/, "undeclared region must survive"); + assert.match(result, /#region alpha\nconst alpha = 42;\n\/\/ #endregion alpha/, "declared region body must be replaced"); + assert.doesNotMatch(result, /const alpha = 1;/, "old region body must be gone"); + }); + + test("appends a region declared in markdown but absent from the file", async () => { + const dir = await tempDir(); + const original = [ + `import assert from "node:assert/strict";`, + "", + "// #region alpha", + "const alpha = 1;", + "// #endregion alpha", + "", + ].join("\n"); + const target = await writeSource(dir, "src/demo.ts", original); + + const source = [ + "```typescript file=./src/demo.ts region=gamma", + "const gamma = 3;", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + const result = await readFile(target, "utf-8"); + + assert.ok(result.startsWith(original), "existing content must be untouched"); + assert.match(result, /\/\/ #region gamma\nconst gamma = 3;\n\/\/ #endregion gamma/, "missing region must be appended"); + }); + + test("aliased file= paths resolving to one file do not clobber each other", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "real/demo.ts", [ + "// #region alpha", + "const alpha = 1;", + "// #endregion alpha", + "", + "// #region beta", + "const beta = 2;", + "// #endregion beta", + "", + ].join("\n")); + await symlink(join(dir, "real"), join(dir, "link"), "dir"); + + const source = [ + "```typescript file=./real/demo.ts region=alpha", + "const alpha = 42;", + "```", + "", + "```typescript file=./link/demo.ts region=beta", + "const beta = 99;", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + const result = await readFile(target, "utf-8"); + + assert.match(result, /const alpha = 42;/, "region from first spelling must survive"); + assert.match(result, /const beta = 99;/, "region from aliased spelling must survive"); + }); + + test("splices hash-comment regions instead of duplicating them", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "src/demo.py", [ + "import sys", + "", + "# #region greet", + "print('old')", + "# #endregion greet", + "", + "sys.exit(0)", + "", + ].join("\n")); + + const source = [ + "```python file=./src/demo.py region=greet", + "print('new')", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + const result = await readFile(target, "utf-8"); + + assert.equal(result.match(/#region greet/g)?.length, 1, "region must not be duplicated"); + assert.match(result, /# #region greet\nprint\('new'\)\n# #endregion greet/); + assert.match(result, /sys\.exit\(0\)/, "surrounding code must survive"); + }); + + test("splices regions whose markers use block comments", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "app.js", [ + "const keep = 1;", + "/* #region body */", + "old();", + "/* #endregion body */", + "const tail = 2;", + "", + ].join("\n")); + + const source = [ + "```js file=app.js region=body", + "fresh();", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + const result = await readFile(target, "utf-8"); + + assert.equal(result.match(/#region body/g)?.length, 1, "region must not be duplicated"); + assert.match(result, /\/\* #region body \*\/\nfresh\(\);\n\/\* #endregion body \*\//); + assert.doesNotMatch(result, /old\(\);/, "old region body must be gone"); + assert.match(result, /const keep = 1;[\s\S]*const tail = 2;/, "surrounding code must survive"); + }); + + test("re-matches markers it appended itself, so repeated runs are idempotent", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "page.html", "

existing

\n"); + + const source = [ + "```html file=page.html region=body", + "

fresh

", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + const first = await readFile(target, "utf-8"); + + assert.match(first, /\n

fresh<\/p>\n/, "marker must use the language's comment syntax"); + + await extract({ source, outputDir: dir, quiet: true }); + const second = await readFile(target, "utf-8"); + + assert.equal(second, first, "a second run must splice, not append again"); + assert.equal(second.match(/#region body/g)?.length, 1, "region must not be duplicated"); + }); +}); + +describe("extract: --force for non-region overwrites", () => { + const source = [ + "```typescript file=./src/demo.ts", + "const replaced = true;", + "```", + "", + ].join("\n"); + + test("leaves an existing file untouched without force", async () => { + const dir = await tempDir(); + const original = "const original = true;\n"; + const target = await writeSource(dir, "src/demo.ts", original); + + await extract({ source, outputDir: dir, quiet: true }); + + assert.equal(await readFile(target, "utf-8"), original, "file must not be overwritten"); + }); + + test("overwrites an existing file with force", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "src/demo.ts", "const original = true;\n"); + + await extract({ source, outputDir: dir, quiet: true, force: true }); + + assert.equal(await readFile(target, "utf-8"), "const replaced = true;", "force must overwrite"); + }); + + test("still creates a missing file without force", async () => { + const dir = await tempDir(); + + await extract({ source, outputDir: dir, quiet: true }); + + assert.equal(await readFile(join(dir, "src/demo.ts"), "utf-8"), "const replaced = true;"); + }); +}); + +describe("extract: refuses unsafe targets", () => { + test("refuses a file= that escapes the output directory", async () => { + const dir = await tempDir(); + const outside = await writeSource(dir, "outside.txt", "PRECIOUS\n"); + const out = join(dir, "out"); + await mkdir(out, { recursive: true }); + + const source = [ + "```text file=../outside.txt region=alpha", + "pwned", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: out, quiet: true, force: true }); + + assert.equal(await readFile(outside, "utf-8"), "PRECIOUS\n", "a path outside --dir must not be written"); + assert.deepEqual(result.extractedFiles, []); + assert.equal(result.skippedFiles.length, 1); + }); + + test("refuses an absolute file=", async () => { + const dir = await tempDir(); + const outside = await writeSource(dir, "abs.txt", "PRECIOUS\n"); + + const source = [ + `\`\`\`text file=${outside}`, + "pwned", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: join(dir, "out"), quiet: true, force: true }); + + assert.equal(await readFile(outside, "utf-8"), "PRECIOUS\n"); + assert.deepEqual(result.extractedFiles, []); + }); + + test("refuses to splice through a symlinked target, leaving the link and its target intact", async () => { + const dir = await tempDir(); + const real = await writeSource(dir, "real.ts", [ + "// #region alpha", + "const alpha = 1;", + "// #endregion alpha", + "", + ].join("\n")); + await symlink(real, join(dir, "link.ts")); + + const source = [ + "```typescript file=link.ts region=alpha", + "const alpha = 99;", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: dir, quiet: true }); + + assert.ok(!(await readFile(real, "utf-8")).includes("99"), "the link target must not be rewritten"); + assert.ok((await lstat(join(dir, "link.ts"))).isSymbolicLink(), "the link must still be a link"); + assert.equal(await readlink(join(dir, "link.ts")), real); + assert.deepEqual(result.skippedFiles, [ join(dir, "link.ts") ]); + }); + + test("refuses a target that is not valid UTF-8, byte for byte", async () => { + const dir = await tempDir(); + const target = join(dir, "bin.ts"); + const bytes = Buffer.concat([ + Buffer.from("// #region alpha\n"), + Buffer.from([ 0xff, 0xfe ]), + Buffer.from("\n// #endregion alpha\n"), + ]); + await writeFile(target, bytes); + + const source = [ + "```typescript file=bin.ts region=alpha", + "const alpha = 1;", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: dir, quiet: true }); + + assert.ok(bytes.equals(await readFile(target)), "invalid bytes must survive untouched"); + assert.deepEqual(result.skippedFiles, [ target ]); + }); + + test("refuses a group mixing region and whole-file blocks, even with force", async () => { + const dir = await tempDir(); + const original = [ + "// #region alpha", + "const alpha = 1;", + "// #endregion alpha", + "const keep = 2;", + "", + ].join("\n"); + const target = await writeSource(dir, "m.ts", original); + + const source = [ + "```typescript file=m.ts region=alpha", + "const alpha = 42;", + "```", + "", + "```typescript file=m.ts", + "whole file", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: dir, quiet: true, force: true }); + + assert.equal(await readFile(target, "utf-8"), original, "a mixed group has no coherent result"); + assert.deepEqual(result.skippedFiles, [ target ]); + }); + + test("refuses to splice a region the target never closes", async () => { + const dir = await tempDir(); + const original = [ + "const keep = 1;", + "// #region alpha", + "const old = 1;", + "export function important() { return 42; }", + "", + ].join("\n"); + const target = await writeSource(dir, "u.ts", original); + + const source = [ + "```typescript file=u.ts region=alpha", + "const alpha = 42;", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: dir, quiet: true }); + + assert.equal(await readFile(target, "utf-8"), original, "an unclosed region must never be written"); + assert.deepEqual(result.skippedFiles, [ target ]); + }); +}); + +describe("extract: write fidelity", () => { + test("preserves the target's permission bits across a splice", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "run.sh", [ + "#!/bin/sh", + "# #region body", + "echo old", + "# #endregion body", + "", + ].join("\n")); + await chmod(target, 0o755); + + const source = [ + "```bash file=run.sh region=body", + "echo new", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + assert.match(await readFile(target, "utf-8"), /echo new/); + assert.equal((await stat(target)).mode & 0o777, 0o755, "an executable target must stay executable"); + }); + + test("skips outline=true blocks instead of splicing a marker skeleton over real code", async () => { + const dir = await tempDir(); + const original = [ + "// #region alpha", + "const realImplementation = 1;", + "// #endregion alpha", + "", + ].join("\n"); + const target = await writeSource(dir, "o.ts", original); + + const source = [ + "```typescript file=o.ts region=alpha outline=true", + "// #region alpha", + "// #endregion alpha", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: dir, quiet: true }); + + assert.equal(await readFile(target, "utf-8"), original, "an outline block describes shape, not content"); + assert.deepEqual(result.extractedFiles, []); + }); + + test("leaves no temp file behind after an atomic write", async () => { + const dir = await tempDir(); + await writeSource(dir, "a.ts", "// #region alpha\nold\n// #endregion alpha\n"); + + const source = [ + "```typescript file=a.ts region=alpha", + "fresh", + "```", + "", + ].join("\n"); + + await extract({ source, outputDir: dir, quiet: true }); + + const entries = await readdir(dir); + + assert.deepEqual(entries, [ "a.ts" ], "the sibling temp file must be renamed away"); + }); +}); + +describe("extract: reporting", () => { + test("aliased non-region blocks resolve to one group, one write, and no false skip", async () => { + const dir = await tempDir(); + await mkdir(join(dir, "real"), { recursive: true }); + await symlink(join(dir, "real"), join(dir, "link"), "dir"); + + // Same content via two spellings: the point under test is that they resolve + // to one target, not what happens when they disagree. + const source = [ + "```typescript file=./real/demo.ts", + "const shared = 1;", + "```", + "", + "```typescript file=./link/demo.ts", + "const shared = 1;", + "```", + "", + ].join("\n"); + + const result = await extract({ source, outputDir: dir, quiet: true }); + + assert.equal(result.extractedFiles.length, 1, "one physical file must be reported once"); + assert.deepEqual(result.skippedFiles, [], "a file this run just created must not report as pre-existing"); + assert.deepEqual(await readdir(join(dir, "real")), [ "demo.ts" ], "no second copy via the link"); + }); + + test("refuses whole-file blocks that disagree, and allows identical ones", async () => { + const dir = await tempDir(); + + const disagreeing = [ + "```typescript file=d.ts", + "const first = 1;", + "```", + "", + "```typescript file=d.ts", + "const second = 2;", + "```", + "", + ].join("\n"); + + const refused = await extract({ source: disagreeing, outputDir: dir, quiet: true }); + + assert.deepEqual(refused.extractedFiles, [], "one of the two blocks would have been discarded"); + assert.equal(refused.skippedFiles.length, 1); + + const agreeing = disagreeing.replace("const second = 2;", "const first = 1;"); + const accepted = await extract({ source: agreeing, outputDir: dir, quiet: true }); + + assert.equal(accepted.extractedFiles.length, 1, "identical blocks are not ambiguous"); + assert.equal(await readFile(join(dir, "d.ts"), "utf-8"), "const first = 1;"); + }); + + test("names the file and the reason when it refuses to write", async () => { + const dir = await tempDir(); + const target = await writeSource(dir, "s.ts", "const original = true;\n"); + const lines: Array = []; + mock.method(console, "error", (...args: Array) => { + lines.push(args.map(String).join(" ")); + }); + + try { + const result = await extract({ + source: "```typescript file=s.ts\nconst replaced = true;\n```\n", + outputDir: dir, + }); + + const stderr = lines.join("\n"); + + assert.match(stderr, /Skipped/); + assert.match(stderr, /s\.ts/, "the warning must name the file"); + assert.match(stderr, /--force/, "the warning must name the way forward"); + assert.deepEqual(result.skippedFiles, [ target ]); + } + finally { + mock.restoreAll(); + } + }); +}); diff --git a/packages/mdcode/src/commands/extract.ts b/packages/mdcode/src/commands/extract.ts index ee87d67..ff981d6 100644 --- a/packages/mdcode/src/commands/extract.ts +++ b/packages/mdcode/src/commands/extract.ts @@ -1,8 +1,10 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { chmod, lstat, mkdir, readFile, realpath, rename, stat, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { styleText } from "node:util"; import { parse, updateInfoStrings } from "../parser.ts"; +import type { RegionEdit } from "../region.ts"; +import { isValidRegionName, spliceRegions, wrapRegion } from "../region.ts"; import type { FilterOptions } from "../types.ts"; export type ExtractOptions = { @@ -12,14 +14,27 @@ export type ExtractOptions = { quiet?: boolean; updateSource?: boolean; ignoreAnonymous?: boolean; - sourcePath?: string; + force?: boolean; }; -type ExtractResult = { +export type ExtractResult = { extractedFiles: Array; + /** Targets deliberately left untouched; each one also produced a warning. */ + skippedFiles: Array; updatedSource?: string; }; +type BlockRef = { + block: { meta: Record; lang: string; code: string; }; + index: number; +}; + +type TargetGroup = { + /** Path as written in the markdown, for messages. */ + display: string; + items: Array; +}; + /** * Extract code blocks to files based on their metadata */ @@ -31,6 +46,7 @@ export async function extract(options: ExtractOptions): Promise { quiet = false, updateSource = false, ignoreAnonymous = false, + force = false, } = options; // Validate mutual exclusivity @@ -51,7 +67,7 @@ export async function extract(options: ExtractOptions): Promise { if (!quiet) { console.error(styleText("yellow", "No code blocks with file metadata found.")); } - return { extractedFiles: [] }; + return { extractedFiles: [], skippedFiles: [] }; } } @@ -59,97 +75,131 @@ export async function extract(options: ExtractOptions): Promise { if (!quiet) { console.error(styleText("yellow", "No code blocks found to extract.")); } - return { extractedFiles: [] }; + return { extractedFiles: [], skippedFiles: [] }; } // Track generated filenames for anonymous blocks (for --update-source) const metadataUpdates = new Map>(); - // Group blocks by file path - const fileMap = new Map; lang: string; code: string; }; index: number; }>>(); + const extractedFiles: Array = []; + const skippedFiles: Array = []; + const root = resolve(outputDir); + + // The root must exist before it can be canonicalised: realpath on a missing + // directory falls back to the lexical path, and on macOS comparing a lexical + // /var/... root against a resolved /private/var/... target reads as an escape. + await mkdir(root, { recursive: true }); + const canonicalRoot = await realpath(root).catch(() => root); + + const skip = (path: string, reason: string): void => { + skippedFiles.push(path); + if (!quiet) { + console.error(styleText("yellow", `⚠ Skipped ${path}: ${reason}`)); + } + }; + + // Group blocks by the file they resolve to. Keying on the resolved path means + // two spellings of one file (`./src/a.ts` and a symlinked `./link/a.ts`) form + // a single group and produce a single write, rather than racing each other. + const groups = new Map(); for (const block of blocks) { - // Find the original index of this block in allBlocks const index = allBlocks.findIndex(b => b.position?.start === block.position?.start); - let filePath: string; - let generatedFilename: string | undefined; - - if (block.meta.file) { - filePath = join(outputDir, block.meta.file); + // extract writes files; a block asking for a marker-only skeleton has no + // file content to contribute, so it is not an extraction target. + if (block.meta.outline === "true") { + continue; } - else { - // Generate a filename if not specified - const ext = getExtensionForLang(block.lang); - generatedFilename = `block-${index + 1}${ext}`; - filePath = join(outputDir, generatedFilename); - // Track for --update-source + let declared = block.meta.file; + + if (declared === undefined) { + const generated = `block-${index + 1}${getExtensionForLang(block.lang)}`; + declared = generated; + if (updateSource && index >= 0) { - metadataUpdates.set(index, { file: generatedFilename }); + metadataUpdates.set(index, { file: generated }); } } - if (!fileMap.has(filePath)) { - fileMap.set(filePath, []); + const display = join(outputDir, declared); + + // Reject an escape before creating any directory for it. + if (isAbsolute(declared) || escapesRoot(root, display)) { + skip(display, `file= must stay inside ${outputDir}`); + continue; } - fileMap.get(filePath)!.push({ block, index }); - } - const extractedFiles: Array = []; + if (block.meta.region !== undefined && !isValidRegionName(block.meta.region)) { + skip(display, `invalid region name ${JSON.stringify(block.meta.region)}`); + continue; + } - // Write files, handling multiple regions per file - for (const [ filePath, items ] of fileMap.entries()) { - // Create directory if needed - const dir = dirname(filePath); - await mkdir(dir, { recursive: true }); + await mkdir(dirname(display), { recursive: true }); - // If all blocks for this file have regions, combine them with markers - const allHaveRegions = items.every(item => item.block.meta.region); + const key = await resolveTarget(display); - if (allHaveRegions && items.length > 1) { - // Combine multiple regions into one file - const lang = items?.[0]?.block.lang || "text"; + if (escapesRoot(canonicalRoot, key)) { + skip(display, `file= resolves outside ${outputDir}`); + continue; + } - const commentStyle = getCommentStyle(lang); - const parts: Array = []; + const group = groups.get(key); - for (const { block } of items) { - parts.push(`${commentStyle} #region ${block.meta.region}`); - parts.push(block.code); - parts.push(`${commentStyle} #endregion ${block.meta.region}`); - parts.push(""); // Empty line between regions - } + if (group) { + group.items.push({ block, index }); + } + else { + groups.set(key, { display, items: [{ block, index }] }); + } + } - await writeFile(filePath, parts.join("\n").trim() + "\n", "utf-8"); - if (!quiet) { - console.error(styleText("green", `✓ Extracted ${items.length} region(s) to ${filePath}`)); - } + for (const [ , { display, items }] of groups) { + const withRegion = items.filter(item => item.block.meta.region !== undefined); + const existing = await stat(display).catch(rethrowUnlessMissing); + + // A group mixing whole-file and region blocks has no coherent result: the + // whole-file block would erase the very region the other block splices. + if (withRegion.length > 0 && withRegion.length !== items.length) { + skip(display, "blocks for this file mix region= with whole-file blocks"); + continue; } - else if (items.length === 1 && items[0]?.block.meta.region) { - // Single region - wrap with markers - const { block } = items[0]; - const commentStyle = getCommentStyle(block.lang); - const content = [ - `${commentStyle} #region ${block.meta.region}`, - block.code, - `${commentStyle} #endregion ${block.meta.region}`, - ].join("\n") + "\n"; - - await writeFile(filePath, content, "utf-8"); - if (!quiet) { - console.error(styleText("green", `✓ Extracted to ${filePath}`)); - } + + // Several blocks each claiming to be the whole file only agree if they are + // byte-identical; otherwise picking one would silently discard the others. + if (withRegion.length === 0 && new Set(items.map(item => item.block.code)).size > 1) { + skip(display, `${items.length} whole-file blocks disagree about its contents`); + continue; } - else { - // No regions or mixed - write the first block's code - await writeFile(filePath, items[0]?.block.code || "", "utf-8"); - if (!quiet) { - console.error(styleText("green", `✓ Extracted to ${filePath}`)); + + if (existing !== undefined && withRegion.length === items.length) { + const spliced = await spliceInPlace(display, items, { quiet, skip }); + + if (spliced) { + extractedFiles.push(display); } + continue; } - extractedFiles.push(filePath); + if (existing !== undefined && !force) { + skip(display, "exists and has block(s) without region=. Use --force to overwrite."); + continue; + } + + const content = withRegion.length === items.length + ? items.map(({ block }) => wrapRegion(block.lang, block.meta.region!, block.code)).join("\n") + : items[0]!.block.code; + + await writeAtomic(display, content, existing?.mode); + + if (!quiet) { + const what = withRegion.length === items.length && items.length > 1 + ? `${items.length} region(s) to` + : "to"; + console.error(styleText("green", `✓ Extracted ${what} ${display}`)); + } + extractedFiles.push(display); } // Update source if requested @@ -158,7 +208,131 @@ export async function extract(options: ExtractOptions): Promise { updatedSourceContent = updateInfoStrings(source, metadataUpdates); } - return { extractedFiles, updatedSource: updatedSourceContent }; + return { extractedFiles, skippedFiles, updatedSource: updatedSourceContent }; +} + +/** + * Splice every region block for one existing file in a single pass, appending + * any region the file does not already declare. Returns false when the file was + * left untouched. + */ +async function spliceInPlace( + target: string, + items: Array, + reporters: { quiet: boolean; skip: (path: string, reason: string) => void; } +): Promise { + const { quiet, skip } = reporters; + + // rename() would replace a symlink with a regular file rather than write + // through it; refuse outright so the link's meaning is never silently changed. + if ((await lstat(target)).isSymbolicLink()) { + skip(target, "target is a symlink; refusing to splice through it"); + return false; + } + + const raw = await readFile(target); + let existing: string; + + try { + // A lossy decode would rewrite every invalid byte in the file as U+FFFD, + // even though only one region was asked for. + existing = new TextDecoder("utf-8", { fatal: true }).decode(raw); + } + catch { + skip(target, "not valid UTF-8"); + return false; + } + + const edits = new Map( + items.map(({ block }) => [ block.meta.region!, { code: block.code, lang: block.lang }]) + ); + const result = spliceRegions(existing, edits); + + if (!result.ok) { + skip(target, spliceRefusal(result)); + return false; + } + + let content = result.content; + + // A region the markdown declares but the file lacks is appended rather than + // dropped, so the block is not silently lost. + for (const name of result.unmatched) { + const { block } = items.find(item => item.block.meta.region === name)!; + const separator = content.trim() === "" ? "" : "\n"; + content = `${content.replace(/\n*$/, content.trim() === "" ? "" : "\n")}${separator}${wrapRegion(block.lang, name, block.code)}`; + } + + await writeAtomic(target, content, (await stat(target)).mode); + + if (!quiet) { + console.error(styleText("green", `✓ Updated ${items.length} region(s) in ${target}`)); + } + + return true; +} + +/** Explain, in one clause, why a splice was refused. */ +function spliceRefusal(result: { unclosed: Array; duplicated: Array; overlapping: Array; }): string { + if (result.unclosed.length > 0) { + return `region ${result.unclosed.join(", ")} is never closed`; + } + if (result.duplicated.length > 0) { + return `region ${result.duplicated.join(", ")} appears more than once`; + } + + return `regions ${result.overlapping.join(", ")} overlap`; +} + +/** + * Write via a sibling temp file and rename, so an interrupted or out-of-space + * write cannot leave a hand-written source file truncated. rename() drops the + * destination's permissions, so they are copied over first. + * + * Tradeoff: rename() replaces the file rather than rewriting it, so the target + * gets a new inode. Hard links to the old file keep the old contents, and + * extended attributes and ACLs are not carried over. Both are accepted because + * the alternative — truncate and rewrite in place — can leave a source file + * half-written, which is worse. It also means a symlinked target would be + * replaced rather than followed, which is why the splice path refuses symlinks + * outright instead of relying on this. + */ +async function writeAtomic(target: string, content: string, mode?: number): Promise { + const temp = join(dirname(target), `.${basename(target)}.mdcode-${process.pid}`); + + await writeFile(temp, content, "utf-8"); + + if (mode !== undefined) { + await chmod(temp, mode & 0o7777); + } + + await rename(temp, target); +} + +/** ENOENT means "not there yet"; anything else is a real failure to surface. */ +function rethrowUnlessMissing(error: unknown): undefined { + if (error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return undefined; + } + throw error; +} + +/** True when `path` is not inside `root`. */ +function escapesRoot(root: string, path: string): boolean { + const rel = relative(root, resolve(path)); + + return rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel); +} + +/** + * Canonical identity for a target: the realpath of its parent (which exists by + * now) plus its own name, so aliased spellings collapse to one key without + * requiring the file itself to exist. + */ +async function resolveTarget(path: string): Promise { + const parent = await realpath(dirname(path)).catch(() => resolve(dirname(path))); + + return join(parent, basename(path)); } /** @@ -207,36 +381,3 @@ function getExtensionForLang(lang: string): string { return extensions[lang.toLowerCase()] || ".txt"; } - -/** - * Get comment style for a given language - */ -function getCommentStyle(lang: string): string { - const styles: Record = { - js: "//", - javascript: "//", - ts: "//", - typescript: "//", - java: "//", - c: "//", - cpp: "//", - "c++": "//", - cs: "//", - "c#": "//", - go: "//", - rust: "//", - swift: "//", - kotlin: "//", - php: "//", - py: "#", - python: "#", - rb: "#", - ruby: "#", - sh: "#", - bash: "#", - yaml: "#", - yml: "#", - }; - - return styles[lang.toLowerCase()] || "//"; -} diff --git a/packages/mdcode/src/commands/update.test.ts b/packages/mdcode/src/commands/update.test.ts new file mode 100644 index 0000000..fa28824 --- /dev/null +++ b/packages/mdcode/src/commands/update.test.ts @@ -0,0 +1,111 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ +import * as assert from "node:assert"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { after, describe, it, mock } from "node:test"; + +import { update } from "./update.ts"; + +const dirs: Array = []; + +async function tempDir(): Promise { + const dir = await mkdtemp(join(tmpdir(), "mdcode-update-")); + dirs.push(dir); + return dir; +} + +async function writeSource(dir: string, relative: string, content: string): Promise { + const target = join(dir, relative); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, content, "utf-8"); + return target; +} + +/** + * Capture console.error for the duration of one test; update.ts reports region + * failures there. `mock.restoreAll()` in a `finally` keeps the stub from leaking + * into sibling tests even when an assertion throws. + */ +async function captureStderr(run: () => Promise): Promise<{ result: string; stderr: string; }> { + const lines: Array = []; + mock.method(console, "error", (...args: Array) => { + lines.push(args.map(String).join(" ")); + }); + + try { + return { result: await run(), stderr: lines.join("\n") }; + } + finally { + mock.restoreAll(); + } +} + +after(async () => { + await Promise.all(dirs.map(async dir => rm(dir, { recursive: true, force: true }))); +}); + +describe("update from file regions", () => { + it("leaves the block unchanged and reports when the region is never closed", async () => { + const dir = await tempDir(); + await writeSource(dir, "a.js", [ + "const keep = 1;", + "// #region alpha", + "const inside = 2;", + "", + ].join("\n")); + + const source = [ + "```js file=a.js region=alpha", + "ORIGINAL", + "```", + "", + ].join("\n"); + + const { result, stderr } = await captureStderr(async () => update({ source, basePath: dir, quiet: true })); + + assert.equal(result, source, "an unterminated region must not rewrite the markdown block"); + assert.match(stderr, /Failed to read a\.js/); + assert.match(stderr, /alpha/); + }); + + it("leaves the block unchanged and reports when the region is absent", async () => { + const dir = await tempDir(); + await writeSource(dir, "a.js", "const keep = 1;\n"); + + const source = [ + "```js file=a.js region=missing", + "ORIGINAL", + "```", + "", + ].join("\n"); + + const { result, stderr } = await captureStderr(async () => update({ source, basePath: dir, quiet: true })); + + assert.equal(result, source, "a missing region must not empty the markdown block"); + assert.match(stderr, /Failed to read a\.js/); + }); + + it("still fills the block from a well-formed region", async () => { + const dir = await tempDir(); + await writeSource(dir, "a.js", [ + "const keep = 1;", + "// #region alpha", + "const inside = 2;", + "// #endregion alpha", + "", + ].join("\n")); + + const source = [ + "```js file=a.js region=alpha", + "ORIGINAL", + "```", + "", + ].join("\n"); + + const result = await update({ source, basePath: dir, quiet: true }); + + assert.match(result, /const inside = 2;/); + assert.ok(!result.includes("ORIGINAL")); + }); +}); diff --git a/packages/mdcode/src/commands/update.ts b/packages/mdcode/src/commands/update.ts index bc04111..67558aa 100644 --- a/packages/mdcode/src/commands/update.ts +++ b/packages/mdcode/src/commands/update.ts @@ -53,7 +53,13 @@ export async function update(options: UpdateOptions): Promise { } // If a region is specified (and not using outline), extract only that region else if (block.meta.region) { - fileContent = readRegion(fileContent, block.meta.region, block.lang).content/*.trim()*/; + const region = readRegion(fileContent, block.meta.region, block.lang); + + if (!region.found) { + throw new Error(`region ${block.meta.region} not found or not closed in ${filePath}`); + } + + fileContent = region.content; } currentCode = fileContent; diff --git a/packages/mdcode/src/index.ts b/packages/mdcode/src/index.ts index a90705e..f52817e 100644 --- a/packages/mdcode/src/index.ts +++ b/packages/mdcode/src/index.ts @@ -9,6 +9,7 @@ export * from "./parser.ts"; export * from "./cli.ts"; // Export commands for programmatic use +export type { ExtractOptions, ExtractResult } from "./commands/extract.ts"; export { extract } from "./commands/extract.ts"; export { update } from "./commands/update.ts"; export { list } from "./commands/list.ts"; @@ -26,7 +27,7 @@ export { transform, transformWithFunction } from "./commands/transform.ts"; * * @example * ```typescript - * import mdcode from '@gcm/mdcode'; + * import mdcode from 'mdcode-ts'; * * const result = await mdcode('/path/to/file.md', (tag, meta, code) => { * if (tag === 'sql') return code.toUpperCase(); diff --git a/packages/mdcode/src/region.test.ts b/packages/mdcode/src/region.test.ts index f947ee7..29e886f 100644 --- a/packages/mdcode/src/region.test.ts +++ b/packages/mdcode/src/region.test.ts @@ -4,7 +4,7 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, it } from "node:test"; -import { outline, read, replace } from "./region.ts"; +import { getCommentStyle, outline, read, replace, spliceRegions, wrapRegion } from "./region.ts"; // Helper to load test fixtures async function loadFixture(filename: string): Promise { @@ -61,6 +61,22 @@ content here assert.equal(result.found, true); assert.equal(result.content, "content here"); }); + + it("reads a block-comment region when given a C-family lang", () => { + const source = "/* #region body */\nx();\n/* #endregion body */"; + + const result = read(source, "body", "ts"); + + assert.equal(result.found, true, "C-family langs accept /* as well as //"); + assert.equal(result.content, "x();"); + }); + + it("does not read a region whose marker uses another language's syntax", () => { + const source = "# #region body\nx()\n# #endregion body"; + + assert.equal(read(source, "body", "ts").found, false, "# is not a TypeScript comment"); + assert.equal(read(source, "body", "python").found, true); + }); }); describe("region.outline", () => { @@ -122,6 +138,26 @@ function test() { }); describe("region.replace", () => { + it("should replace hash-comment regions when given a lang", () => { + const source = [ + "import sys", + "", + "# #region greet", + "print('old')", + "# #endregion greet", + "", + "sys.exit(0)", + ].join("\n"); + + const result = replace(source, "greet", "print('new')", "python"); + + assert.equal(result.found, true); + assert.match(result.content, /# #region greet\nprint\('new'\)\n# #endregion greet/); + assert.ok(!result.content.includes("print('old')")); + assert.match(result.content, /^import sys/); + assert.match(result.content, /sys\.exit\(0\)$/); + }); + it("should replace content in regions", async () => { const source = await loadFixture("testdoc.js"); @@ -311,3 +347,256 @@ content 2 assert.ok(!result.content.includes("content 1")); }); }); + +describe("region.replace refuses to destroy", () => { + it("leaves an unterminated region untouched instead of truncating to EOF", () => { + const source = [ + "const keep = 1;", + "// #region alpha", + "const old = 1;", + "export function important() { return 42; }", + "const tail = 3;", + "", + ].join("\n"); + + const result = replace(source, "alpha", "fresh", "ts"); + + assert.equal(result.closed, false, "an unclosed region must be reported"); + assert.equal(result.content, source, "source must be returned byte-identical"); + }); + + it("closes on a JSDoc-style end marker", () => { + const source = "head\n// #region alpha\nold\n/** #endregion alpha */\nconst tail = 3;\n"; + + const result = replace(source, "alpha", "fresh", "ts"); + + assert.equal(result.closed, true); + assert.ok(result.content.includes("const tail = 3;"), "content after the region must survive"); + assert.ok(!result.content.includes("old")); + }); + + it("closes on a trailing-position end marker", () => { + const source = "function f() {\n// #region alpha\nold\n} // #endregion alpha\nconst tail = 3;\n"; + + const result = replace(source, "alpha", "fresh", "ts"); + + assert.equal(result.closed, true); + assert.ok(result.content.includes("const tail = 3;"), "content after the region must survive"); + assert.ok(result.content.includes("} // #endregion alpha"), "the closing line must be kept verbatim"); + }); + + it("is not closed by an end marker naming a different region", () => { + const source = [ + "const keep = 1;", + "// #region alpha", + "const old = 1;", + "// #endregion beta", + "export function important() { return 42; }", + "", + ].join("\n"); + + const result = replace(source, "alpha", "fresh", "ts"); + + assert.equal(result.closed, false, "#endregion beta must not close #region alpha"); + assert.equal(result.content, source, "source must be returned byte-identical"); + }); + + it("refuses a duplicated region name rather than doubling both bodies", () => { + const source = "// #region a\none\n// #endregion\n// #region a\ntwo\n// #endregion\n"; + + const result = spliceRegions(source, new Map([[ "a", { code: "X", lang: "ts" }]])); + + assert.equal(result.ok, false); + assert.deepEqual(result.duplicated, [ "a" ]); + assert.equal(result.content, source, "an ambiguous target must not be written"); + }); + + it("refuses a region nested inside one of the same name", () => { + const source = "// #region a\nouter\n// #region a\ninner\n// #endregion a\n// #endregion a\n"; + + const result = spliceRegions(source, new Map([[ "a", { code: "X", lang: "ts" }]])); + + assert.equal(result.ok, false); + assert.deepEqual(result.duplicated, [ "a" ]); + assert.equal(result.content, source, "ambiguous nesting must not be written"); + }); + + it("refuses to close a region while a region opened inside it is still open", () => { + const source = [ + "// #region outer", + "const a = 1;", + "// #region inner", + "const b = 2;", + "// #endregion outer", + "const tail = 3;", + "", + ].join("\n"); + + const result = replace(source, "outer", "NEW", "ts"); + + assert.equal(result.closed, false, "outer must not close while inner is open"); + assert.equal(result.content, source, "malformed nesting must not be written"); + }); + + it("replaces a nested region wholesale without orphaning inner markers", () => { + const source = [ + "// #region outer", + "const a = 1;", + "// #region inner", + "const b = 2;", + "// #endregion inner", + "const c = 3;", + "// #endregion outer", + "", + ].join("\n"); + + const result = replace(source, "outer", "NEW", "ts"); + + assert.equal(result.closed, true); + assert.equal(result.content, "// #region outer\nNEW\n// #endregion outer\n"); + assert.ok(!result.content.includes("#endregion inner"), "inner marker must not be orphaned"); + assert.ok(!result.content.includes("const c = 3;"), "old body must not leak past the region"); + }); + + it("splices hash-comment regions in shell languages", () => { + const source = "#!/bin/zsh\n# #region init\nold\n# #endregion init\n"; + + const result = replace(source, "init", "new", "zsh"); + + assert.equal(result.found, true, "a zsh marker must be recognised, not appended to"); + assert.ok(result.content.includes("# #region init\nnew\n")); + }); + + it("re-indents the inserted body to its start marker", () => { + const source = "def f():\n # #region body\n return 1\n # #endregion body\n"; + + const result = replace(source, "body", "return 2", "python"); + + assert.equal(result.content, "def f():\n # #region body\n return 2\n # #endregion body\n"); + }); + + it("keeps a CRLF file free of mixed line endings", () => { + const source = "const keep = 1;\r\n// #region a\r\nold\r\n// #endregion a\r\nconst tail = 2;\r\n"; + + const result = replace(source, "a", "fresh", "ts"); + + assert.equal(result.content, "const keep = 1;\r\n// #region a\r\nfresh\r\n// #endregion a\r\nconst tail = 2;\r\n"); + assert.ok(!/[^\r]\n/.test(result.content), "no lone LF may be introduced"); + }); +}); + +describe("region.spliceRegions", () => { + it("splices every region in one pass and reports the ones it could not find", () => { + const source = [ + "head", + "// #region a", + "old a", + "// #endregion a", + "middle", + "// #region b", + "old b", + "// #endregion b", + "tail", + "", + ].join("\n"); + + const result = spliceRegions(source, new Map([ + [ "a", { code: "new a", lang: "ts" }], + [ "b", { code: "new b", lang: "ts" }], + [ "c", { code: "new c", lang: "ts" }], + ])); + + assert.equal(result.ok, true); + assert.deepEqual(result.spliced, [ "a", "b" ]); + assert.deepEqual(result.unmatched, [ "c" ]); + assert.ok(result.content.includes("new a") && result.content.includes("new b")); + assert.ok(result.content.includes("head") && result.content.includes("middle") && result.content.includes("tail")); + assert.ok(!result.content.includes("old a") && !result.content.includes("old b")); + }); + + it("locates each region with its own block's comment syntax", () => { + const source = [ + "# #region a", + "old a", + "# #endregion a", + "// #region b", + "old b", + "// #endregion b", + "", + ].join("\n"); + + const result = spliceRegions(source, new Map([ + [ "a", { code: "new a", lang: "python" }], + [ "b", { code: "new b", lang: "ts" }], + ])); + + assert.equal(result.ok, true); + assert.deepEqual(result.spliced, [ "a", "b" ]); + assert.ok(result.content.includes("new a") && result.content.includes("new b")); + }); + + it("refuses when two requested regions are nested in each other", () => { + const source = [ + "// #region outer", + "a", + "// #region inner", + "b", + "// #endregion inner", + "c", + "// #endregion outer", + "", + ].join("\n"); + + const result = spliceRegions(source, new Map([ + [ "outer", { code: "NEW OUTER", lang: "ts" }], + [ "inner", { code: "NEW INNER", lang: "ts" }], + ])); + + assert.equal(result.ok, false, "nested requested regions have no well-defined result"); + assert.deepEqual([ ...result.overlapping ].sort(), [ "inner", "outer" ]); + assert.deepEqual(result.unmatched, [], "both regions were located, neither is missing"); + assert.equal(result.content, source); + }); + + it("preserves a target that has no final newline", () => { + const source = "head\n// #region a\nold\n// #endregion a"; + + const result = spliceRegions(source, new Map([[ "a", { code: "new", lang: "ts" }]])); + + assert.equal(result.ok, true); + assert.equal(result.content, "head\n// #region a\nnew\n// #endregion a"); + }); + + it("empties a region without collapsing its markers", () => { + const source = "head\n// #region a\nold\nmore old\n// #endregion a\ntail\n"; + + const result = spliceRegions(source, new Map([[ "a", { code: "", lang: "ts" }]])); + + assert.equal(result.ok, true); + assert.equal(result.content, "head\n// #region a\n// #endregion a\ntail\n"); + }); + + it("does not match a marker written in another language's syntax", () => { + const source = "-- #region a\nold\n-- #endregion a\n"; + + const result = spliceRegions(source, new Map([[ "a", { code: "new", lang: "ts" }]])); + + assert.deepEqual(result.unmatched, [ "a" ], "a SQL marker must not open a TypeScript region"); + assert.equal(result.content, source); + }); +}); + +describe("region markers", () => { + it("writes markers in the target language's own comment syntax", () => { + assert.equal(wrapRegion("zsh", "q", "echo hi"), "# #region q\necho hi\n# #endregion q\n"); + assert.equal(wrapRegion("sql", "q", "SELECT 1"), "-- #region q\nSELECT 1\n-- #endregion q\n"); + assert.equal(wrapRegion("css", "q", "a{}"), "/* #region q */\na{}\n/* #endregion q */\n"); + assert.equal(wrapRegion("html", "q", "

"), "\n

\n\n"); + assert.deepEqual(getCommentStyle("nosuchlang"), getCommentStyle("ts"), "unknown languages fall back to the C family"); + }); + + it("rejects a region name that would close the comment early", () => { + assert.throws(() => wrapRegion("ts", "evil */ code", "x"), /Invalid region name/); + assert.throws(() => wrapRegion("html", "evil --> code", "x"), /Invalid region name/); + }); +}); diff --git a/packages/mdcode/src/region.ts b/packages/mdcode/src/region.ts index 3055a6c..a4126a6 100644 --- a/packages/mdcode/src/region.ts +++ b/packages/mdcode/src/region.ts @@ -11,6 +11,8 @@ export type RegionReadResult = { export type RegionReplaceResult = { content: string; found: boolean; + /** False when the start marker was found but never closed. `content` is then the untouched source. */ + closed: boolean; }; export type RegionOutlineResult = { @@ -18,167 +20,490 @@ export type RegionOutlineResult = { hasRegions: boolean; }; +/** One region's replacement body, plus the language whose markers delimit it. */ +export type RegionEdit = { + code: string; + lang?: string; +}; + +/** + * Outcome of splicing one or more regions in a single pass. + * + * `ok` is false when the source cannot be spliced safely — an unclosed region + * would truncate the file, a duplicated name is ambiguous, and overlapping + * regions have no well-defined result. In all three cases `content` is the + * original source, unchanged. + */ +export type RegionSpliceResult = { + content: string; + ok: boolean; + /** Regions whose body was replaced. */ + spliced: Array; + /** Requested regions with no start marker in the source. */ + unmatched: Array; + /** Regions whose start marker was never closed. */ + unclosed: Array; + /** Regions whose start marker appears more than once. */ + duplicated: Array; + /** Regions whose extent overlaps another region's. */ + overlapping: Array; +}; + +/** A comment style: the opening prefix, plus the closer a block comment needs. */ +export type CommentStyle = { + open: string; + close: string; +}; + +/** Non-empty list; the first entry is canonical for writing new markers. */ +type CommentStyles = readonly [CommentStyle, ...Array]; + +const LINE_SLASH: CommentStyle = { open: "//", close: "" }; +const BLOCK_SLASH: CommentStyle = { open: "/*", close: " */" }; +const HASH: CommentStyle = { open: "#", close: "" }; +const DASH: CommentStyle = { open: "--", close: "" }; +const SEMI: CommentStyle = { open: ";", close: "" }; +const HTML: CommentStyle = { open: "" }; + +const C_FAMILY: CommentStyles = [ LINE_SLASH, BLOCK_SLASH ]; +const HASH_ONLY: CommentStyles = [ HASH ]; +const DASH_ONLY: CommentStyles = [ DASH ]; +const SEMI_ONLY: CommentStyles = [ SEMI ]; +const HTML_ONLY: CommentStyles = [ HTML ]; +const BLOCK_ONLY: CommentStyles = [ BLOCK_SLASH ]; + +/** + * Comment styles per language. Kept in step with `getExtensionForLang` in + * commands/extract.ts: a language that can be extracted must also be able to + * carry a region marker in its own syntax. + */ +const LANG_COMMENT_STYLES: Readonly> = { + // C family + js: C_FAMILY, + javascript: C_FAMILY, + jsx: C_FAMILY, + mjs: C_FAMILY, + cjs: C_FAMILY, + ts: C_FAMILY, + typescript: C_FAMILY, + tsx: C_FAMILY, + java: C_FAMILY, + c: C_FAMILY, + h: C_FAMILY, + cpp: C_FAMILY, + "c++": C_FAMILY, + cs: C_FAMILY, + "c#": C_FAMILY, + csharp: C_FAMILY, + go: C_FAMILY, + rust: C_FAMILY, + rs: C_FAMILY, + swift: C_FAMILY, + kotlin: C_FAMILY, + kt: C_FAMILY, + scala: C_FAMILY, + dart: C_FAMILY, + php: C_FAMILY, + json: C_FAMILY, + jsonc: C_FAMILY, + + // Hash comments + py: HASH_ONLY, + python: HASH_ONLY, + rb: HASH_ONLY, + ruby: HASH_ONLY, + sh: HASH_ONLY, + bash: HASH_ONLY, + zsh: HASH_ONLY, + ksh: HASH_ONLY, + fish: HASH_ONLY, + shell: HASH_ONLY, + perl: HASH_ONLY, + pl: HASH_ONLY, + r: HASH_ONLY, + yaml: HASH_ONLY, + yml: HASH_ONLY, + toml: HASH_ONLY, + ini: HASH_ONLY, + conf: HASH_ONLY, + dockerfile: HASH_ONLY, + makefile: HASH_ONLY, + make: HASH_ONLY, + elixir: HASH_ONLY, + ex: HASH_ONLY, + exs: HASH_ONLY, + txt: HASH_ONLY, + text: HASH_ONLY, + + // Double-dash comments + sql: DASH_ONLY, + lua: DASH_ONLY, + hs: DASH_ONLY, + haskell: DASH_ONLY, + elm: DASH_ONLY, + ada: DASH_ONLY, + + // Semicolon comments + lisp: SEMI_ONLY, + clj: SEMI_ONLY, + clojure: SEMI_ONLY, + scm: SEMI_ONLY, + asm: SEMI_ONLY, + + // Block-comment-only + css: BLOCK_ONLY, + scss: BLOCK_ONLY, + sass: BLOCK_ONLY, + less: BLOCK_ONLY, + + // Markup + html: HTML_ONLY, + xml: HTML_ONLY, + svg: HTML_ONLY, + vue: HTML_ONLY, + svelte: HTML_ONLY, + md: HTML_ONLY, + markdown: HTML_ONLY, +}; + +/** Region names are interpolated into comment markers, so keep them inert. */ +const VALID_REGION_NAME = /^[\w.:-]+$/; + +/** + * Get the comment style(s) recognised for a given language. + * The first entry is canonical — used when writing new markers; the rest are + * additional prefixes accepted when matching existing markers. + */ +export function getCommentStyle(lang: string): CommentStyles { + return LANG_COMMENT_STYLES[lang.toLowerCase()] ?? C_FAMILY; +} + +/** + * True when `name` is safe to interpolate into a comment marker. + * Rejects anything that could terminate a block or markup comment early, or + * span lines. + */ +export function isValidRegionName(name: string): boolean { + return VALID_REGION_NAME.test(name); +} + +type RawLine = { + /** Line text without its end-of-line sequence. */ + text: string; + /** The line's own end-of-line sequence, or "" for a final line without one. */ + eol: string; +}; + +type Marker = { + kind: "region" | "endregion"; + name?: string; +}; + +/** + * Split into lines that each retain their own line ending, so a mixed or CRLF + * file round-trips byte-for-byte. + */ +function splitLines(source: string): Array { + return source.split(/(?<=\n)/).map(raw => { + const eol = /\r?\n$/.exec(raw)?.[0] ?? ""; + + return { text: eol ? raw.slice(0, -eol.length) : raw, eol }; + }); +} + +/** + * Build the marker pattern for one language. + * + * A marker is accepted anywhere on the line as long as it terminates the line, + * so both ` // #region x` and `} // #endregion x` match, as do JSDoc-style + * openers. The name charset matches VALID_REGION_NAME, so a block-comment + * terminator is never captured as the name, and names are compared exactly, so + * `join` never matches `join-sql`. + */ +function markerScanner(lang?: string): RegExp { + const opens = getCommentStyle(lang ?? "") + .map(style => escapeRegex(style.open)) + .join("|"); + + return new RegExp(`(?:${opens})[\\s*]*#(region|endregion)(?:\\s+([\\w.:-]+))?\\s*(?:\\*/|-->)?\\s*$`); +} + +function markerAt(text: string, scanner: RegExp): Marker | undefined { + const match = scanner.exec(text); + + return match ? { kind: match[1] as "region" | "endregion", name: match[2] } : undefined; +} + /** * Read a specific region from source code. - * Joins all occurrences of the same-named region. - * Pass `lang` to use language-specific comment styles; defaults to // and /* *\/. + * Joins all occurrences of the same-named region; nested regions are part of + * the enclosing region's content and are returned verbatim. + * + * An unterminated region has no well-defined content, so it reads as not found + * rather than as everything up to end-of-file. */ export function read(source: string, regionName: string, lang?: string): RegionReadResult { - const lines = source.split("\n"); + const scanner = markerScanner(lang); + const content: Array = []; + const nesting: Array = []; let inRegion = false; - const regionContent: Array = []; let found = false; - let startPattern: RegExp; - let endPattern: RegExp; + for (const line of splitLines(source)) { + const marker = markerAt(line.text, scanner); - if (lang) { - const styles = getCommentStyle(lang).map(escapeRegex) - .join("|"); - startPattern = new RegExp(`^\\s*(?:${styles})\\s*#region\\s+${escapeRegex(regionName)}(?:\\s|$)`); - endPattern = new RegExp(`^\\s*(?:${styles})\\s*#endregion`); - } - else { - startPattern = new RegExp(`^\\s*(?://|/\\*)\\s*#region\\s+${escapeRegex(regionName)}(?:\\s|\\*/|$)`); - endPattern = /^\s*(?:\/\/|\/\*)\s*#endregion(?:\s|\*\/|$)/; - } - - for (const line of lines) { if (!inRegion) { - if (startPattern.test(line)) { + if (marker?.kind === "region" && marker.name === regionName) { inRegion = true; + nesting.length = 0; found = true; } + continue; } - else { - if (endPattern.test(line)) { - inRegion = false; - } - else { - regionContent.push(line); - } + + if (marker?.kind === "endregion" && nesting.length === 0 && (marker.name === undefined || marker.name === regionName)) { + inRegion = false; + continue; + } + + if (marker?.kind === "region") { + nesting.push(marker.name); } + else if (marker?.kind === "endregion" && (marker.name === undefined || marker.name === nesting.at(-1))) { + nesting.pop(); + } + + content.push(line.text); } - return { - content: regionContent.join("\n"), - found, - }; + if (inRegion) { + return { content: "", found: false }; + } + + return { content: content.join("\n"), found }; } /** - * Generate an outline showing only region markers - * Removes all content between #region and #endregion markers + * Generate an outline showing only region markers. + * + * Every marker line is kept — including nested ones — and only non-marker lines + * inside a region are removed, so the marker structure of the file survives + * whatever depth it is written at. */ -export function outline(source: string): RegionOutlineResult { - const lines = source.split("\n"); +export function outline(source: string, lang?: string): RegionOutlineResult { + const scanner = markerScanner(lang); const result: Array = []; - let inRegion = false; + let depth = 0; let hasRegions = false; - // Match both // #region and /* #region */ - const startPattern = /^\s*(?:\/\/|\/\*)\s*#region\b/; - const endPattern = /^\s*(?:\/\/|\/\*)\s*#endregion\b/; + for (const line of splitLines(source)) { + const marker = markerAt(line.text, scanner); - for (const line of lines) { - if (!inRegion) { - result.push(line); - if (startPattern.test(line)) { - inRegion = true; - hasRegions = true; - } + if (marker?.kind === "region") { + depth++; + hasRegions = true; } - else { - if (endPattern.test(line)) { - result.push(line); - inRegion = false; - } - // Skip all lines inside regions + else if (marker?.kind === "endregion") { + depth = Math.max(0, depth - 1); + } + else if (depth > 0) { + continue; } + + result.push(line.text + line.eol); } - return { - content: result.join("\n"), - hasRegions, - }; + return { content: result.join(""), hasRegions }; } +/** The extent of one region in the source, as line indices of its markers. */ +type RegionSpan = { + name: string; + start: number; + end: number; + code: string; +}; + /** - * Replace content within a specific region - * Preserves the region markers and surrounding code + * Replace the bodies of one or more regions. + * + * Each region is located with its own block's comment syntax, so one language's + * marker never opens a region belonging to another. Extents are discovered once + * per distinct language and then applied in a single pass, so cost scales with + * the number of languages involved, not the number of regions. + * + * The inserted body is re-indented to its start marker and takes that marker's + * line ending, so an indented or CRLF file stays consistent. + * + * Refuses (returns `ok: false` and the original source) when a region's start + * marker is never closed — splicing would otherwise drop everything from the + * marker to end-of-file — when a name appears more than once, or when two + * regions overlap. */ -export function replace(source: string, regionName: string, newContent: string): RegionReplaceResult { - const lines = source.split("\n"); - const result: Array = []; - let inRegion = false; - let found = false; +export function spliceRegions(source: string, edits: ReadonlyMap): RegionSpliceResult { + const lines = splitLines(source); + const namesByLang = new Map>(); - // Match both // #region name and /* #region name */ - const startPattern = new RegExp(`^\\s*(?://|/\\*)\\s*#region\\s+${escapeRegex(regionName)}(?:\\s|\\*/|$)`); - const endPattern = /^\s*(?:\/\/|\/\*)\s*#endregion(?:\s|\*\/|$)/; + for (const [ name, edit ] of edits) { + const key = edit.lang ?? ""; + const existing = namesByLang.get(key); - for (const line of lines) { - if (!inRegion) { - result.push(line); - if (startPattern.test(line)) { - inRegion = true; - found = true; - // Insert new content after the region start marker - // Remove trailing newline from newContent if it exists, since we'll add it via join - const contentToInsert = newContent.endsWith("\n") ? newContent.slice(0, -1) : newContent; - if (contentToInsert) { - result.push(contentToInsert); - } - } + if (existing) { + existing.push(name); } else { - if (endPattern.test(line)) { - result.push(line); - inRegion = false; + namesByLang.set(key, [ name ]); + } + } + + const spans: Array = []; + const started = new Set(); + const duplicated = new Set(); + const unclosed = new Set(); + const overlapping = new Set(); + + for (const [ lang, names ] of namesByLang) { + const scanner = markerScanner(lang); + const wanted = new Set(names); + const nesting: Array = []; + let open: { name: string; start: number; } | undefined; + + for (const [ index, line ] of lines.entries()) { + const marker = markerAt(line.text, scanner); + if (marker === undefined) continue; + + if (open === undefined) { + if (marker.kind === "region" && marker.name !== undefined && wanted.has(marker.name)) { + if (started.has(marker.name)) { + duplicated.add(marker.name); + } + started.add(marker.name); + open = { name: marker.name, start: index }; + nesting.length = 0; + } + continue; + } + + // Only the region's own terminator closes it, and only once everything + // opened inside it has closed. A closer naming neither the innermost open + // region nor nothing leaves the nesting malformed, so the region never + // closes and the whole splice is refused. + if (marker.kind === "endregion" && nesting.length === 0 && (marker.name === undefined || marker.name === open.name)) { + spans.push({ name: open.name, start: open.start, end: index, code: edits.get(open.name)!.code }); + open = undefined; + continue; + } + + if (marker.kind === "region") { + // A region nested inside one of the same name has no unambiguous + // terminator; refuse rather than guess which marker closes which. + if (marker.name === open.name) { + duplicated.add(open.name); + } + // Two requested regions cannot nest: splicing one would discard the + // other's markers along with the body being replaced. The inner one was + // still located, so it counts as started and is never "unmatched". + else if (marker.name !== undefined && wanted.has(marker.name)) { + started.add(marker.name); + overlapping.add(open.name); + overlapping.add(marker.name); + } + nesting.push(marker.name); } - // Skip old content inside region + else if (nesting.length > 0 && (marker.name === undefined || marker.name === nesting.at(-1))) { + nesting.pop(); + } + } + + if (open !== undefined) { + unclosed.add(open.name); } } - return { - content: result.join("\n"), - found, + const ordered = [ ...spans ].sort((a, b) => a.start - b.start); + + for (const [ index, span ] of ordered.entries()) { + const previous = index === 0 ? undefined : ordered[index - 1]; + + if (previous !== undefined && span.start <= previous.end) { + overlapping.add(previous.name); + overlapping.add(span.name); + } + } + + const ok = unclosed.size === 0 && duplicated.size === 0 && overlapping.size === 0; + const result: RegionSpliceResult = { + content: source, + ok, + spliced: ok ? ordered.map(span => span.name) : [], + unmatched: [ ...edits.keys() ].filter(name => !started.has(name)), + unclosed: [ ...unclosed ], + duplicated: [ ...duplicated ], + overlapping: [ ...overlapping ], }; + + if (!ok) { + return result; + } + + const spanByStart = new Map(ordered.map(span => [ span.start, span ])); + const out: Array = []; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]!; + out.push(line.text + line.eol); + + const span = spanByStart.get(index); + if (span === undefined) continue; + + const eol = line.eol || "\n"; + const indent = /^[ \t]*/.exec(line.text)?.[0] ?? ""; + + for (const bodyLine of span.code.replace(/\r?\n$/, "").split(/\r?\n/)) { + if (bodyLine === "" && span.code === "") continue; + out.push((bodyLine === "" ? "" : indent + bodyLine) + eol); + } + + // Skip the old body; the next iteration emits the closing marker. + index = span.end - 1; + } + + return { ...result, content: out.join("") }; } /** - * Get comment prefix(es) for a given language + * Replace content within a single region, preserving the markers and + * surrounding code. Thin wrapper over `spliceRegions`. */ -export function getCommentStyle(lang: string): Array { - const styles: Record> = { - js: [ "//" ], - javascript: [ "//" ], - ts: [ "//" ], - typescript: [ "//" ], - java: [ "//" ], - c: [ "//" ], - cpp: [ "//" ], - "c++": [ "//" ], - cs: [ "//" ], - "c#": [ "//" ], - go: [ "//" ], - rust: [ "//" ], - swift: [ "//" ], - kotlin: [ "//" ], - php: [ "//" ], - py: [ "#" ], - python: [ "#" ], - rb: [ "#" ], - ruby: [ "#" ], - sh: [ "#" ], - bash: [ "#" ], - yaml: [ "#" ], - yml: [ "#" ], - html: [ "