diff --git a/.gitignore b/.gitignore index 979ccd7d..e3e6c62e 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ +# Comparative benchmark harness output (tests/benchmarks/SharpCoreDB.Benchmarks.Comparative) results/ # .NET Core diff --git a/ROADMAP.md b/ROADMAP.md index 2e8e5e56..404a34fb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -131,9 +131,12 @@ >> **Why:** v2.0 closed the read/insert gap; single-row UPDATE/DELETE is still ~5–7x behind SQLite >> because row-store writes are append-on-update instead of in-place. -- In-place record updates (avoid append-on-update) for row stores -- Fixed-width record layout for hot tables (SQLite-style C record format) -- Eliminate read-modify-write in `UpdateMultiple` +- ✅ **In-place record updates for columnar/append-only (#6)** — fixed-width/unchanged-length + records overwrite their slot; no file growth +- ✅ **Single-pass SQL DELETE/UPDATE (#7/#8)** — no more double materialization for RETURNING / + `CHANGES()`; PK fast path in `Delete`/`DeleteMultiple`/`UpdateMultiple` +- ⬜ Fixed-width record layout for hot tables (SQLite-style C record format) +- ⬜ Storage-level DELETE reuse (free-slot reuse / compaction on PageBased) - Track in [`docs/performance/V2_PERFORMANCE_PLAN.md`](docs/performance/V2_PERFORMANCE_PLAN.md) --- diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 11071678..cbf31f6e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,79 +9,166 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance -- **Dedicated SQL batch-INSERT fast path (WP14)** — `ExecuteBatchSQL` INSERTs no longer build a +- **Dedicated SQL batch-INSERT fast path (WP14)** ÔÇö `ExecuteBatchSQL` INSERTs no longer build a per-row `Dictionary`; VALUES clauses are parsed directly into column-ordered `object[]` rows (`PreparedInsertStatement.ParseValuesToArray`) and inserted via the new `Table.InsertBatch(object[][], columnOrder)` path with full dict-path parity (defaults, AUTO, explicit NULL, NOT NULL, PK, hash/B-tree indexes). SQL INSERT throughput measured **+80%** - (54.5K/s → 98.2K/s in the comparative benchmark), closing the INSERT gap vs SQLite from ~1.9× to - ~1.5×. Batch UPDATE also reuses the WP11 in-place field-overwrite fast path (runtime offsets now + (54.5K/s ÔåÆ 98.2K/s in the comparative benchmark), closing the INSERT gap vs SQLite from ~1.9├ù to + ~1.5├ù. Batch UPDATE also reuses the WP11 in-place field-overwrite fast path (runtime offsets now resolve fixed-size fields after variable-length columns; monitored via `Table.TotalInPlacePatches`). -- **AVX-512 validation on real hardware (2026-09-01)** — 6-run benchmark on an AVX-512 machine - confirmed the adaptive SIMD tier (AVX-512 **2–26× over scalar**, up to **2.7× over AVX2** for - `EuclidSq`/`Normalize`, dims 64–1024) and the CRUD profile (beats LiteDB on every operation; INSERT - at 0.69–0.85× of SQLite). Full report: +- **AVX-512 validation on real hardware (2026-09-01)** ÔÇö 6-run benchmark on an AVX-512 machine + confirmed the adaptive SIMD tier (AVX-512 **2ÔÇô26├ù over scalar**, up to **2.7├ù over AVX2** for + `EuclidSq`/`Normalize`, dims 64ÔÇô1024) and the CRUD profile (beats LiteDB on every operation; INSERT + at 0.69ÔÇô0.85├ù of SQLite). Full report: `docs/benchmarks/AVX512_2026-09-01.md` (+ raw per-run `.md`/`.json` in `docs/benchmarks/avx512-2026-09-01/`). ## [2.0.0.1] - 2026-09-01 ### Fixed -- **Single-file data corruption under concurrent writes (critical)** — the WAL manager wrote to +- **Single-file data corruption under concurrent writes (critical)** ÔÇö the WAL manager wrote to the shared file stream with a bare `Position` + `WriteAsync`, while the background write-behind worker wrote data pages under a lock. A concurrent `Position` mutation could land WAL bytes on a data page, so a table's data block could read back as WAL/registry bytes instead of JSON after a reopen (sporadic `JsonException '0x02'` / "Expected 100 rows, got 0"). All `FileStream.Position` use is now serialized through `SingleFileStorageProvider.WriteAt` (header, WAL, delta writes, reads, defrag). Regression covered by `VacuumStressTests` (failed ~50% before, stable after). -- **4-part patch versioning** — this patch ships as `2.0.0.1` (NuGet shows `2.0.0.1`). +- **4-part patch versioning** ÔÇö this patch ships as `2.0.0.1` (NuGet shows `2.0.0.1`). ## [2.0.0.0] - 2026-09-01 ### Release highlights -- **Performance-first engine** — point reads **beat SQLite** on the default engine, batch INSERTs - beat SQLite on PageBased (**194–206K vs 109K ops/s**), and the UPDATE/DELETE gap vs SQLite narrowed - from ~5–7× to ~1–4× (in-place field patches + unified delete core). -- **Single-file storage format v2** — dynamic/growable metadata layout (Block Registry, FSM, Table - Directory) with **automatic crash-safe v1 → v2 migration on open** (original preserved as +- **Performance-first engine** ÔÇö point reads **beat SQLite** on the default engine, batch INSERTs + beat SQLite on PageBased (**194ÔÇô206K vs 109K ops/s**), and the UPDATE/DELETE gap vs SQLite narrowed + from ~5ÔÇô7├ù to ~1ÔÇô4├ù (in-place field patches + unified delete core). +- **Single-file storage format v2** ÔÇö dynamic/growable metadata layout (Block Registry, FSM, Table + Directory) with **automatic crash-safe v1 ÔåÆ v2 migration on open** (original preserved as `.backup`). -- **Block-level compression** — Brotli/GZip/Zstd with configurable presets +- **Block-level compression** ÔÇö Brotli/GZip/Zstd with configurable presets (`BlockCompressionLevel`, `MetadataCompressionLevel`, `CompressionThreshold`). - **Envelope encryption + full at-rest metadata encryption** (`EncryptionPassword`, per-file DEK, key/password rotation) and **configurable metadata sizing** (`FsmSizePages`, `BlockRegistrySizePages`, `TableDirectorySizePages`). -- **4-part versioning** — all packages now use `n.n.n.n` (this release: `2.0.0.0`). -- **Full change/benchmark report** — see +- **4-part versioning** ÔÇö all packages now use `n.n.n.n` (this release: `2.0.0.0`). +- **Full change/benchmark report** ÔÇö see [`docs/2.0.0.0_WHAT_CHANGED.md`](2.0.0.0_WHAT_CHANGED.md): everything that changed vs the 1.9 line, plus the SharpCoreDB vs SQLite vs LiteDB benchmark tables and graphs. -### SingleFile storage — critical compression read-path fixes + configurable presets (PR #352) +### SingleFile storage ÔÇö critical compression read-path fixes + configurable presets (PR #352) -- **Fix: zero-copy read paths returning compressed bytes** — `GetReadStream()` and `GetReadSpan()` +- **Fix: zero-copy read paths returning compressed bytes** ÔÇö `GetReadStream()` and `GetReadSpan()` served the raw Brotli/GZip bytes when encryption was disabled, causing `JsonException` on database reopen. Both methods now check the block's `BlockFlags.Compressed` bit and fall back to `ReadBlockAsync` so compressed blocks are always decompressed. Affected databases created with `BlockCompression != None` and `EnableEncryption = false` in v1.9.8. -- **Fix: stale `Compressed` flag on block overwrite** — `WriteBlockAsync` preserved old flags and +- **Fix: stale `Compressed` flag on block overwrite** ÔÇö `WriteBlockAsync` preserved old flags and never updated the `Compressed` bit based on the current write, so a block that grew past the compression threshold (256 B default) could be stored compressed but marked uncompressed. The flag is now cleared and re-set on every write while preserving all other flags. -- **Configurable compression presets** — new `DatabaseOptions.MetadataCompressionLevel` +- **Configurable compression presets** ÔÇö new `DatabaseOptions.MetadataCompressionLevel` (default `Fastest`) and `BlockCompressionLevel` (default `Optimal`) map to the BCL `CompressionLevel` via the new `SharpCoreDB.Compression.OptionalCompressionLevel` enum; `BlockBrotliCompressionLevel` remains as an obsolete alias. `VacuumMode.Full` preserves the block compression level when it creates the temporary file. -- **Zstd support** — `BlockCompressionMode.Zstd` (`.NET 11+`, `ZstandardStream`) with a +- **Zstd support** ÔÇö `BlockCompressionMode.Zstd` (`.NET 11+`, `ZstandardStream`) with a `PlatformNotSupportedException` fallback on older runtimes. - **Regression tests:** `CompressionLevelTests` (31 tests) cover preset defaults, roundtrips, size ordering across levels, metadata roundtrips, `GetReadStream`/`GetReadSpan` decompression without encryption, and the multi-write stale-flag scenario. + +## [2.1.0-preview] - 2026-08-31 + +### Performance +- **Single-pass SQL DELETE/UPDATE (Issue #7/#8)** ÔÇö the SQL paths no longer materialize matching + rows twice: + - `ITable.DeleteAffectedRows(where)` deletes AND returns the affected rows; `ExecuteDelete` uses + it for RETURNING + `CHANGES()` from a single pass (`Table`/`SingleFileTable` override the + default; third-party `ITable` implementers keep the two-pass fallback). + - `ITable.UpdateAffectedCount(where, updates)` applies the update and returns the affected count; + `ExecuteUpdate` no longer runs a full `Select().Count` for change-tracking. +- **PK fast path extended to batch DML** ÔÇö simple `pk = value` WHERE clauses resolve via the + primary-key B-tree directly (single search + one read) in `Delete`/`DeleteMultiple`/ + `UpdateMultiple` instead of full-row materialization + per-row PK re-search. +- **Field-level in-place patch on the columnar UPDATE path (fixed-width layout step)** ÔÇö when the + row's storage position is known (PK B-tree / hash index), only the updated fields are patched at + their **actual** record offsets (`ComputeActualColumnOffsets` + `TryOverwriteFieldsInPlaceActual`) + instead of deserialize ÔåÆ mutate ÔåÆ re-serialize of the whole row. A fixed-size field keeps the + record length unchanged ÔåÆ the write is an in-place overwrite (no file growth), even for columns + that sit after variable-length TEXT columns. `UpdateAffectedCount`/`UpdateMultiple` now resolve + rows as (position, row) pairs; variable-width fields that change size still fall back to append. +- **Stale-index regression fix** ÔÇö WHERE-based UPDATE/DELETE entry points load all registered hash + indexes up front (`EnsureAllRegisteredIndexesLoaded`), so append updates / logical deletes remove + the stale record from every index (an unloaded index would otherwise be rebuilt from the data + file including the stale record, resurrecting the pre-update row). +- **Regression tests:** `DmlSinglePassTests` (9 cases) + `FixedWidthPatchTests` (5 cases) ÔÇö + affected counts, RETURNING pre-delete rows, range/non-indexed WHERE fallbacks, batch PK + deletes/updates, in-place patch no-growth (after variable columns / by PK), variable-growth + append fallback, compound WHERE. Full suite green: **1,649 tests, 0 failures**. +- **Single-file `.scdb` (A-track):** + - **PK hash index (A1)** ÔÇö `FindByPrimaryKey` / `UpdateByPrimaryKey` / `DeleteByPrimaryKey` and + `SELECT ÔǪ WHERE pk = value` resolve in O(1) instead of an O(N) cache scan (index maintained on + all mutations, rebuilt on reopen/rollback; numeric literals normalized). + - **In-place block overwrite (A2)** ÔÇö pinned: a same-length update does not grow the `.scdb` + (`WriteBlockAsync` reuses the table block offset when the JSON fits). +- **Out-of-line overflow (B1, opt-in):** `DatabaseConfig.FixedWidthRecordLayout` ÔÇö fixed-width + records with constant size per schema; TEXT/BLOB values in a per-table overflow arena (`.ovf`), + referenced by a 4-byte offset in the record. Every UPDATE (fixed **or** variable column) is an + in-place overwrite (`.dat` does not grow). Includes `OverflowArena` (append + cache + + copy-on-compact), `FixedWidthRecordLayout`, and fixed-width serialize/deserialize/in-place-patch + wired into the Table dispatcher, PK index rebuild, full-scan guards and StructRow fallback. + Flag persisted in table metadata, restored from config on reopen. +- **Overflow arena GC (B3)** ÔÇö `CompactStorage` now compacts the overflow arena together with the + data file: live arena offsets are collected from the current records, the `.ovf` is rewritten + (copy-on-compact), and the active records' variable slots are re-pointed in place. Dead arena + blocks from variable updates / deletes are reclaimed. +- **Constant-offset read-path wins (B4)** ÔÇö early-WHERE re-enabled for fixed-width tables using the + constant slot offsets of `FixedWidthRecordLayout`: numeric predicates read the column directly at + its slot offset (also when a variable-length column precedes it), string predicates compare the + arena payload byte-wise against the pre-encoded expected UTF-8, and the StructRow numeric-SIMD + batch filter (`Vector`) now serves fixed-width tables. Also fixed a latent bug where arena + block offset 0 (the first block) was treated as "no block" and dropped by compaction / early-WHERE. +- **1.x ÔåÆ 2.0 record-format migration path (B5)** ÔÇö the fixed-width flag is now persisted per table + in metadata (authoritative on reopen; config no longer overrides the on-disk format). A legacy + (variable-length) database opened with `DatabaseConfig.FixedWidthRecordLayout = true` auto-migrates + its columnar tables, and `IDatabase.MigrateTableToFixedWidth(tableName)` provides on-demand + conversion. A format probe adopts already-fixed-width tables that predate flag persistence and + skips byte-identical fixed-size-only legacy tables. +- **Arena free-list (B6)** ÔÇö freed overflow blocks are tracked per payload length and reused in + place (`OverwriteRecordAt`) when a new value has the exact same length, so same-length + variable-column updates no longer grow the `.ovf` within a session (copy-on-compact still + reclaims the rest). Also fixed a latent B1 leak where the first arena block (offset 0) was never + freed on update. +- **Single-file (.scdb) fixed-width (B6)** ÔÇö the fixed-width out-of-line-overflow model now also + serves single-file tables: with `DatabaseConfig.FixedWidthRecordLayout` the table stores binary + fixed-width records (variable values in a dedicated overflow block) instead of the legacy JSON row + array, so value-only updates keep the data block constant-size. The on-disk format is detected on + reopen (binary blocks are parsed untrimmed), legacy JSON tables migrate via + `MigrateTableToFixedWidth` (or automatically when the config opts in), and the shared + `FixedWidthCodec` keeps directory-mode and single-file record formats in sync. +- **Automatic PageBased ÔåÆ Columnar + fixed-width conversion (B6)** ÔÇö `MigrateToFixedWidth` now + converts page-based tables to Columnar storage in-process (rows re-read via the page engine, + `.pages` files removed, `DataFile`/`StorageMode`/metadata updated) before rewriting the records + as fixed-width, and the database-load auto-migration covers PageBased tables as well. Also fixed + a pre-existing PageBased data-loss bug: single INSERT/UPDATE never flushed the page cache (only + `CommitAsync`/`Flush` did), so reopened tables returned zero rows ÔÇö dirty pages are now flushed + when the table/storage engine is disposed. +- **Cross-session arena free-list (B6)** ÔÇö the directory-mode `OverflowArena` derives its free-list + on load: the fixed-width records in the data file are scanned and every arena block no record + references is freed, so same-length value updates reuse dead blocks across sessions without + persisting the free-list itself (single-file tables already restore it per flush via the + unreferenced-sweep). This closes the last open storage-performance follow-up. +- **Regression tests:** `SingleFilePkIndexTests` (7), `SingleFileWriteTests` (2), + `FixedWidthRecordLayoutTests` (14, incl. cross-session free-list), `FixedWidthMigrationTests` (8), + `SingleFileFixedWidthTests` (5). Full suite green: **1,686 tests, 0 failures**. + + ## [2.0.0-preview.3] - 2026-08-30 ### Added -- **Dynamic metadata layout (format v2, #345 Phase 2)** — the Free Space Map and Block Registry +- **Dynamic metadata layout (format v2, #345 Phase 2)** ÔÇö the Free Space Map and Block Registry are no longer fixed header regions but **growable named blocks**: - the Block Registry is a single growable block rooted at `header.RegistryRootOffset` (`[RegistryChunkHeader][BlockEntry...]`), which relocates (grows) automatically when it @@ -89,38 +176,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - the FSM is a named block (`sys:fsm`) tracked in the registry; its serialized bitmap relocates (grows) automatically when the database outgrows the initial `FsmSizePages` capacity; - `FormatVersion` is bumped to **2** (`FEATURE_DYNAMIC_METADATA`); `BlockRegistrySizePages` - sizes the **initial** registry block (default 4 pages ≈ 170 entries; the registry still + sizes the **initial** registry block (default 4 pages Ôëê 170 entries; the registry still grows on demand beyond that); - - **automatic v1 → v2 migration on open**: legacy files with fixed-offset metadata are rebuilt - via a crash-safe temp-file swap (data blocks are never moved — checksums/ciphertexts stay + - **automatic v1 ÔåÆ v2 migration on open**: legacy files with fixed-offset metadata are rebuilt + via a crash-safe temp-file swap (data blocks are never moved ÔÇö checksums/ciphertexts stay valid) and the original is preserved as `.backup`; - system metadata blocks (`sys:fsm`) are hidden from `EnumerateBlocks()`. -- **Regression tests** — format-v1 → v2 migration round-trip (`LegacyMigrationTests`), +- **Regression tests** ÔÇö format-v1 ÔåÆ v2 migration round-trip (`LegacyMigrationTests`), FSM-block growth + data round-trip, dynamic registry growth (300+ blocks), tampered registry detection at the new dynamic location. ### Added -- **Block-level Brotli/GZip compression for single-file (`.scdb`) storage** (#344) — transparent +- **Block-level Brotli/GZip compression for single-file (`.scdb`) storage** (#344) ÔÇö transparent per-block compression applied before encryption on write and removed after decryption on read. A per-block `Compressed` flag tracks state, so compressed and uncompressed blocks can coexist in one file; defaults to `None` (fully backward compatible). New `DatabaseOptions.BlockCompression` and `CompressionThreshold` options. -- **Configurable SingleFile metadata region sizes** (#345) — the FSM, Block Registry and Table +- **Configurable SingleFile metadata region sizes** (#345) ÔÇö the FSM, Block Registry and Table Directory are no longer hard-coded to 4 pages: `DatabaseOptions.FsmSizePages`, `BlockRegistrySizePages` and `TableDirectorySizePages` size the regions for large databases (>512 MB), and the minimum file extension is now byte-based (~10 MB regardless of `PageSize`). -- **Unicode & large-blob storage regression tests** (#346) — CJK, emoji (incl. ZWJ sequences), RTL +- **Unicode & large-blob storage regression tests** (#346) ÔÇö CJK, emoji (incl. ZWJ sequences), RTL and combining-character roundtrips, plus 16 MB blob block-chaining coverage. -- **Full at-rest encryption for single-file (`.scdb`) databases** — beyond block data (#341), +- **Full at-rest encryption for single-file (`.scdb`) databases** ÔÇö beyond block data (#341), the **block registry, free-space map and WAL are now encrypted too** (`EncryptionMode = 2`), closing the metadata-leakage gap: block/table names, offsets, lengths and allocation patterns are no longer visible in plaintext on disk (header + wrapped-key bundle remain the only plaintext bootstrap). -- **Envelope-encryption key model** — `DatabaseOptions.EncryptionPassword` creates a random +- **Envelope-encryption key model** ÔÇö `DatabaseOptions.EncryptionPassword` creates a random per-file data-encryption-key (DEK) wrapped by a PBKDF2-HMAC-SHA256-derived key (per-file salt, OWASP-2024 iteration default). Raw `EncryptionKey` mode remains supported. -- **Password & key rotation** — +- **Password & key rotation** ÔÇö - `IDatabase.ChangeEncryptionPasswordAsync(newPassword)` re-wraps the same DEK with the new password (O(1), no data rewrite; increments the header `EncryptionKeyId` rotation counter). - `IDatabase.RotateEncryptionKeyAsync(newKey|newPassword)` fully re-keys the database @@ -130,7 +217,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 silently returning an empty schema. ### Fixed -- **Issue #343 — `VacuumAsync(VacuumMode.Full)` crashed with `ObjectDisposedException` under .NET 10 +- **Issue #343 ÔÇö `VacuumAsync(VacuumMode.Full)` crashed with `ObjectDisposedException` under .NET 10 trimming / Native AOT** (same fix set as the v1.9.8 line on `master`): - the full-vacuum stream swap uses direct field assignment (`SwapFileStream`) instead of reflection, and error paths read the file size safely (`GetFileSizeSafely`, `-1` fallback); @@ -144,7 +231,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 publishes with `PublishAot=true` and runs exit 0 including the single-file full-vacuum path. - **Follow-up:** full VACUUM now reloads the block registry / FSM / WAL after the file swap so in-memory offsets match the compacted file (fixes stale-offset writes after vacuum). -- **Issue #344 — compressed single-file (`.scdb`) databases crashed on reopen + SELECT with +- **Issue #344 ÔÇö compressed single-file (`.scdb`) databases crashed on reopen + SELECT with `JsonException: '0x0B' is an invalid start of a value`**: - `WriteBlockAsync` only stamped the per-block `Compressed` flag for brand-new blocks; a table row-cache block that was rewritten while it already existed (auto-flush as the JSON grows past @@ -159,55 +246,55 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.0.0] - 2026-08-28 -### Performance-first release 🚀 +### Performance-first release ­ƒÜÇ -The v2.0 release closes the v1.x benchmark gap (was **16–52x slower than SQLite** on point reads, +The v2.0 release closes the v1.x benchmark gap (was **16ÔÇô52x slower than SQLite** on point reads, updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: | Operation | v2.0 | SQLite | LiteDB | |-----------|-----:|-------:|-------:| -| READ — Direct / StructRow | 70–126K ops/s | 87–97K | 14–16K | -| READ — SQL | 51–59K ops/s | 87–97K | 14–16K | -| INSERT (batch) | 91–133K ops/s | 145–150K | 66–77K | -| UPDATE (batch) | 41–59K ops/s | 241–296K | 10–11K | -| DELETE (batch) | 30–142K ops/s | 320–367K | 13–14K | +| READ ÔÇö Direct / StructRow | 70ÔÇô126K ops/s | 87ÔÇô97K | 14ÔÇô16K | +| READ ÔÇö SQL | 51ÔÇô59K ops/s | 87ÔÇô97K | 14ÔÇô16K | +| INSERT (batch) | 91ÔÇô133K ops/s | 145ÔÇô150K | 66ÔÇô77K | +| UPDATE (batch) | 41ÔÇô59K ops/s | 241ÔÇô296K | 10ÔÇô11K | +| DELETE (batch) | 30ÔÇô142K ops/s | 320ÔÇô367K | 13ÔÇô14K | ### Added -- **`ExecuteQueryStruct(sql, params)`** — first-class zero-allocation struct-row SQL reads with a +- **`ExecuteQueryStruct(sql, params)`** ÔÇö first-class zero-allocation struct-row SQL reads with a cached `VariableLengthSchema` (column layout parsed once, not per row). -- **`FindByPrimaryKey(table, key)` / `FindByIndex(table, col, value)` direct reads** — no-SQL +- **`FindByPrimaryKey(table, key)` / `FindByIndex(table, col, value)` direct reads** ÔÇö no-SQL point lookups (Direct API tier, the benchmark's "Direct" path). -- **`SimpleSelectPlan` zero-reparse SELECT fast path** — simple `SELECT … WHERE key = @p` plans +- **`SimpleSelectPlan` zero-reparse SELECT fast path** ÔÇö simple `SELECT ÔǪ WHERE key = @p` plans resolve from the query plan cache without re-lexing or re-parsing. -- **SIMD numeric WHERE batch filters** — `Vector` batch predicate evaluation for Integer/Long +- **SIMD numeric WHERE batch filters** ÔÇö `Vector` batch predicate evaluation for Integer/Long columns plus a fixed-offset numeric predicate fast path in columnar scans. -- **Native AOT readiness** — AOT-safe `TypeConverter`, `Option` reader, +- **Native AOT readiness** ÔÇö AOT-safe `TypeConverter`, `Option` reader, `[RequiresDynamicCode]` annotations, source-generated `TableMetadataDto` and `SharpCoreDBJsonContext`. `tools/SharpCoreDB.AotSmoke` publishes and runs successfully (CREATE/INSERT/query/StructRow/reopen, exit 0). -- **New benchmark APIs & tests** — `ExecuteQueryStruct` benchmark path + 5 tests; +- **New benchmark APIs & tests** ÔÇö `ExecuteQueryStruct` benchmark path + 5 tests; regression test for positional `?` placeholders falling back to the legacy binder. ### Changed -- **Removed hot-path debug file I/O** — unconditional `File.AppendAllText` writes to `D:\*.log` +- **Removed hot-path debug file I/O** ÔÇö unconditional `File.AppendAllText` writes to `D:\*.log` (per SELECT, per ExecuteSQL, per transaction, per INSERT) are gone. This single artifact was the dominant v1.x read bottleneck. - **`NormalizeSql` is regex-free** with an allocation short-circuit for query-plan-cache keys. - **All hot-path regexes are compiled** (batch UPDATE/DELETE parsing, provider detection, `ExecuteQueryFast`). -- **`HashIndex.Add/Remove` operate on the key only** — no full row copies during index maintenance. +- **`HashIndex.Add/Remove` operate on the key only** ÔÇö no full row copies during index maintenance. - **`UpdateMultiple` no longer copies rows** (`new Dictionary(row)` removed); `DeduplicateByPrimaryKey` early-exits on redundant keys. -- **`LookupPositionsUnsafe`** — no-copy position lookup under an explicit write-lock contract. -- **DI cached** — `IGraphRagProvider` is resolved once instead of per call in +- **`LookupPositionsUnsafe`** ÔÇö no-copy position lookup under an explicit write-lock contract. +- **DI cached** ÔÇö `IGraphRagProvider` is resolved once instead of per call in `GetSharedSqlParser`. -- **Provider fast paths** — `OPTIONALLY` keyword check avoids a full parse per `ExecuteReader`; +- **Provider fast paths** ÔÇö `OPTIONALLY` keyword check avoids a full parse per `ExecuteReader`; span-based single-file and `sqlite_master` detection. -- **Fixed regression** — positional `?` placeholders now fall back to the legacy parameter binder +- **Fixed regression** ÔÇö positional `?` placeholders now fall back to the legacy parameter binder (previously treated as SQL literals by the fast path). ### Compatibility -- **100% backward compatible** with v1.9.x — no public API breaking changes; the fast-path APIs are additive. +- **100% backward compatible** with v1.9.x ÔÇö no public API breaking changes; the fast-path APIs are additive. - Toolchain locked to **.NET 10 / C# 14** for v2.0.x; .NET 11 / C# 15 planned for v2.1. ### Validation @@ -219,13 +306,13 @@ updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: ## [1.9.6] - 2026-08-28 ### Fixed -- **Issue 339 — `WHERE col IN (...)` silently returned ALL rows (regression)**: every `IN` variant +- **Issue 339 ÔÇö `WHERE col IN (...)` silently returned ALL rows (regression)**: every `IN` variant (literal lists, parameterized lists, single-value lists, `NOT IN`) was ignored by the predicate evaluators and fell through to an "accept all" path: - `SingleFileTable.EvaluateSingleCondition` did not recognize `IN`/`NOT IN` at all (single-file `.scdb` mode) and returned `true` for every row. - - `Table.EvaluateWhere` (directory mode) split the value list on spaces — `IN ('a', 'b')` lost - everything after the first value — and non-string columns fell into the switch's `default: + - `Table.EvaluateWhere` (directory mode) split the value list on spaces ÔÇö `IN ('a', 'b')` lost + everything after the first value ÔÇö and non-string columns fell into the switch's `default: return true`. - `SqlParser.EvaluateOperator` (enhanced/AST path) did not strip the surrounding parentheses from the value list. @@ -246,38 +333,38 @@ updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: ### Added - **Regression tests for parameter binding**: `ParametricInsertTests` (9 tests) round-trip - parameterized INSERT/SELECT/UPDATE with 4–11 named parameters and assert the values land in the + parameterized INSERT/SELECT/UPDATE with 4ÔÇô11 named parameters and assert the values land in the columns the SQL specifies. - **Regression tests for server parameter pass-through**: `ParameterRoundTripTests` (2 tests) validate parameterized INSERT + SELECT over gRPC. - **ULID specification compatibility tests**: 6 new tests in `UlidTests` validate generation, parsing and timestamp extraction against the official ULID test vector - (`0000XSNJG0MQJHBF4QX1EFD6Y3` / timestamp `1000000000` ms), the 128-bit range (`7ZZZ…Z` accepted, - `8ZZZ…Z` rejected) and the 48-bit timestamp limit. + (`0000XSNJG0MQJHBF4QX1EFD6Y3` / timestamp `1000000000` ms), the 128-bit range (`7ZZZÔǪZ` accepted, + `8ZZZÔǪZ` rejected) and the 48-bit timestamp limit. ### Fixed -- **Issue 336 — parameterized INSERT bound values to the wrong columns**: `SqlParser.BindParameters` +- **Issue 336 ÔÇö parameterized INSERT bound values to the wrong columns**: `SqlParser.BindParameters` used substring-based replacement, so a parameter name that is a prefix of another (`@t` vs `@tid`) - corrupted the longer placeholder (e.g. `@tid` → `200id`). Binding is now token-aware via - `ParameterBinder.Bind` — the single source of truth for named and positional parameters — and + corrupted the longer placeholder (e.g. `@tid` ÔåÆ `200id`). Binding is now token-aware via + `ParameterBinder.Bind` ÔÇö the single source of truth for named and positional parameters ÔÇö and replaces every occurrence of each placeholder. -- **Issue 337 — SharpCoreDB.Server dropped `request.Parameters`**: `DatabaseService.ExecuteQuery` and +- **Issue 337 ÔÇö SharpCoreDB.Server dropped `request.Parameters`**: `DatabaseService.ExecuteQuery` and `ExecuteNonQuery` now translate `request.Parameters` into the parameter dictionary expected by the engine. The binary protocol handler now parses bind-message parameter values (and `$n` placeholders) and forwards them, and the WebSocket handler forwards parameters as well. - **ULID encoding was not standards-compliant**: the Crockford Base32 encoder/decoder treated a ULID as a plain 128-bit bit stream (RFC-4648 style), so generated ULIDs were not interchangeable with other standards-compliant implementations (Python/Java/Go). Encoding now follows the ULID - specification — the first character carries only 3 significant bits — and decoding rejects values + specification ÔÇö the first character carries only 3 significant bits ÔÇö and decoding rejects values above the 128-bit range. `Ulid.NewUlid(long)` also enforces the 48-bit timestamp limit. *Breaking change vs 1.9.4 for previously stored ULID strings, mirroring posseth.global.ulid v2.0.0.* - **Upgrade path for legacy ULIDs**: new `Ulid.FromLegacy(string)` / `Ulid.TryFromLegacy(...)` convert ULIDs generated before 1.9.5 into the current spec-compliant encoding. The 128-bit value - (timestamp + randomness) is preserved exactly — only the Base32 text changes — so existing + (timestamp + randomness) is preserved exactly ÔÇö only the Base32 text changes ÔÇö so existing `_rowid` values and ULID columns can be migrated one-to-one. The legacy encoder/decoder is kept as `Base32.LegacyEncode`/`Base32.LegacyDecode` for migration tooling. - **Automatic legacy-database detection and one-shot ULID migration**: `Database.NeedsLegacyUlidMigration()` - tells you whether a database was created before 1.9.5 — the ULID encoding generation is recorded in + tells you whether a database was created before 1.9.5 ÔÇö the ULID encoding generation is recorded in the database metadata (directory mode) and in the file-header feature flags (single-file `.scdb` mode), so no schema or version guessing is needed. `Database.MigrateLegacyUlids()` rewrites every ULID value in every `ULID`-typed column of every table (including hidden `_rowid` primary keys) to the @@ -310,17 +397,17 @@ updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: ## [1.9.4] - 2026-08-22 ### Added -- **Known Issue 1 — opt-in at-rest per-record encryption**: `DatabaseConfig.EnableAtRestRecordEncryption` +- **Known Issue 1 ÔÇö opt-in at-rest per-record encryption**: `DatabaseConfig.EnableAtRestRecordEncryption` (default `false` for full backward compatibility). When enabled, table data files carry an 8-byte magic header and each appended record is AES-256-GCM encrypted; point reads, full scans, PK index rebuilds and compaction decrypt transparently. Legacy plaintext files and `NoEncryptMode` remain byte-for-byte unchanged; legacy/encrypted file mixing is prevented per file. -- **Known Issue 6 — opt-in SQLite integer affinity**: `DatabaseConfig.UseSqliteIntegerAffinity` +- **Known Issue 6 ÔÇö opt-in SQLite integer affinity**: `DatabaseConfig.UseSqliteIntegerAffinity` (default `false`). When enabled, `INTEGER` DDL maps to `DataType.Long` (Int64) so values like `DateTime.UtcNow.Ticks` fit; the default Int32 path now throws an actionable overflow message pointing to `BIGINT`/the flag. -- **Single-file ↔ directory SQL parity**: single-file mode now handles the full WHERE operator set - identically to directory mode — `LIKE` / `NOT LIKE` (case-insensitive, `%`/`_`, NULL never matches), +- **Single-file Ôåö directory SQL parity**: single-file mode now handles the full WHERE operator set + identically to directory mode ÔÇö `LIKE` / `NOT LIKE` (case-insensitive, `%`/`_`, NULL never matches), `IS NULL` / `IS NOT NULL`, and `BETWEEN` (inclusive, culture-independent numeric comparison). Aggregates (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`), `GROUP BY`, `IN`, `ORDER BY`, `LIMIT`, `DISTINCT` and JOINs already matched via the shared `SqlParser` and are now covered by regression tests. @@ -329,20 +416,20 @@ updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: 15 intentionally-skipped CPU-timing performance benchmarks. ### Changed -- **Version bump 1.9.3 → 1.9.4** across all packable `.csproj` files, `Directory.Packages.props` +- **Version bump 1.9.3 ÔåÆ 1.9.4** across all packable `.csproj` files, `Directory.Packages.props` (`SharpCoreDBVersion`), test projects, and documentation (hub docs, per-package READMEs, NuGet-readme info, script clients). `DocumentationConsistencyTests` now enforces `1.9.4` as the current release label. -- **Known Issue 2 — reopen AOORE fix**: `Database.Load()` now pads `DefaultExpressions`, +- **Known Issue 2 ÔÇö reopen AOORE fix**: `Database.Load()` now pads `DefaultExpressions`, `ColumnCheckExpressions` and `ColumnLocaleNames` to the column count, so `ITable.Insert` after a reopen no longer throws `ArgumentOutOfRangeException`. -- **Known Issue 3 — single-file point operations**: `SingleFileTable.FindByPrimaryKey` / +- **Known Issue 3 ÔÇö single-file point operations**: `SingleFileTable.FindByPrimaryKey` / `UpdateByPrimaryKey` / `DeleteByPrimaryKey` are now functional (transaction-aware, respect `AutoFlush`) instead of returning `null`/`false`. -- **Known Issue 4 — read-after-write**: `ExecuteQuery` flushes pending batch-update writes +- **Known Issue 4 ÔÇö read-after-write**: `ExecuteQuery` flushes pending batch-update writes (`_batchUpdateActive`) before executing, matching `ExecuteSQL(SELECT)`; plain metadata dirtiness is no longer force-flushed per query (avoids page-based engine read regressions). -- **Known Issue 5 — SQL validator**: parameter keys are normalized by stripping `@`/`:` prefixes +- **Known Issue 5 ÔÇö SQL validator**: parameter keys are normalized by stripping `@`/`:` prefixes (consistent with `SqlParser.ResolveParameter`), removing false "Missing/Unused parameters" warnings while genuine mismatches are still reported. - **Benchmark test fix**: `InsertOptimizationsTests.Baseline_10K_Inserts_Without_Optimizations` used @@ -357,10 +444,10 @@ updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: ## [1.9.3] - 2026-07-28 ### Added -- **SharpCoreDB.Functional.Linq2DB v1.9.3** — Full production release of the linq2db adapter. +- **SharpCoreDB.Functional.Linq2DB v1.9.3** ÔÇö Full production release of the linq2db adapter. - `FunctionalLinq2DbContext` providing `Option`, `Fin`, `Seq` APIs over linq2db (`FindOneAsync`, `QueryAsync` with builder/predicate, `GetAllAsync`, `InsertAsync`/`InsertBatchAsync` (BulkCopy), `UpdateAsync`, `Delete*Async`, `CountAsync`, `ExistsAsync`, `TransactionAsync`). - High-performance `BulkCopyAsync` support for batch operations (critical for GraphRAG, AI ingestion, analytics). - - Complete type mapping schema (`Ulid`, `Guid` (compact N format), `DateTime`/`DateTimeOffset` (ISO), `bool` ↔ integer for SQLite compatibility). + - Complete type mapping schema (`Ulid`, `Guid` (compact N format), `DateTime`/`DateTimeOffset` (ISO), `bool` Ôåö integer for SQLite compatibility). - Modern `DataOptions`-based constructors (fixes linq2db deprecation warnings). - Comprehensive documentation, examples, and cross-references in root README, `FEATURE_MATRIX`, GraphRAG guide, functional SQL docs, and dedicated package README. @@ -369,7 +456,7 @@ updates and deletes). Measured final two-run ranges vs SQLite/LiteDB: - All documentation refreshed to highlight the new library as a first-class, production-ready functional LINQ option (especially valuable for agentic/AI and GraphRAG workloads). ### Fixed -- Test projects updated to use compatible SQLite connection strings (`"Data Source=..."`) — resolves linq2db `Microsoft.Data.Sqlite` provider parsing errors with SharpCoreDB's `"Path=..."` format. +- Test projects updated to use compatible SQLite connection strings (`"Data Source=..."`) ÔÇö resolves linq2db `Microsoft.Data.Sqlite` provider parsing errors with SharpCoreDB's `"Path=..."` format. - `GetByIdAsync` improved with safe fallback and explicit limits. - All tests in `SharpCoreDB.Functional.Linq2DB.Tests` now pass reliably. - Build and CI compatibility verified (including Release configuration). @@ -406,8 +493,8 @@ This release is a pure preparation/synchronization release with zero functional ## [1.7.2] - 2026-04-28 ### Added -- **SIMD LoadUnsafe Optimization**: All 16 columnar SIMD aggregate methods (`SumInt32`, `SumInt64`, `SumDouble`, `MinInt32`, `MinInt64`, `MinDouble`, `MaxInt32`, `MaxInt64` — both single-threaded and parallel variants) now use `Vector256.LoadUnsafe(ref data[i])` instead of `Vector256.Create(data.AsSpan(i))`. This eliminates per-iteration `Span` construction and bounds checking overhead in SIMD hot loops, yielding tighter codegen on AVX2 hardware. -- **Auto-ROWID**: Tables created without an explicit `PRIMARY KEY` now receive a hidden `_rowid` column (ULID type, auto-generated). Follows the SQLite rowid pattern — invisible in `SELECT *`, visible when explicitly queried via `SELECT _rowid, ...`. See [`docs/features/AUTO_ROWID.md`](features/AUTO_ROWID.md) for full documentation. +- **SIMD LoadUnsafe Optimization**: All 16 columnar SIMD aggregate methods (`SumInt32`, `SumInt64`, `SumDouble`, `MinInt32`, `MinInt64`, `MinDouble`, `MaxInt32`, `MaxInt64` ÔÇö both single-threaded and parallel variants) now use `Vector256.LoadUnsafe(ref data[i])` instead of `Vector256.Create(data.AsSpan(i))`. This eliminates per-iteration `Span` construction and bounds checking overhead in SIMD hot loops, yielding tighter codegen on AVX2 hardware. +- **Auto-ROWID**: Tables created without an explicit `PRIMARY KEY` now receive a hidden `_rowid` column (ULID type, auto-generated). Follows the SQLite rowid pattern ÔÇö invisible in `SELECT *`, visible when explicitly queried via `SELECT _rowid, ...`. See [`docs/features/AUTO_ROWID.md`](features/AUTO_ROWID.md) for full documentation. - `Table.HasInternalRowId` property (persisted in metadata) to track tables with auto-generated `_rowid`. - `Table.SelectIncludingRowId()` method for queries that explicitly request `_rowid`. - `Database.GetColumnsIncludingHidden()` for schema discovery including hidden columns (with `IsHidden` flag). @@ -426,7 +513,7 @@ This release is a pure preparation/synchronization release with zero functional - Added parser support for scalar function expressions in SELECT columns (including `COALESCE(...)`) and parenthesized subquery expressions. - Improved `EnhancedSqlParser` malformed SQL detection by flagging unparsed trailing content via `HasErrors`. - Added LINQ translator handling for `ExpressionType.Convert` / `ConvertChecked` in enum-related comparison scenarios. -- Improved German locale comparison behavior for `ß/ss` equivalence in locale-aware matching. +- Improved German locale comparison behavior for `├ƒ/ss` equivalence in locale-aware matching. - Fixed PAGE_BASED mixed-predicate filtering (`column = value AND other_column <= value`) by routing scan-time predicate evaluation through the shared SQL condition evaluator; added regression coverage for `ORDER BY ... LIMIT` retrieval. - **ColumnStore SIMD consistency**: Cleaned up inconsistent `MaxInt64SIMDDirect` implementation (previously used manual `ref` + `Unsafe.Add` pattern) to use the same `Vector256.LoadUnsafe(ref data[i])` pattern as all other SIMD methods. @@ -463,11 +550,11 @@ This release is a pure preparation/synchronization release with zero functional ## [1.6.0] - 2026-03-30 -### 🎉 Major Achievement - Phase 12: GraphRAG Enhancement & Vector Search Integration COMPLETE +### ­ƒÄë Major Achievement - Phase 12: GraphRAG Enhancement & Vector Search Integration COMPLETE SharpCoreDB v1.6.0 introduces **GraphRAG (Graph Retrieval-Augmented Generation)** - a comprehensive graph analytics platform with semantic vector search integration for contextually rich search results. -### ✨ Added - Phase 12: GraphRAG Enhancement +### Ô£¿ Added - Phase 12: GraphRAG Enhancement #### GraphRAG Engine - **Real Semantic Search**: Vector search integration with HNSW indexing and SIMD acceleration (50-100x faster than SQLite) @@ -484,9 +571,9 @@ SharpCoreDB v1.6.0 introduces **GraphRAG (Graph Retrieval-Augmented Generation)* #### Comprehensive Centrality Metrics - **Degree Centrality**: O(n) - Direct connection count measuring popularity -- **Betweenness Centrality**: O(n × m) - Bridge detection for information flow analysis -- **Closeness Centrality**: O(n²) - Distance efficiency measuring accessibility -- **Eigenvector Centrality**: O(k × m) - Influence measurement for prestige analysis +- **Betweenness Centrality**: O(n ├ù m) - Bridge detection for information flow analysis +- **Closeness Centrality**: O(n┬▓) - Distance efficiency measuring accessibility +- **Eigenvector Centrality**: O(k ├ù m) - Influence measurement for prestige analysis - **SQL Functions**: Direct database functions for all centrality calculations #### Advanced Subgraph Queries @@ -501,7 +588,7 @@ SharpCoreDB v1.6.0 introduces **GraphRAG (Graph Retrieval-Augmented Generation)* - **Scaling Strategies**: Horizontal/vertical partitioning for massive graph processing - **Health Monitoring**: Cache statistics, performance alerts, and diagnostic tools -### 📚 Documentation & Examples +### ­ƒôÜ Documentation & Examples #### Comprehensive Documentation Suite - **API Reference**: Complete XML-documented API with complexity analysis @@ -515,7 +602,7 @@ SharpCoreDB v1.6.0 introduces **GraphRAG (Graph Retrieval-Augmented Generation)* - **Custom Providers**: Extensible interface for any embedding service - **Production Patterns**: Error handling, caching, monitoring, and scaling -### 🧪 Testing & Quality Assurance +### ­ƒº¬ Testing & Quality Assurance #### Comprehensive Test Suite - **20 integration tests** covering all major functionality @@ -523,7 +610,7 @@ SharpCoreDB v1.6.0 introduces **GraphRAG (Graph Retrieval-Augmented Generation)* - **Performance validation** with automated benchmarking - **Memory safety** verified through comprehensive testing -### 📊 Performance Metrics +### ­ƒôè Performance Metrics #### Benchmark Results ``` @@ -539,7 +626,7 @@ Enhanced Ranking: 5ms (2000 ops/sec) - **SIMD acceleration**: Hardware-optimized vector operations - **Batch processing**: Handles large datasets without memory pressure -### 🧹 Documentation Migration & Cleanup +### ­ƒº╣ Documentation Migration & Cleanup - Removed obsolete phase-status, kickoff, completion, and superseded planning documents across `docs/archived`, `docs/server`, and `docs/graphrag`. - Consolidated documentation navigation to canonical entry points: - `docs/INDEX.md` diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 73974763..0adf62d9 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -55,8 +55,18 @@ SharpCoreDB core .NET packages are release-labeled on `2.0.0` and build successf ## Roadmap / TODO (v2.1) -- [ ] **Close UPDATE/DELETE gap vs SQLite** — in-place record updates, fixed-width record layout - for hot tables, eliminate read-modify-write in `UpdateMultiple`. +- [ ] **Close UPDATE/DELETE gap vs SQLite** (in progress — details in + `docs/performance/V2_PERFORMANCE_PLAN.md` §3.4 / §3.5): + - ✅ **In-place UPDATE for columnar/append-only (Issue #6)** — fixed-width / unchanged-length + records overwrite their existing slot (`TryUpdateInPlace`); no new version, no file growth. + - ✅ **Single-pass SQL DELETE/UPDATE (Issue #7/#8)** — `DeleteAffectedRows` / `UpdateAffectedCount` + return the affected rows/count from the table operation itself, so the SQL paths no longer + materialize matching rows twice for RETURNING / change-tracking. + - ✅ **PK fast path in `Delete` / `DeleteMultiple` / `UpdateMultiple`** — a simple `pk = value` + WHERE resolves via the primary-key B-tree directly (single search + one read) instead of + full-row materialization + per-row re-search. + - [ ] Fixed-width record layout for hot tables (SQLite-style C record format) + - [ ] Storage-level DELETE reuse (free-slot reuse / compaction on PageBased deletes) - [ ] **.NET 11 / C# 15 migration** (after Nov 2026 GA) — Runtime Async, AVX-VNNI-512/SVE2 behind `SIMD_ENABLED`, optional Zstandard compression. - [ ] **Native AOT warning cleanup** — interface-based B-tree factory (replace `GetMethod`/ diff --git a/docs/backlog/NET11_C15_BACKLOG.md b/docs/backlog/NET11_C15_BACKLOG.md new file mode 100644 index 00000000..86f37321 --- /dev/null +++ b/docs/backlog/NET11_C15_BACKLOG.md @@ -0,0 +1,40 @@ +# V2 Backlog — .NET 11 / C# 15-gebonden werk + +**Status:** Gevuld · 2026-08-31 · v2.1-lijn (`release/v2.1.0.0`) +**Doel:** Items hieronder kunnen pas worden uitgevoerd zodra de bijbehorende runtime-/compiler- +functies beschikbaar zijn (target: .NET 11 GA, november 2026). Alles wat **nu al kan** en de +performance verhoogt om het SQLite-gat te dichten, wordt buiten deze backlog uitgevoerd — zie +[`docs/performance/V2_PERFORMANCE_PLAN.md`](../performance/V2_PERFORMANCE_PLAN.md). + +## Waarom deze items geblokkeerd zijn + +Zie `V2_PERFORMANCE_PLAN.md` §4.0 (preview-7-metingen, 2026-08-30): + +| Functie | In preview 7? | Blokkade | +|---|---|---| +| Numeriek `LangVersion 15.0` | ❌ | Preview-compiler geeft `CS1617`; `LangVersion latest` is de tijdelijke workaround | +| Runtime Async | ✅ (net11) | automatic; geen code-wijziging tot GA-baseline | +| AVX-VNNI-512 / Arm SVE2 | ⚠️ | SVE2 is `SYSLIB5003` evaluation-only; SVE2 uitstellen tot GA | +| SIMD lane APIs | ✅ (preview 7) | vereist een columnar-layout refactor, geen point-edit | +| Zstandard (`ZstdCompressor`) | ❌ | niet in preview 7; uitgesteld tot later preview/GA | +| IEEE 754 `Decimal32/64/128` | ❌ | niet in preview 7; uitgesteld tot GA | +| C# 15 union types / closed hierarchies | ⚠️ | nog niet gestabiliseerd; valideren vóór AST-refactor | + +## Backlog + +| # | Item | Afhankelijkheid | Aanraakgebied (indicatie) | +|---|------|-----------------|---------------------------| +| B1 | `LangVersion latest` → `15.0` | .NET 11 GA | `Directory.Build.props`, `global.json` | +| B2 | Runtime-native async in async hot paths | Runtime Async (net11 GA) | `Execute*Async`, `InsertBatchAsync`, `ExecuteBatchSQLAsync`, server-paden | +| B3 | AVX-VNNI-512 (x64) + Arm SVE2 intrinsics achter `SIMD_ENABLED`-guards | AVX-VNNI-512 net11; SVE2 eval-only tot GA | `DistanceMetrics`, `SimdHelper`, vector search (HNSW) | +| B4 | SIMD lane APIs (`Zip`/`Unzip`/`CreateGeometricSequence`/`Concat`) in columnar codecs | APIs ✓ in preview 7, maar vereist columnar-layout refactor | Delta, Gorilla, XorFloat, RLE, bit-packing, SIMD row scanning | +| B5 | Zstandard WAL/page-compressie (opt-in, default uit) | `ZstdCompressor` in `System.IO.Compression` | WAL + page-compressie (net als bestaande Brotli/GZip block-compressie) | +| B6 | IEEE 754 `Decimal32/64/128` + `INumberBase.TryParsePartial` | runtime (niet in preview 7) | decimal-column parsing/serialisatie | +| B7 | C# 15 union types / closed hierarchies voor de SQL-AST | compiler-stabilisatie (Phase 4) | `SqlParser`, planner, AST/SQL-node design | +| B8 | Automatic JIT / NativeAOT-dispatch wins meten | net11 GA (geen code-wijziging) | re-benchmark + `V2_PERFORMANCE_PLAN.md` §3.2 bijwerken | + +## Niet performance-gerelateerd (apart bijhouden, niet deze backlog) + +- Native AOT-waarschuwings-cleanup: B-tree factory ipv. reflectie, `ParseVectorValue`/`.scdb` JSON + naar source-generated context. +- `SingleFileDatabase` → `IMetadataProvider` pariteit (metadata-detectie via `db is IMetadataProvider`). diff --git a/docs/benchmarks/FIXED_WIDTH_BENCHMARK.md b/docs/benchmarks/FIXED_WIDTH_BENCHMARK.md new file mode 100644 index 00000000..6b9ba468 --- /dev/null +++ b/docs/benchmarks/FIXED_WIDTH_BENCHMARK.md @@ -0,0 +1,72 @@ +# Fixed-Width vs Legacy — benchmark results + +Run: 2026-09-01, .NET 11.0.0-preview.7, Windows, Release. +Command: `dotnet run --project tests/benchmarks/SharpCoreDB.Benchmarks.Comparative -- --fixedwidth` + +The same workloads run against a legacy (variable-length records) database and a fixed-width +database (directory-mode Columnar, `DatabaseConfig.FixedWidthRecordLayout = true`). Both databases +use identical settings (no encryption, memory mapping, page cache). + +## Results + +| Workload | Metric | Legacy | Fixed-width | Win | +|---|---|---|---|---| +| A · 10,000 growing variable-column updates | elapsed | 17.99 s | **2.12 s** | **~8.5× faster** | +| A · 10,000 growing variable-column updates | storage growth (post-auto-compact) | 0.0 KB | 20.4 KB | ≈ | +| B · 1,000 variable updates + arena compaction | elapsed | 0.59 s | **0.14 s** | **~4× faster** | +| B · 1,000 variable updates + arena compaction | storage growth | 23 B | 17 B | ≈ | +| C · 30 full scans, non-indexed `WHERE category = -1` over 100,000 rows | time per query | 6.31 ms | **2.63 ms** | **~2.4× faster** | +| D · 100,000 batch inserts | throughput | 242,487 rows/s | 208,692 rows/s | ~14% slower | + +## Interpretation + +- **Updates (growing variable values) — ~8.5× faster.** Legacy appends a new record per growing + update and pays for full `.dat` compactions (1000-update threshold); fixed-width keeps the `.dat` + constant (in-place overwrite), grows only the overflow arena, and compacts only the arena (B1/B3). +- **Non-indexed full-scan WHERE — ~2.4× faster.** Fixed-width reads the predicate column at its + constant slot offset (numeric early-WHERE) or compares the arena payload (string early-WHERE) and + skips full-row deserialization for non-matches (B4). +- **Variable updates + compaction — ~4× faster.** The arena copy-on-compact is cheaper than a + `.dat` rewrite (B3). +- **Inserts — ~14% slower.** Fixed-width writes each variable value into the overflow arena + (payload encoding + free-list bookkeeping); for insert-heavy workloads the legacy format is + slightly faster. This is the expected trade-off: fixed-width targets update-heavy / point-read + workloads. + +## Comparative CRUD vs SQLite/LiteDB (post B7 update-path work) + +AppendOnly engine (`--engine=appendonly`), 100K inserts / 10K reads / 10K updates / 10K deletes: + +| Database | INSERT ops/s | READ ops/s | UPDATE ops/s | DELETE ops/s | +|---|---|---|---|---| +| SharpCoreDB (SQL) | 91,873 | 65,858 | 37,203 | 60,211 | +| SharpCoreDB (Direct) | 116,146 | 141,293 | 46,189 | 61,554 | +| SharpCoreDB (StructRow) | 135,322 | 120,224 | – | – | +| SQLite | 148,654 | 95,143 | 281,072 | 351,863 | +| LiteDB | 78,569 | 13,721 | 9,641 | 14,710 | + +PageBased engine (`--engine=pagebased`): + +| Database | INSERT ops/s | READ ops/s | UPDATE ops/s | DELETE ops/s | +|---|---|---|---|---| +| SharpCoreDB (SQL) | 123,995 | 30,807 | 69,574 | 124,176 | +| SharpCoreDB (Direct) | 124,917 | 39,669 | 102,160 | 175,887 | +| SharpCoreDB (StructRow) | 209,068 | 51,438 | – | – | +| SQLite | 146,301 | 94,523 | 266,673 | 372,029 | +| LiteDB | 71,677 | 13,616 | 10,337 | 14,061 | + +- **vs LiteDB: SharpCoreDB wins every workload** (1.5–8×). +- **vs SQLite:** SharpCoreDB wins on INSERT (StructRow) and on READ (AppendOnly Direct + SQL after + B9); SQLite remains ~6–8× faster on UPDATE/DELETE. The batch UPDATE path is now in-place and + **deserialize-free on the hot path** (only the changed fields are patched at their slot + offsets) — the remaining gap is the per-statement parser + write-behind bookkeeping vs SQLite's + specialized b-tree writes. + +### Point-read micro-benchmark (`--readtest`, median of 7 × 10K reads on 100K rows) + +| Path | ops/s | notes | +|---|---|---| +| SQL `SELECT * FROM docs WHERE name = @name` | ~120,000–166,000 | B9 direct hash-index lookup (was ~65,000) | +| Direct `FindByIndex("docs", "name", …)` | ~160,000–188,000 | reference | +| SQL/Direct overhead | 1.1–1.5× | was ~2× | +| SQLite (same workload) | ~95,000 | — | diff --git a/docs/benchmarks/V198_V20_V21_PERFORMANCE_COMPARISON.md b/docs/benchmarks/V198_V20_V21_PERFORMANCE_COMPARISON.md index 37a559e9..ae81409e 100644 --- a/docs/benchmarks/V198_V20_V21_PERFORMANCE_COMPARISON.md +++ b/docs/benchmarks/V198_V20_V21_PERFORMANCE_COMPARISON.md @@ -61,6 +61,7 @@ - SQL UPDATE is ~5–10× slower than SQLite in every version (2.x: 27–43K vs SQLite 218–280K). - SQL DELETE is ~5–17× slower (2.x: 21–61K vs SQLite 295–364K), high variance. - Root cause is structural: SharpCoreDB's row-store updates/deletes are row-copy based, while SQLite uses fixed-length C records with direct field offsets and in-place writes. This is the targeted v2.1+ engine work (in-place records), **not** something the runtime or allocations fix. +- **Progress (#6, 2026-08-31):** the in-place UPDATE engine landed on `release/v2.1.0.0` (`3d4cee77` + `68cb5dab`) and `release/v2.0.0.0` (`116fc30e` + `8a13ba2b`). On the columnar/append-only engine, **fixed-width UPDATEs no longer append a new version**: measured ~3.5–5.3K ops/s with **0 file growth** vs ~1.5K ops/s with +90 KB growth per 2,000 updates before. Variable-width updates fall back to the append path unchanged. See `V2_PERFORMANCE_PLAN.md` §3.4. The DELETE gap and the numbers above (measured on the default single-file path) are unchanged by this work. ### 3.4 Versus competitors - SharpCoreDB 2.x beats **LiteDB on every operation** (~5–8× reads, ~4–5× updates, ~3–9× deletes). diff --git a/docs/performance/V2_PERFORMANCE_PLAN.md b/docs/performance/V2_PERFORMANCE_PLAN.md index f79353ff..6f50790d 100644 --- a/docs/performance/V2_PERFORMANCE_PLAN.md +++ b/docs/performance/V2_PERFORMANCE_PLAN.md @@ -192,7 +192,31 @@ What changed: > (2–26× over scalar). See > [`docs/benchmarks/AVX512_2026-09-01.md`](../benchmarks/AVX512_2026-09-01.md). - +### 3.5 #7/#8 single-pass DML — SQL DELETE/UPDATE no longer materialize twice (2026-08-31, `release/v2.1.0.0`) + +The SQL DELETE path previously materialized every matching row **twice** per statement: +`ExecuteDelete` ran a full `Select` (for RETURNING + affected-count) and then `Table.Delete` +re-scanned/re-deserialized the same rows. The SQL UPDATE path was worse: a full `Select().Count` +for change-tracking, the update pass itself, and — for RETURNING — a second full `Select`. + +Changes: + +- **`ITable.DeleteAffectedRows(where)`** — default implementation keeps the historic two-pass + behavior for third-party `ITable` implementers; `Table` and `SingleFileTable` override with a + single pass (delete AND return the affected pre-delete rows). `ExecuteDelete` now uses it: + one scan, RETURNING + count from the same rows. +- **`ITable.UpdateAffectedCount(where, updates)`** — same default/override pattern; applies the + update and returns the affected count. `ExecuteUpdate` now uses it; the separate `Select().Count` + pass is gone (RETURNING still re-selects, only when requested). +- **PK fast path (Issue #7) extended to `DeleteMultiple` and `UpdateMultiple`** — a simple + `pk = value` WHERE on a columnar table resolves via the PK B-tree directly (single search + one + read) instead of `SelectInternal` full-row materialization + a per-row PK re-search. Range / + compound / non-indexed WHERE clauses bypass the fast path and keep their (correct) generic + behavior — `TryParseSimpleWhereClause` only accepts a plain `col = value`. + +Regression coverage: `DmlSinglePassTests` (affected counts, RETURNING pre-delete rows, range + +non-indexed fallbacks, batch PK deletes/updates) + the existing RETURNING / `CHANGES()` tests. +Full suite green: **1,644 tests, 0 failures** (16 skipped). --- diff --git a/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs b/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs new file mode 100644 index 00000000..10d9c112 --- /dev/null +++ b/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs @@ -0,0 +1,154 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.DataStructures; + +using System.Buffers.Binary; +using System.Collections.Generic; + +/// +/// Shared fixed-width record codec (out-of-line overflow model). Every column occupies a constant +/// slot in the record's fixed part: fixed-size columns store [null-flag(1)][payload] inline, +/// variable-length columns (String / Blob) store a 5-byte slot [null-flag(1)][arena-offset(4)] +/// referencing a block in the overflow arena. Used by both the directory-mode +/// and the single-file (.scdb) table so the two storage modes share one record format. +/// +public static class FixedWidthCodec +{ + /// Serializes a row dictionary into a fixed-width record (variable values → arena). + public static byte[] SerializeRow( + Dictionary row, + IReadOnlyList columns, + IReadOnlyList types, + FixedWidthRecordLayout layout, + IOverflowArena arena) + { + var buffer = new byte[layout.FixedSize]; + var span = buffer.AsSpan(); + + for (int i = 0; i < columns.Count; i++) + { + var slot = span.Slice(layout.Offsets[i], layout.SlotSizes[i]); + var value = row.TryGetValue(columns[i], out var v) ? v : DBNull.Value; + + if (layout.IsVariable[i]) + { + if (value == null || value == DBNull.Value) + { + slot[0] = 0; + BinaryPrimitives.WriteInt32LittleEndian(slot[1..], 0); + } + else + { + var payload = Table.EncodeVariablePayload(types[i], value); + var offset = arena.Write(payload); + slot[0] = 1; + BinaryPrimitives.WriteInt32LittleEndian(slot[1..], (int)offset); + } + } + else + { + _ = Table.WriteTypedValueToSpan(slot, value, types[i]); + } + } + + return buffer; + } + + /// Deserializes a fixed-width record into a row dictionary (variable values ← arena). + public static Dictionary DeserializeRow( + ReadOnlySpan data, + IReadOnlyList columns, + IReadOnlyList types, + FixedWidthRecordLayout layout, + IOverflowArena arena) + { + var row = new Dictionary(columns.Count, System.StringComparer.Ordinal); + + for (int i = 0; i < columns.Count; i++) + { + if (layout.Offsets[i] + layout.SlotSizes[i] > data.Length) + { + break; // truncated / corrupt record + } + + var slot = data.Slice(layout.Offsets[i], layout.SlotSizes[i]); + if (layout.IsVariable[i]) + { + if (slot[0] == 0) + { + row[columns[i]] = DBNull.Value; + } + else + { + var offset = BinaryPrimitives.ReadInt32LittleEndian(slot[1..]); + var payload = arena.Read(offset); + row[columns[i]] = payload is null ? DBNull.Value : Table.DecodeVariablePayload(types[i], payload); + } + } + else + { + row[columns[i]] = Table.ReadTypedValueFromSpan(slot, types[i], out _); + } + } + + return row; + } + + /// Collects the arena offsets referenced by a fixed-width record's variable slots. + public static void CollectVariableOffsets(byte[] record, FixedWidthRecordLayout layout, HashSet live) + { + for (int i = 0; i < layout.ColumnCount; i++) + { + if (!layout.IsVariable[i]) + { + continue; + } + + var slot = layout.Offsets[i]; + if (slot + 5 > record.Length || record[slot] == 0) + { + continue; // truncated or null slot + } + + var blockOffset = BinaryPrimitives.ReadInt32LittleEndian(record.AsSpan(slot + 1, 4)); + // NOTE: offset 0 is a valid block offset (first arena block) — the flag byte above + // already excluded NULL slots, so collect every referenced offset unconditionally. + live.Add(blockOffset); + } + } + + /// + /// Returns a copy of a fixed-width record with its variable slots re-pointed through the + /// compaction mapping, or null when no slot moved. + /// + public static byte[]? RepointVariableSlots(byte[] record, FixedWidthRecordLayout layout, Dictionary mapping) + { + byte[]? result = null; + + for (int i = 0; i < layout.ColumnCount; i++) + { + if (!layout.IsVariable[i]) + { + continue; + } + + var slot = layout.Offsets[i]; + if (slot + 5 > record.Length || record[slot] == 0) + { + continue; + } + + var blockOffset = BinaryPrimitives.ReadInt32LittleEndian(record.AsSpan(slot + 1, 4)); + // NOTE: offset 0 is a valid block offset (first arena block) — re-point it like any other. + if (mapping.TryGetValue(blockOffset, out var newOffset) && newOffset != blockOffset) + { + result ??= (byte[])record.Clone(); + BinaryPrimitives.WriteInt32LittleEndian(result.AsSpan(slot + 1, 4), (int)newOffset); + } + } + + return result; + } +} diff --git a/src/SharpCoreDB/DataStructures/FixedWidthRecordLayout.cs b/src/SharpCoreDB/DataStructures/FixedWidthRecordLayout.cs new file mode 100644 index 00000000..390a0e23 --- /dev/null +++ b/src/SharpCoreDB/DataStructures/FixedWidthRecordLayout.cs @@ -0,0 +1,90 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.DataStructures; + +using System.Collections.Generic; + +/// +/// Describes the fixed-width record layout for a table schema (out-of-line overflow model). +/// Every column gets a constant-size slot in the record's "fixed part": +/// - fixed-size columns (Integer, Long, Real, Boolean, DateTime, Decimal, Guid, Ulid) store their +/// value inline as [null-flag(1)][payload]; +/// - variable-length columns (String, Blob) store a 5-byte slot [null-flag(1)][overflowOffset(4)] +/// referencing a payload block in the table's overflow arena. +/// The record length is therefore constant per schema, so every update is an in-place overwrite. +/// +public sealed class FixedWidthRecordLayout +{ + /// Gets the per-column byte offset of each slot in the fixed part. + public required int[] Offsets { get; init; } + + /// Gets the per-column slot size (fixed = 1 + fixed payload size; variable = 5). + public required int[] SlotSizes { get; init; } + + /// Gets whether each column is variable-length (String / Blob → overflow arena). + public required bool[] IsVariable { get; init; } + + /// Gets the fixed part size in bytes (constant per schema). + public required int FixedSize { get; init; } + + /// Gets the number of columns. + public int ColumnCount => Offsets.Length; + + /// + /// Computes the fixed-width record layout for the given column types. Always succeeds — every + /// supported column type maps to either an inline fixed slot or a 5-byte overflow reference. + /// + public static FixedWidthRecordLayout Compute(IReadOnlyList columnTypes) + { + var count = columnTypes.Count; + var offsets = new int[count]; + var sizes = new int[count]; + var isVariable = new bool[count]; + + int offset = 0; + for (int i = 0; i < count; i++) + { + int fixedSize = GetFixedEncodedSize(columnTypes[i]); + if (fixedSize < 0) + { + // Variable-length column: [null-flag(1)][overflow offset(4)]. + isVariable[i] = true; + sizes[i] = 5; + } + else + { + // Fixed column: [null-flag(1)][payload] — GetFixedEncodedSize already includes the flag. + isVariable[i] = false; + sizes[i] = fixedSize; + } + + offsets[i] = offset; + offset += sizes[i]; + } + + return new FixedWidthRecordLayout + { + Offsets = offsets, + SlotSizes = sizes, + IsVariable = isVariable, + FixedSize = offset + }; + } + + // Slot size including the 1-byte null flag — must match Table.Serialization.GetFixedEncodedSize. + private static int GetFixedEncodedSize(DataType type) => type switch + { + DataType.Integer => 5, + DataType.Long => 9, + DataType.RowRef => 9, + DataType.Real => 9, + DataType.Boolean => 2, + DataType.DateTime => 9, + DataType.Decimal => 17, + DataType.Ulid => 31, + DataType.Guid => 17, + _ => -1 // String / Blob are variable-length + }; +} diff --git a/src/SharpCoreDB/DataStructures/IOverflowArena.cs b/src/SharpCoreDB/DataStructures/IOverflowArena.cs new file mode 100644 index 00000000..aedf96c5 --- /dev/null +++ b/src/SharpCoreDB/DataStructures/IOverflowArena.cs @@ -0,0 +1,23 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.DataStructures; + +/// +/// Shared contract for the fixed-width "out-of-line overflow" arena. A fixed-width record stores +/// variable-length column values as a 4-byte arena block offset in its fixed part; the arena owns +/// the payload blocks and (optionally) reuses freed blocks of equal length in place. +/// +public interface IOverflowArena +{ + /// Writes a payload and returns the block offset to store in a record's variable slot. + long Write(byte[] payload); + + /// Reads the payload stored at , or null when absent. + byte[]? Read(long offset); + + /// Drops the block at from the live set (space is reclaimed + /// by compaction or exact-length reuse). + void Free(long offset); +} diff --git a/src/SharpCoreDB/DataStructures/OverflowArena.cs b/src/SharpCoreDB/DataStructures/OverflowArena.cs new file mode 100644 index 00000000..ebf5b09b --- /dev/null +++ b/src/SharpCoreDB/DataStructures/OverflowArena.cs @@ -0,0 +1,246 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.DataStructures; + +using SharpCoreDB.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; + +/// +/// Append-only arena for variable-length (TEXT/BLOB) record values in fixed-width-record tables +/// (the SQLite-model "out-of-line overflow"). Blocks are [length(4)][payload] appended to a +/// per-table .ovf file; a fixed-width record stores the block's offset in its fixed part, so +/// every record update stays in place (the record length is constant per schema). Payloads are +/// cached in memory for the lifetime of the table. B6: freed blocks are tracked in a free-list and +/// reused in place when a new payload has the exact same length (in-memory); the remaining dead +/// space is reclaimed by the copy-on-compact pass. +/// +public sealed class OverflowArena : IDisposable, IOverflowArena +{ + private readonly IStorage _storage; + private readonly string _filePath; + private readonly Dictionary _cache = new(); + // B6: freed block offsets grouped by their payload length, for exact-length in-place reuse. + private readonly Dictionary> _freeByLength = new(); + private int _blockReuses; + private bool _loaded; + + /// + /// Initializes a new instance of the class. + /// + /// The storage provider used to read/write the arena file. + /// The arena file path (normally the table .dat path with a .ovf extension). + public OverflowArena(IStorage storage, string filePath) + { + _storage = storage ?? throw new ArgumentNullException(nameof(storage)); + _filePath = filePath ?? throw new ArgumentNullException(nameof(filePath)); + } + + /// Gets the arena file path. + public string FilePath => _filePath; + + /// Gets the number of payload blocks currently cached. + public int Count => _cache.Count; + + /// Enumerates all block offsets currently cached (live + freed), loading the arena first. + public IEnumerable GetAllOffsets() + { + EnsureLoaded(); + return _cache.Keys; + } + + /// B6: gets the number of times a freed block was reused in place (diagnostics). + public int BlockReuses => _blockReuses; + + /// B6: gets the number of freed blocks currently tracked for in-place reuse (diagnostics). + public int FreeBlockCount + { + get + { + int total = 0; + foreach (var list in _freeByLength.Values) + { + total += list.Count; + } + + return total; + } + } + + private void EnsureLoaded() + { + if (_loaded) + { + return; + } + + _cache.Clear(); + _freeByLength.Clear(); // in-memory free-list: rebuilt (empty) on a fresh session + + // ReadAllRecords yields (physical length-prefix offset, record payload) for both legacy + // plaintext and per-record encrypted files (it handles the encryption magic header), so the + // arena offsets stored in fixed-width records always resolve. + foreach (var (offset, payload) in _storage.ReadAllRecords(_filePath)) + { + _cache[offset] = payload; + } + + _loaded = true; + } + + /// + /// Writes a payload to the arena and returns the block offset (the position of the storage + /// record's length prefix — the value stored in a fixed-width record's variable slot). B6: when + /// a previously freed block has the exact same payload length, it is reused in place (the + /// storage layer requires identical plaintext length for in-place overwrites); otherwise the + /// block is appended. + /// + public long Write(byte[] payload) + { + ArgumentNullException.ThrowIfNull(payload); + EnsureLoaded(); + + if (TryReuseFreeBlock(payload, out var reusedOffset)) + { + return reusedOffset; + } + + var offset = _storage.AppendBytes(_filePath, payload); + _cache[offset] = payload; + return offset; + } + + /// + /// B6: attempts to reuse a freed block of the exact same payload length via an in-place + /// overwrite. Returns false when no suitable block is free or the storage refuses the + /// in-place write (e.g. inside a transaction) — the caller then appends. + /// + private bool TryReuseFreeBlock(byte[] payload, out long offset) + { + offset = 0; + if (!_freeByLength.TryGetValue(payload.Length, out var offsets)) + { + return false; + } + + while (offsets.Count > 0) + { + offset = offsets[^1]; + offsets.RemoveAt(offsets.Count - 1); + + if (_storage.OverwriteRecordAt(_filePath, offset, payload)) + { + if (offsets.Count == 0) + { + _freeByLength.Remove(payload.Length); + } + + _cache[offset] = payload; + _blockReuses++; + return true; + } + + // In-place overwrite refused (e.g. transaction active): keep the block free for a + // later write and try the next candidate; if none succeeds we fall back to append. + offsets.Add(offset); + break; + } + + offset = 0; + return false; + } + + /// Reads the payload stored at , or null when absent. + public byte[]? Read(long offset) + { + EnsureLoaded(); + return _cache.TryGetValue(offset, out var payload) ? payload : null; + } + + /// Drops the block at from the live cache. B6: the freed block + /// is tracked for exact-length in-place reuse; otherwise its disk space is reclaimed by the next + /// copy-on-compact pass. + public void Free(long offset) + { + EnsureLoaded(); + if (!_cache.Remove(offset, out var payload)) + { + return; // already freed (or unknown) — never double-track + } + + if (!_freeByLength.TryGetValue(payload.Length, out var offsets)) + { + offsets = []; + _freeByLength[payload.Length] = offsets; + } + + offsets.Add(offset); + } + + /// + /// Copy-on-compact: rewrites the live blocks (those in ) into a + /// fresh arena file and returns a mapping from old offset to new offset. Callers must update + /// the fixed-width records that reference the moved blocks. The free (dropped) blocks are + /// reclaimed, the cache is rebuilt from the compacted file and the free-list is cleared. + /// + public Dictionary Compact(IReadOnlyCollection activeOffsets) + { + EnsureLoaded(); + + var tempPath = _filePath + ".compact.tmp"; + try + { + if (File.Exists(tempPath)) + { + File.Delete(tempPath); + } + + var mapping = new Dictionary(activeOffsets.Count); + var newCache = new Dictionary(activeOffsets.Count); + foreach (var offset in activeOffsets) + { + if (_cache.TryGetValue(offset, out var payload)) + { + var newOffset = _storage.AppendBytes(tempPath, payload); + mapping[offset] = newOffset; + newCache[newOffset] = payload; + } + } + + if (File.Exists(_filePath)) + { + File.Delete(_filePath); + } + + if (newCache.Count > 0) + { + File.Move(tempPath, _filePath); + } + + _cache.Clear(); + foreach (var (newOffset, payload) in newCache) + { + _cache[newOffset] = payload; + } + + _freeByLength.Clear(); // freed blocks were dropped by the compact pass + _loaded = true; + return mapping; + } + catch + { + try { if (File.Exists(tempPath)) File.Delete(tempPath); } catch { /* best effort */ } + throw; + } + } + + /// + public void Dispose() + { + _cache.Clear(); + _freeByLength.Clear(); + } +} diff --git a/src/SharpCoreDB/DataStructures/Table.BatchUpdate.cs b/src/SharpCoreDB/DataStructures/Table.BatchUpdate.cs index 37e1a3d6..5e8ccfbf 100644 --- a/src/SharpCoreDB/DataStructures/Table.BatchUpdate.cs +++ b/src/SharpCoreDB/DataStructures/Table.BatchUpdate.cs @@ -113,7 +113,7 @@ public int UpdateBatch( string? newPkValue = PrimaryKeyIndex >= 0 ? row[Columns[PrimaryKeyIndex]]?.ToString() : null; - RepointIndexesAfterRelocation(updatedPos, oldPkValue, newPkValue); + RepointIndexesAfterRelocation(oldPos, updatedPos, oldPkValue, newPkValue); } updatedCount++; @@ -351,7 +351,7 @@ private int UpdateBatchViaPrimaryKeyLookup( string? newPkValue = PrimaryKeyIndex >= 0 ? row[Columns[PrimaryKeyIndex]]?.ToString() : null; - RepointIndexesAfterRelocation(updatedPos, oldPkValue, newPkValue); + RepointIndexesAfterRelocation(pos, updatedPos, oldPkValue, newPkValue); } updatedCount++; @@ -493,7 +493,7 @@ private int UpdateBatchViaBulkSelect( string? newPkValue = PrimaryKeyIndex >= 0 ? row[Columns[PrimaryKeyIndex]]?.ToString() : null; - RepointIndexesAfterRelocation(updatedPos, oldPkValue, newPkValue); + RepointIndexesAfterRelocation(pos, updatedPos, oldPkValue, newPkValue); } totalUpdated++; @@ -745,7 +745,7 @@ private int UpdateBatchMultiColumnViaPrimaryKey( string? newPkValue = PrimaryKeyIndex >= 0 ? row[Columns[PrimaryKeyIndex]]?.ToString() : null; - RepointIndexesAfterRelocation(updatedPos, oldPkValue, newPkValue); + RepointIndexesAfterRelocation(pos, updatedPos, oldPkValue, newPkValue); } updatedCount++; @@ -886,7 +886,7 @@ private int UpdateBatchMultiColumnViaBulkSelect( string? newPkValue = PrimaryKeyIndex >= 0 ? row[Columns[PrimaryKeyIndex]]?.ToString() : null; - RepointIndexesAfterRelocation(updatedPos, oldPkValue, newPkValue); + RepointIndexesAfterRelocation(pos, updatedPos, oldPkValue, newPkValue); } totalUpdated++; diff --git a/src/SharpCoreDB/DataStructures/Table.BatchUpdateParallel.cs b/src/SharpCoreDB/DataStructures/Table.BatchUpdateParallel.cs index d7654004..72c262b5 100644 --- a/src/SharpCoreDB/DataStructures/Table.BatchUpdateParallel.cs +++ b/src/SharpCoreDB/DataStructures/Table.BatchUpdateParallel.cs @@ -187,7 +187,7 @@ private int UpdateBatchMultiColumnViaPrimaryKeyParallel( string? newPkValue = PrimaryKeyIndex >= 0 ? row[Columns[PrimaryKeyIndex]]?.ToString() : null; - RepointIndexesAfterRelocation(updatedPos, oldPkValue, newPkValue); + RepointIndexesAfterRelocation(pos, updatedPos, oldPkValue, newPkValue); } updatedCount++; diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index cb07c782..fdaee9fd 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -1291,6 +1291,11 @@ private List> ScanRowsWithSimdAndFilterStale(byte[] d // RTrim, Locale) require collation-aware comparison that only EvaluateWhere provides. int earlyWhereColIdx = -1; string? earlyWhereValue = null; + // B4: fixed-width tables use a constant slot offset + arena payload compare (pre-encoded + // UTF-8, Binary collation) — no per-record variable-length walk needed. + int earlyWhereSlotOffset = -1; + byte[]? earlyWhereUtf8 = null; + OverflowArena? earlyWhereArena = null; if (!string.IsNullOrEmpty(where) && TryParseSimpleWhereClause(where, out var ewCol, out var ewValObj) && ewValObj is string ewStr) @@ -1304,18 +1309,34 @@ private List> ScanRowsWithSimdAndFilterStale(byte[] d if (collation == CollationType.Binary) { - earlyWhereColIdx = idx; - earlyWhereValue = ewStr; + if (_fixedWidthRecords) + { + var fwLayout = GetFixedWidthLayout(); + if (idx < fwLayout.ColumnCount) + { + earlyWhereSlotOffset = fwLayout.Offsets[idx]; + earlyWhereValue = ewStr; + earlyWhereUtf8 = System.Text.Encoding.UTF8.GetBytes(ewStr); + earlyWhereArena = GetOverflowArena(); + } + } + else + { + earlyWhereColIdx = idx; + earlyWhereValue = ewStr; + } } } } // v2 (WP9-C): numeric early-WHERE — direct fixed-offset binary reads (no boxing/string // allocation), enabled for fixed-width numeric columns at a constant per-record offset. + // B4: also enabled for fixed-width tables — the layout provides the constant slot offset + // (null flag + raw payload), identical to the variable-length encoding for the offset path. int earlyNumericOffset = -1; DataType earlyNumericType = DataType.String; object? earlyNumericExpected = null; - if (earlyWhereColIdx < 0 && !string.IsNullOrEmpty(where) && + if (earlyWhereColIdx < 0 && earlyWhereSlotOffset < 0 && !string.IsNullOrEmpty(where) && TryParseSimpleWhereClause(where, out var ewCol2, out var ewVal2) && TryGetFixedNumericWhereInfo(ewCol2, out var ewOffset, out var ewType) && TryParseNumericExpected(ewVal2, ewType, out var ewExpected)) @@ -1331,7 +1352,7 @@ private List> ScanRowsWithSimdAndFilterStale(byte[] d while (filePosition < dataSpan.Length) { - // Read the 4-byte record length prefix + // Read length prefix (4 bytes) if (filePosition + 4 > dataSpan.Length) break; @@ -1373,6 +1394,15 @@ private List> ScanRowsWithSimdAndFilterStale(byte[] d continue; } } + else if (earlyWhereSlotOffset >= 0 && earlyWhereUtf8 is not null && earlyWhereArena is not null) + { + // B4: fixed-width string predicate — constant slot offset + arena payload compare. + if (!MatchesFixedWidthStringDirect(recordData, earlyWhereSlotOffset, earlyWhereArena, earlyWhereUtf8)) + { + filePosition += 4 + recordLength; + continue; + } + } else if (earlyWhereColIdx >= 0 && earlyWhereValue != null) { bool earlyMismatch = false; @@ -1460,13 +1490,23 @@ private List> ScanRowsWithSimdAndFilterStale(byte[] d /// /// Updates rows in the table that match the WHERE condition. /// Routes to storage engine with different semantics per mode: - /// - Columnar: Append new version (old becomes stale) - /// - PageBased: In-place update via engine.Update() + /// - Columnar: in-place overwrite when the new record fits (Issue #6), append otherwise + /// - PageBased: in-place update via engine.Update() + /// This entry point returns no count; see for the + /// single-pass variant that also reports the number of affected rows. /// /// Optional WHERE clause to filter rows. /// Dictionary of column names and new values. /// Thrown when table is readonly. - public void Update(string? where, Dictionary updates) + public void Update(string? where, Dictionary updates) => UpdateAffectedCount(where, updates); + + /// + /// Updates rows matching and returns the number of affected rows. + /// Single-pass variant used by the SQL UPDATE path so change-tracking no longer needs a + /// separate full Select pass (Issue #8: ExecuteUpdate previously materialized + /// every matching row just to count them). + /// + public int UpdateAffectedCount(string? where, Dictionary updates) { if (this.isReadOnly) throw new InvalidOperationException("Cannot update in readonly mode"); @@ -1474,20 +1514,31 @@ public void Update(string? where, Dictionary updates) try { var engine = GetOrCreateStorageEngine(); - // Use SelectInternal to preserve _rowid in results when it's the PK, - // so PK-based storage position lookups work correctly during update. - var rows = SelectInternal(where, orderBy: null, asc: true, noEncrypt: false); + // Load every registered hash index before the write loop: the append fallback leaves a + // stale record in the data file, and an unloaded index would later be rebuilt from the + // file INCLUDING that stale record (regression: stale row returned for the same PK). + EnsureAllRegisteredIndexesLoaded(); - foreach (var row in rows) + // Position-aware resolution: simple `pk = value` / hash-indexed WHEREs return the + // storage position too, so the write path can patch fields in place (fixed-width + // layout). Compound / range / unindexed WHEREs fall back to SelectInternal and resolve + // positions via the PK when present. + var rows = ResolveUpdateRows(where); + int affected = 0; + + foreach (var (rowPos, row) in rows) { - UpdateSingleRow(row, engine, updates); + affected++; + UpdateSingleRow(row, engine, updates, rowPos); } - // Auto-compact if the threshold was reached (only the append path creates stale versions). + // ✅ NEW: Auto-compact if threshold reached if (StorageMode == StorageMode.Columnar) { TryAutoCompact(); } + + return affected; } finally { @@ -1495,24 +1546,22 @@ public void Update(string? where, Dictionary updates) } } - private void UpdateSingleRow(Dictionary row, IStorageEngine engine, Dictionary updates) + private void UpdateSingleRow(Dictionary row, IStorageEngine engine, Dictionary updates, long rowPos) { // WP13: capture only what index maintenance needs instead of copying the // whole row (CASCADE is not wired in this path). - string? oldPkValue = null; - if (this.PrimaryKeyIndex >= 0 && row.TryGetValue(this.Columns[this.PrimaryKeyIndex], out var oldPk)) - { - oldPkValue = oldPk?.ToString(); - } + string? oldPkValue = this.PrimaryKeyIndex >= 0 + ? row[this.Columns[this.PrimaryKeyIndex]]?.ToString() + : null; // Snapshot old values of hash-indexed columns for key-only removal. Dictionary? oldHashKeys = null; - foreach (var hashColumn in this.hashIndexes.Keys) + foreach (var kvp in this.hashIndexes) { - if (row.TryGetValue(hashColumn, out var oldVal)) + if (row.TryGetValue(kvp.Key, out var oldVal)) { oldHashKeys ??= new Dictionary(); - oldHashKeys[hashColumn] = oldVal; + oldHashKeys[kvp.Key] = oldVal; } } @@ -1526,27 +1575,27 @@ private void UpdateSingleRow(Dictionary row, IStorageEngine engi if (StorageMode == StorageMode.Columnar) { - UpdateColumnarRow(row, engine, oldPkValue, oldHashKeys); + UpdateColumnarRow(row, engine, updates, oldPkValue, oldHashKeys, rowPos); } else { - UpdatePageBasedRow(row, engine, oldPkValue, oldHashKeys, updates); + UpdatePageBasedRow(row, engine, updates, oldPkValue, oldHashKeys); } } private void ValidateUpdatedRow(Dictionary row) { - // NOT NULL validation for UPDATE. + // ✅ NOT NULL validation for UPDATE for (int i = 0; i < this.Columns.Count; i++) { - // Bounds check for IsNotNull array. + // ✅ FIX: Bounds check for IsNotNull array if (i < this.IsNotNull.Count && this.IsNotNull[i] && (row[this.Columns[i]] == null || row[this.Columns[i]] == DBNull.Value)) { throw new InvalidOperationException($"Column '{this.Columns[i]}' cannot be NULL"); } } - // Column-level CHECK constraint validation for UPDATE. + // ✅ CHECK constraint validation for UPDATE for (int i = 0; i < this.Columns.Count; i++) { if (i < this.ColumnCheckExpressions.Count && this.ColumnCheckExpressions[i] is not null @@ -1556,7 +1605,7 @@ private void ValidateUpdatedRow(Dictionary row) } } - // Table-level CHECK constraints for UPDATE. + // Table-level CHECK constraints for UPDATE foreach (var checkExpr in this.TableCheckConstraints) { if (!TypeConverter.EvaluateCheckConstraint(checkExpr, row, this.ColumnTypes)) @@ -1566,32 +1615,39 @@ private void ValidateUpdatedRow(Dictionary row) } } - private void UpdateColumnarRow(Dictionary row, IStorageEngine engine, string? oldPkValue, Dictionary? oldHashKeys) + private void UpdateColumnarRow(Dictionary row, IStorageEngine engine, Dictionary updates, string? oldPkValue, Dictionary? oldHashKeys, long rowPos) { - // WP13: exact-size allocation - no ArrayPool.Rent + ToArray double allocation. - var rowData = SerializeRowExact(row); - - // Get old position from primary key index. - long oldPosition = -1; - if (this.PrimaryKeyIndex >= 0) + // Fixed-width layout step: when the row's existing bytes can be located, patch + // only the updated fields at their actual offsets (avoiding a full serialize + // and re-serialize round trip and a full string re-encoding). A fixed-size field keeps + // the record length unchanged, so the write is an in-place overwrite (Issue #6) + // and the file does not grow. Falls back to full serialization when a field + // cannot be patched in place (e.g. a variable-length field that changes size). + byte[] rowData; + if (rowPos >= 0) { - var pkVal = oldPkValue ?? string.Empty; - var searchResult = this.Index.Search(pkVal); - if (searchResult.Found) - { - oldPosition = searchResult.Value; - } + var existingData = engine.Read(Name, rowPos); + rowData = existingData is { Length: > 0 } + && (_fixedWidthRecords + ? TryOverwriteFixedWidthInPlace(existingData, updates) + : TryOverwriteFieldsInPlaceActual(existingData, updates)) is { } patched + ? patched + : SerializeRowExact(row); + } + else + { + rowData = SerializeRowExact(row); } // Issue #6: in-place UPDATE — overwrite the record in its existing slot when // the new record fits (fixed-width rows, or variable-width rows whose stored // length is unchanged). No new version is appended, the storage reference and // the PK index stay valid, and no stale version is left for compaction. - if (oldPosition >= 0 && engine.TryUpdateInPlace(Name, oldPosition, rowData)) + if (rowPos >= 0 && engine.TryUpdateInPlace(Name, rowPos, rowData)) { // Position unchanged: move hash entries in place (values may have changed). - MoveHashIndexesInPlace(row, oldHashKeys, oldPosition); - RepointPrimaryKeyIfChanged(row, oldPkValue, oldPosition); + MoveHashIndexesInPlace(row, oldHashKeys, rowPos); + RepointPrimaryKeyIfChanged(row, oldPkValue, rowPos); } else { @@ -1606,20 +1662,20 @@ private void UpdateColumnarRow(Dictionary row, IStorageEngine en foreach (var kvp in this.hashIndexes) { - if (oldPosition >= 0 && oldHashKeys != null && oldHashKeys.TryGetValue(kvp.Key, out var oldKey)) + if (rowPos >= 0 && oldHashKeys != null && oldHashKeys.TryGetValue(kvp.Key, out var oldKey)) { - kvp.Value.Remove(oldKey, oldPosition); // Remove old ref + kvp.Value.Remove(oldKey, rowPos); // Remove old ref } kvp.Value.Add(row, newPosition); // Add new ref } - // Track updates for compaction (only the append path creates stale versions). + // ✅ Track updates for compaction (only the append path creates stale versions). Interlocked.Increment(ref _updatedRowCount); } } - private void UpdatePageBasedRow(Dictionary row, IStorageEngine engine, string? oldPkValue, Dictionary? oldHashKeys, Dictionary updates) + private void UpdatePageBasedRow(Dictionary row, IStorageEngine engine, Dictionary updates, string? oldPkValue, Dictionary? oldHashKeys) { // Page-based: In-place update (or relocation when the record grows). // WP11: overwrite only the updated fields at their cached fixed column @@ -1661,7 +1717,7 @@ private void UpdatePageBasedRow(Dictionary row, IStorageEngine e // Record was relocated to another page (growing record on a // full page): re-point the PK index and rebuild hash indexes. var newPkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty; - RepointIndexesAfterRelocation(newPosition, pkVal, newPkVal); + RepointIndexesAfterRelocation(position, newPosition, pkVal, newPkVal); } else { @@ -1707,6 +1763,100 @@ private void RepointPrimaryKeyIfChanged(Dictionary row, string? this.Index.Insert(newPkVal, position); } } + /// + /// Resolves the rows to update as (storage position, row) pairs. A simple pk = value + /// WHERE is resolved through the primary-key B-tree directly (single search + one read); a + /// simple col = value WHERE on an indexed binary-collation column resolves through the + /// hash index. Everything else falls back to (positions resolved + /// via the PK when present). The position lets the columnar write path patch fields in place + /// (fixed-width layout) instead of appending a new version. + /// + private List<(long Position, Dictionary Row)> ResolveUpdateRows(string? where) + { + var engine = GetOrCreateStorageEngine(); + var result = new List<(long, Dictionary)>(); + + // Issue #7 fast path: simple `pk = value` — single search + one read. + if (StorageMode != StorageMode.PageBased && + this.PrimaryKeyIndex >= 0 && + !string.IsNullOrEmpty(where) && + TryParseSimpleWhereClause(where, out var pkCol, out var pkVal) && + string.Equals(pkCol, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + { + var sr = this.Index.Search(pkVal?.ToString() ?? string.Empty); + if (sr.Found) + { + var data = engine.Read(Name, sr.Value); + if (data != null) + { + var row = DeserializeRowFromSpan(data); + if (row != null) + { + result.Add((sr.Value, row)); + } + } + } + + return result; + } + + // Hash index fast path for a simple equality on an indexed binary-collation column. + if (!string.IsNullOrEmpty(where) && + TryParseSimpleWhereClause(where, out var whereCol, out var whereVal) && + this.registeredIndexes.ContainsKey(whereCol)) + { + var colIdx = this.Columns.IndexOf(whereCol); + var collation = colIdx >= 0 && colIdx < this.ColumnCollations.Count + ? this.ColumnCollations[colIdx] + : CollationType.Binary; + + if (collation == CollationType.Binary) + { + EnsureIndexLoaded(whereCol); + if (this.hashIndexes.TryGetValue(whereCol, out var hashIndex) && colIdx >= 0) + { + var key = ParseValueForHashLookup(whereVal?.ToString() ?? string.Empty, this.ColumnTypes[colIdx]); + if (key != null) + { + foreach (var pos in hashIndex.LookupPositions(key)) + { + var data = engine.Read(Name, pos); + if (data != null) + { + var row = DeserializeRowFromSpan(data); + if (row != null) result.Add((pos, row)); + } + } + + return result; + } + } + } + } + + // Fallback: full SELECT (compound/range WHERE or no usable index). Resolve positions via + // the PK when present so the write path can still attempt an in-place update. + var rows = SelectInternal(where, orderBy: null, asc: true, noEncrypt: false); + foreach (var row in rows) + { + long position = -1; + if (this.PrimaryKeyIndex >= 0 && + row.TryGetValue(this.Columns[this.PrimaryKeyIndex], out var pkValue) && + pkValue != null) + { + var sr = this.Index.Search(pkValue.ToString() ?? string.Empty); + if (sr.Found) + { + position = sr.Value; + } + } + + result.Add((position, row)); + } + + return result; + } + /// /// Re-points indexes after the storage engine relocated a record to another page /// (a growing record on a full page). The PK index is re-pointed precisely; hash @@ -1716,7 +1866,7 @@ private void RepointPrimaryKeyIfChanged(Dictionary row, string? /// The storage position after relocation. /// The PK value before the update (may be null if the table has no PK). /// The PK value after the update (may be null if the table has no PK). - private void RepointIndexesAfterRelocation(long newPosition, string? oldPkValue, string? newPkValue) + private void RepointIndexesAfterRelocation(long oldPosition, long newPosition, string? oldPkValue, string? newPkValue) { if (this.PrimaryKeyIndex >= 0) { @@ -1756,16 +1906,73 @@ internal void UpdateMultiple(List<(string where, Dictionary upda try { var engine = GetOrCreateStorageEngine(); - int updatedInBatch = 0; + // Load every registered hash index before the write loop so the append fallback can + // remove the stale record from all indexes (unloaded indexes would later be rebuilt + // from the file INCLUDING the stale record). + EnsureAllRegisteredIndexesLoaded(); + int appendedInBatch = 0; // only appends create stale versions that need compaction foreach (var (where, updates) in operations) { - // Resolve matching rows — prefer hash index point-lookup. - // Carry the storage position + raw bytes so the per-row fast path can - // overwrite updated fields in place instead of re-serializing the row. - List<(long pos, byte[]? data, Dictionary row)>? rows = null; + // B7: when the operation only touches non-indexed, non-PK columns on a table + // without CHECK constraints, matching rows are patched directly on their raw bytes + // (only the changed fields at their actual slot offsets) — no full-row + // deserialization. This is the hot path for + // `UPDATE t SET score = ... WHERE indexed_col = ...`. + bool fastPatch = StorageMode == StorageMode.Columnar && + !string.IsNullOrEmpty(where) && + TryParseSimpleWhereClause(where, out var fastWhereCol, out _) && + !updates.ContainsKey(fastWhereCol) && + (this.PrimaryKeyIndex < 0 || !updates.ContainsKey(this.Columns[this.PrimaryKeyIndex])) && + this.TableCheckConstraints.Count == 0 && + !HasColumnCheckConstraints(); + + // The raw-byte patch writes the record in place without re-pointing hash indexes, so + // when the update touches a hash-indexed column those entries must be re-pointed + // explicitly (old key removed, new key added at the same position) after the write. + bool touchesHashIndexedColumn = false; + if (this.hashIndexes.Count > 0) + { + foreach (var updateKey in updates.Keys) + { + if (this.hashIndexes.ContainsKey(updateKey)) + { + touchesHashIndexedColumn = true; + break; + } + } + } - if (!string.IsNullOrEmpty(where) && + // Resolve matching rows as (storage position, row, raw bytes) so the columnar + // write path can patch fields in place even when the table has no primary key. + // The position comes from the hash index / PK lookup already performed here; in + // fast-patch mode the raw record bytes are kept instead of a deserialized row. + List<(long Position, Dictionary? Row, byte[]? Raw)>? rows = null; + + // Issue #7/#8 fast path (mirrors CollectDeleteRecords): a simple `pk = value` WHERE + // on a columnar table with a PK resolves through the PK B-tree directly (single + // search + one read) instead of SelectInternal full-row materialization. When the + // key is not found the generic machinery below still runs. + if (StorageMode != StorageMode.PageBased && + this.PrimaryKeyIndex >= 0 && + !string.IsNullOrEmpty(where) && + TryParseSimpleWhereClause(where, out var pkWhereCol, out var pkWhereVal) && + string.Equals(pkWhereCol, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + { + var fastSearch = this.Index.Search(pkWhereVal?.ToString() ?? string.Empty); + if (fastSearch.Found) + { + var fastData = engine.Read(Name, fastSearch.Value); + if (fastData != null) + { + rows = fastPatch + ? [(fastSearch.Value, null, fastData)] + : [(fastSearch.Value, DeserializeRow(fastData), null)]; + } + } + } + + if (rows is null && !string.IsNullOrEmpty(where) && TryParseSimpleWhereClause(where, out var whereCol, out var whereVal) && this.registeredIndexes.ContainsKey(whereCol)) { @@ -1787,8 +1994,15 @@ internal void UpdateMultiple(List<(string where, Dictionary upda var data = engine.Read(Name, pos); if (data != null) { - var row = DeserializeRow(data); - if (row != null) rows.Add((pos, data, row)); + if (fastPatch) + { + rows.Add((pos, null, data)); + } + else + { + var row = DeserializeRow(data); + if (row != null) rows.Add((pos, row, null)); + } } } } @@ -1798,14 +2012,87 @@ internal void UpdateMultiple(List<(string where, Dictionary upda if (rows is null) { - var scanned = SelectInternal(where, orderBy: null, asc: true, noEncrypt: false); - rows = new List<(long, byte[]?, Dictionary)>(scanned.Count); - foreach (var r in scanned) - rows.Add((-1, null, r)); + rows = []; + foreach (var row in SelectInternal(where, orderBy: null, asc: true, noEncrypt: false)) + { + long position = -1; + if (this.PrimaryKeyIndex >= 0 && + row.TryGetValue(this.Columns[this.PrimaryKeyIndex], out var pkValue) && + pkValue != null) + { + var sr = this.Index.Search(pkValue.ToString() ?? string.Empty); + if (sr.Found) + { + position = sr.Value; + } + } + + rows.Add((position, row, null)); + } } - foreach (var (storagePos, data, row) in rows) + foreach (var (rowPosition, resolvedRow, rawData) in rows) { + // B7: fast patch — overwrite only the changed fields at their slot offsets in + // the existing record bytes (no full-row deserialization). The in-place write + // keeps the storage position, and since no indexed / PK column is touched the + // index entries stay valid. + Dictionary? row = resolvedRow; + if (fastPatch && rowPosition >= 0 && rawData is { Length: > 0 }) + { + // NOT NULL validation on the changed values only. + for (int i = 0; i < this.Columns.Count; i++) + { + if (i < this.IsNotNull.Count && this.IsNotNull[i] && + updates.TryGetValue(this.Columns[i], out var newVal) && + (newVal == null || newVal == DBNull.Value)) + { + throw new InvalidOperationException($"Column '{this.Columns[i]}' cannot be NULL"); + } + } + + byte[]? patched = _fixedWidthRecords + ? TryOverwriteFixedWidthInPlace(rawData, updates) + : TryOverwriteFieldsInPlaceActual(rawData, updates); + + if (patched is not null && engine.TryUpdateInPlaceSameLength(Name, rowPosition, patched)) + { + // The record was overwritten in place; when the update changed a + // hash-indexed column, re-point its entries (old key decoded from the + // pre-write row bytes, new key added at the same position). Non-indexed + // updates skip this entirely. + if (touchesHashIndexedColumn) + { + var oldRow = DeserializeRow(rawData); + if (oldRow is not null) + { + foreach (var (colName, hashIdx) in this.hashIndexes) + { + if (!updates.TryGetValue(colName, out var newVal) || newVal is null) + { + continue; + } + + if (oldRow.TryGetValue(colName, out var oldVal) && oldVal is not null) + { + hashIdx.Remove(oldVal, rowPosition); + } + + hashIdx.Add(newVal, rowPosition); + } + } + } + + continue; + } + + // The patch did not fit (variable-length growth) → full-row fallback below. + row = DeserializeRow(rawData); + if (row is null) continue; + } + + row ??= resolvedRow; + if (row is null) continue; // v2: capture the old PK and indexed-column values BEFORE applying updates, // avoiding a full row dictionary copy per row (WP3 allocation reduction). object? oldPkValue = null; @@ -1857,17 +2144,13 @@ internal void UpdateMultiple(List<(string where, Dictionary upda } } - // Serialize (WP13: exact-size allocation, no pool + copy). - // Fast path: overwrite the updated fields directly in the existing bytes - // (no full deserialize→mutate→re-serialize round trip); fall back to full - // serialization when a field cannot be patched in place. - var rowData = data is not null && TryOverwriteFieldsInPlace(data, updates) is { } patched - ? patched - : SerializeRowExact(row); + // Serialize (WP13: exact-size allocation, no pool + copy). The columnar + // branch patches only the updated fields at their actual offsets instead. + byte[] rowData; if (StorageMode == StorageMode.Columnar) { - long oldPosition = storagePos; + long oldPosition = rowPosition; if (oldPosition < 0 && this.PrimaryKeyIndex >= 0) { var pkVal = oldPkValue?.ToString() ?? string.Empty; @@ -1876,6 +2159,27 @@ internal void UpdateMultiple(List<(string where, Dictionary upda oldPosition = searchResult.Value; } + // Fixed-width layout step: patch only the updated fields at their actual + // offsets in the existing record (no deserialize → mutate → re-serialize + // round trip). A fixed-size field keeps the record length unchanged, so + // the write is an in-place overwrite (Issue #6) and the file does not + // grow. Falls back to full serialization when a field cannot be patched. + if (oldPosition >= 0) + { + var existingData = engine.Read(Name, oldPosition); + rowData = existingData is { Length: > 0 } + && (_fixedWidthRecords + ? TryOverwriteFixedWidthInPlace(existingData, updates) + : TryOverwriteFieldsInPlaceActual(existingData, updates)) is { } patched + ? patched + : SerializeRowExact(row); + } + else + { + // Serialize (WP13: exact-size allocation, no pool + copy) + rowData = SerializeRowExact(row); + } + // Issue #6: in-place UPDATE — overwrite the record in its existing slot // when the new record fits; the storage reference and PK index stay valid. if (oldPosition >= 0 && engine.TryUpdateInPlace(Name, oldPosition, rowData)) @@ -1901,15 +2205,10 @@ oldHashValues is not null && var newPkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty; if (!string.Equals(newPkVal, oldPkValue?.ToString(), StringComparison.Ordinal)) { - var oldPkStr = oldPkValue?.ToString(); - if (!string.IsNullOrEmpty(oldPkStr)) - { - this.Index.Delete(oldPkStr); - } + if (!string.IsNullOrEmpty(oldPkValue?.ToString())) + this.Index.Delete(oldPkValue!.ToString()!); if (!string.IsNullOrEmpty(newPkVal)) - { this.Index.Insert(newPkVal, oldPosition); - } } } } @@ -1937,45 +2236,51 @@ oldHashValues is not null && if (row.TryGetValue(hashIndex.Key, out var newKey) && newKey is not null) hashIndex.Value.Add(newKey, newPosition); } - } - updatedInBatch++; + appendedInBatch++; // append fallback: stale version left for compaction + } } else // PageBased { - if (this.PrimaryKeyIndex >= 0) + rowData = SerializeRowExact(row); + + long position = rowPosition; + string? pkVal = this.PrimaryKeyIndex >= 0 ? oldPkValue?.ToString() : null; + if (position < 0 && this.PrimaryKeyIndex >= 0) { - var pkVal = oldPkValue?.ToString() ?? string.Empty; + pkVal = oldPkValue?.ToString() ?? string.Empty; var searchResult = this.Index.Search(pkVal); if (searchResult.Found) - { - long position = searchResult.Value; - long newPosition = engine.Update(Name, position, rowData); + position = searchResult.Value; + } - if (newPosition != position) - { - // Record was relocated to another page: re-point the PK - // index and rebuild hash indexes lazily. - var newPkVal = row.TryGetValue(this.Columns[this.PrimaryKeyIndex], out var newPk) - ? newPk?.ToString() ?? string.Empty - : string.Empty; - RepointIndexesAfterRelocation(newPosition, pkVal, newPkVal); - } - else + if (position >= 0) + { + long newPosition = engine.Update(Name, position, rowData); + + if (newPosition != position) + { + // Record was relocated to another page: re-point the PK + // index and rebuild hash indexes lazily. + var newPkVal = row.TryGetValue(this.Columns[this.PrimaryKeyIndex], out var newPk) + ? newPk?.ToString() ?? string.Empty + : string.Empty; + RepointIndexesAfterRelocation(position, newPosition, pkVal, newPkVal); + } + else + { + // In-place update keeps the position; move hash entries in place. + foreach (var hashIndex in this.hashIndexes) { - // In-place update keeps the position; move hash entries in place. - foreach (var hashIndex in this.hashIndexes) + if (oldHashValues is not null && + oldHashValues.TryGetValue(hashIndex.Key, out var oldKey) && + oldKey is not null) { - if (oldHashValues is not null && - oldHashValues.TryGetValue(hashIndex.Key, out var oldKey) && - oldKey is not null) - { - hashIndex.Value.Remove(oldKey, position); - } - - if (row.TryGetValue(hashIndex.Key, out var newKey) && newKey is not null) - hashIndex.Value.Add(newKey, position); + hashIndex.Value.Remove(oldKey, position); } + + if (row.TryGetValue(hashIndex.Key, out var newKey) && newKey is not null) + hashIndex.Value.Add(newKey, position); } } } @@ -1983,9 +2288,9 @@ oldHashValues is not null && } } - if (StorageMode == StorageMode.Columnar && updatedInBatch > 0) + if (StorageMode == StorageMode.Columnar && appendedInBatch > 0) { - Interlocked.Add(ref _updatedRowCount, updatedInBatch); + Interlocked.Add(ref _updatedRowCount, appendedInBatch); TryAutoCompact(); } } @@ -1995,6 +2300,47 @@ oldHashValues is not null && } } + /// + /// True when any column carries a CHECK expression (the batch fast-patch path is disabled in + /// that case because a CHECK may read non-updated columns). + /// + private bool HasColumnCheckConstraints() + { + var expressions = this.ColumnCheckExpressions; + if (expressions is null || expressions.Count == 0) + { + return false; + } + + foreach (var expr in expressions) + { + if (expr is not null) + { + return true; + } + } + + return false; + } + + /// + /// True when the column has an index created explicitly via CREATE INDEX (the + /// index-name → column map is only populated for named indexes). Used to gate the direct + /// hash-index point lookup on trusted, fully-maintained indexes. + /// + private bool HasExplicitNamedIndex(string column) + { + foreach (var (_, indexedColumn) in this.indexNameToColumn) + { + if (string.Equals(indexedColumn, column, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + /// /// WP12: shared delete core used by every delete path (, DeleteMultiple, /// ). Performs physical engine deletes, primary-key B-tree @@ -2016,7 +2362,17 @@ private void DeleteRecordsCore(List<(long storagePosition, Dictionary= 0) + { + var pkCol = this.Columns[this.PrimaryKeyIndex]; + foreach (var (_, row) in recordsToDelete) + { + if (row.TryGetValue(pkCol, out var pkValue) && pkValue != null) + { + this.Index.Delete(pkValue.ToString() ?? string.Empty); + } + } + } // Key-only hash-index cleanup: extract each indexed column's key once per row and // remove all positions in a single lock per index. @@ -2026,35 +2382,6 @@ private void DeleteRecordsCore(List<(long storagePosition, Dictionary row)> recordsToDelete) - { - if (this.PrimaryKeyIndex < 0) - return; - - var pkCol = this.Columns[this.PrimaryKeyIndex]; - foreach (var (_, row) in recordsToDelete) - { - if (row.TryGetValue(pkCol, out var pkValue) && pkValue != null) - { - this.Index.Delete(pkValue.ToString() ?? string.Empty); - } - } - } - - private void RemoveHashIndexKeys(List<(long storagePosition, Dictionary row)> recordsToDelete, long[] positions) - { foreach (var kvp in this.hashIndexes) { if (!this.loadedIndexes.Contains(kvp.Key)) @@ -2068,18 +2395,23 @@ private void RemoveHashIndexKeys(List<(long storagePosition, Dictionary @@ -2092,24 +2424,119 @@ private void MarkUnloadedIndexesStale() /// Optional WHERE clause to filter rows to delete. /// Thrown when table is readonly. public void Delete(string? where) + { + DeleteAffected(where); + } + + /// + /// Deletes rows matching and returns the number of affected rows. + /// Issue #7: a simple `pk = value` WHERE is resolved through the primary-key index directly + /// (single search + one read) instead of going through , which + /// deserialized the full row set only to re-search the index for every row. The SQL DELETE + /// path also previously materialized matching rows twice (once in ExecuteDelete and once + /// here); callers use this method to delete once and get the affected count for free. + /// + public int DeleteAffected(string? where) { if (this.isReadOnly) throw new InvalidOperationException("Cannot delete in readonly mode"); this.rwLock.EnterWriteLock(); try { - var engine = GetOrCreateStorageEngine(); + var records = CollectDeleteRecords(where); + DeleteRecordsCore(records); + return records.Count; + } + finally + { + this.rwLock.ExitWriteLock(); + } + } - // ✅ OPTIMIZATION: Snapshot-based deletion (Option 1) - // Capture ALL storage references BEFORE any deletions - // This prevents mid-scan invalidation and eliminates exception overhead - // Performance: 50-70% faster for batch deletes, single table scan + /// + /// Deletes rows matching and returns the affected (pre-delete) rows. + /// Single-pass version of used by the SQL DELETE path so RETURNING + /// + affected-count no longer need a separate full Select pass (Issue #8: the SQL + /// DELETE path previously materialized matching rows twice — once in ExecuteDelete and once in + /// ). The returned rows are the exact rows that were deleted. + /// + public List> DeleteAffectedRows(string? where) + { + if (this.isReadOnly) throw new InvalidOperationException("Cannot delete in readonly mode"); - var recordsToDelete = new List<(long storagePosition, Dictionary row)>(); + this.rwLock.EnterWriteLock(); + try + { + var records = CollectDeleteRecords(where); + DeleteRecordsCore(records); - if (StorageMode == StorageMode.PageBased) + var rows = new List>(records.Count); + foreach (var (_, row) in records) { - // PageBased: Collect storage references upfront + rows.Add(row); + } + + return rows; + } + finally + { + this.rwLock.ExitWriteLock(); + } + } + + /// + /// Collects the storage positions + rows to delete for without + /// deleting anything. Issue #7 fast path: a simple `pk = value` WHERE on a columnar table + /// with a PK is resolved via the primary-key B-tree directly (no SelectInternal, no full-row + /// materialization, no redundant re-search). When the key is not found the generic machinery + /// below runs (collation-aware evaluation may still match), so correctness is unchanged. + /// + private List<(long storagePosition, Dictionary row)> CollectDeleteRecords(string? where) + { + var engine = GetOrCreateStorageEngine(); + // Load every registered hash index before the delete so DeleteRecordsCore can remove the + // deleted positions from all of them (an unloaded index would later be rebuilt from the + // file INCLUDING the logically-deleted record, resurrecting it in hash lookups). + EnsureAllRegisteredIndexesLoaded(); + + // ✅ OPTIMIZATION: Snapshot-based deletion (Option 1) + // Capture ALL storage references BEFORE any deletions + // This prevents mid-scan invalidation and eliminates exception overhead + // Performance: 50-70% faster for batch deletes, single table scan + + var recordsToDelete = new List<(long storagePosition, Dictionary row)>(); + + // ✅ Issue #7 fast path: simple "pk = value" WHERE — the PK B-tree search is the complete + // resolution (a primary key has at most one row), so when it hits we skip everything below. + bool fastPathHit = false; + if (StorageMode != StorageMode.PageBased && this.PrimaryKeyIndex >= 0 && + TryParseSimpleWhereClause(where, out var fastCol, out var fastVal) && + string.Equals(fastCol, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + { + var searchResult = this.Index.Search(fastVal?.ToString() ?? string.Empty); + if (searchResult.Found) + { + var data = engine.Read(Name, searchResult.Value); + if (data != null) + { + var row = DeserializeRowFromSpan(data); + if (row != null) + { + recordsToDelete.Add((searchResult.Value, row)); + fastPathHit = true; + } + } + } + } + + if (fastPathHit) + { + return recordsToDelete; + } + + if (StorageMode == StorageMode.PageBased) + { + // PageBased: Collect storage references upfront foreach (var (storageRef, data) in engine.GetAllRecords(Name)) { var row = DeserializeRowFromSpan(data); @@ -2184,28 +2611,22 @@ public void Delete(string? where) } } - if (!scannedViaIndex) - { - // Full scan fallback (no index or compound WHERE clause) - foreach (var (storageRef, data) in engine.GetAllRecords(Name)) + if (!scannedViaIndex) { - var row = DeserializeRowFromSpan(data); - if (row != null && (string.IsNullOrEmpty(where) || EvaluateSimpleWhere(row, where))) + // Full scan fallback (no index or compound WHERE clause) + foreach (var (storageRef, data) in engine.GetAllRecords(Name)) { - recordsToDelete.Add((storageRef, row)); + var row = DeserializeRowFromSpan(data); + if (row != null && (string.IsNullOrEmpty(where) || EvaluateSimpleWhere(row, where))) + { + recordsToDelete.Add((storageRef, row)); + } } } } - } - // ✅ WP12: unified delete core - engine deletes, PK and key-only hash index cleanup. - DeleteRecordsCore(recordsToDelete); + return recordsToDelete; } - finally - { - this.rwLock.ExitWriteLock(); - } - } /// /// Deletes rows matching multiple WHERE conditions under a single write lock. @@ -2223,10 +2644,40 @@ internal void DeleteMultiple(List whereConditions) try { var engine = GetOrCreateStorageEngine(); + // Load every registered hash index before the delete loop (same reason as + // CollectDeleteRecords: stale file records must be removed from every index). + EnsureAllRegisteredIndexesLoaded(); var recordsToDelete = new List<(long storagePosition, Dictionary row)>(); foreach (var where in whereConditions) { + // Issue #7 fast path (mirrors CollectDeleteRecords): a simple `pk = value` WHERE on + // a columnar table with a PK is resolved via the PK B-tree directly (single search + + // one read) instead of SelectInternal (full-row materialization) + a per-row PK + // re-search. When the key is not found the generic machinery below still runs. + if (StorageMode != StorageMode.PageBased && + this.PrimaryKeyIndex >= 0 && + !string.IsNullOrEmpty(where) && + TryParseSimpleWhereClause(where, out var fastPkCol, out var fastPkVal) && + string.Equals(fastPkCol, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + { + var fastSearch = this.Index.Search(fastPkVal?.ToString() ?? string.Empty); + if (fastSearch.Found) + { + var fastData = engine.Read(Name, fastSearch.Value); + if (fastData != null) + { + var fastRow = DeserializeRowFromSpan(fastData); + if (fastRow != null) + { + recordsToDelete.Add((fastSearch.Value, fastRow)); + } + } + + continue; + } + } + // Try hash index fast path if (!string.IsNullOrEmpty(where) && TryParseSimpleWhereClause(where, out var col, out var val) && @@ -2372,6 +2823,92 @@ public List> FindByIndex(string column, object value) return results; } + /// + /// B8: direct hash-index point lookup for the simple-SELECT fast path + /// (SELECT … FROM t WHERE indexed_col = @param|literal). Bypasses the WHERE-string + /// round trip (build a string → parse it again → re-detect the index route). Mirrors + /// SelectInternal's indexed path: read lock + EnsureIndexLoaded + binary + /// collation only. Returns false when the caller must fall back to the full scan / WHERE + /// machinery (no usable index on the column, or a non-binary collation). + /// + internal bool TrySelectIndexedPointLookup(string column, object value, out List> results) + { + results = []; + + // The PK B-tree is authoritative for primary-key lookups (the PK hash index may be + // stale or not built yet); route PK columns through the legacy path. + if (this.PrimaryKeyIndex >= 0 && + string.Equals(column, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + // Only hash indexes created explicitly via CREATE INDEX are trusted for a direct + // point lookup. Auto-registered indexes (primary key / fixed-width layout) can be + // stale or incomplete; those columns fall back to SelectInternal, which probes the + // B-tree and full scan. + if (!HasExplicitNamedIndex(column)) + { + return false; + } + + // Upgradeable read lock, matching SelectWithLock: the first index load inside + // EnsureIndexLoaded upgrades to a write lock (a plain read lock would deadlock). + this.rwLock.EnterUpgradeableReadLock(); + try + { + if (!this.registeredIndexes.ContainsKey(column)) + { + return false; + } + + EnsureIndexLoaded(column); + if (!this.hashIndexes.TryGetValue(column, out var hashIndex)) + { + return false; + } + + var colIdx = this.Columns.IndexOf(column); + if (colIdx < 0) + { + return false; + } + + // The hash index is only used for binary collation (mirrors SelectInternal). + var collation = colIdx < this.ColumnCollations.Count ? this.ColumnCollations[colIdx] : CollationType.Binary; + if (collation != CollationType.Binary) + { + return false; + } + + var key = ParseValueForHashLookup(value?.ToString() ?? string.Empty, this.ColumnTypes[colIdx]); + if (key is null) + { + return true; // no matches — the lookup was handled + } + + var engine = GetOrCreateStorageEngine(); + foreach (var pos in hashIndex.LookupPositionsUnsafe(key)) + { + var data = engine.Read(Name, pos); + if (data != null) + { + var row = DeserializeRow(data); + if (row != null) + { + results.Add(row); + } + } + } + + return true; + } + finally + { + this.rwLock.ExitUpgradeableReadLock(); + } + } + internal bool TryGetConflictingUniquePrimaryKey( Dictionary row, List? conflictTargetColumns, @@ -2626,7 +3163,7 @@ public bool UpdateByPrimaryKey(object key, Dictionary updates) // Record was relocated to another page: re-point the PK index and // rebuild hash indexes lazily. var newPkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty; - RepointIndexesAfterRelocation(newPosition, pkStr, newPkVal); + RepointIndexesAfterRelocation(storagePosition, newPosition, pkStr, newPkVal); } else { diff --git a/src/SharpCoreDB/DataStructures/Table.Compaction.cs b/src/SharpCoreDB/DataStructures/Table.Compaction.cs index 2cbe85cb..ba4271d2 100644 --- a/src/SharpCoreDB/DataStructures/Table.Compaction.cs +++ b/src/SharpCoreDB/DataStructures/Table.Compaction.cs @@ -108,7 +108,12 @@ public CompactionStats CompactStorage() // Count rows before compaction var rowsBeforeCompaction = activePositions.Count; - + + // Fixed-width layout (B3): reclaim the overflow arena too — collect the live arena + // offsets from the current records, compact the .ovf, and re-point the records' variable + // slots in place (records are fixed-width, so re-pointing never changes their length). + CompactOverflowArena(activePositions); + // Perform compaction long bytesReclaimed = appendEngine.CompactTable(Name, activePositions); @@ -139,6 +144,67 @@ public CompactionStats CompactStorage() } } + /// + /// B3: copy-on-compact for the out-of-line overflow arena. Collects the arena offsets referenced + /// by the current (active) fixed-width records, compacts the .ovf, then re-points the + /// variable slots of the active records that referenced a moved block. Called before the data + /// file compaction so the rewritten records carry the new offsets. + /// + private void CompactOverflowArena(List activePositions) + { + if (!_fixedWidthRecords || activePositions.Count == 0) + { + return; + } + + var engine = GetOrCreateStorageEngine(); + var layout = GetFixedWidthLayout(); + var arena = GetOverflowArena(); + var liveOffsets = new HashSet(); + + foreach (var pos in activePositions) + { + var data = engine.Read(Name, pos); + if (data is not null) + { + CollectVariableOffsets(data, layout, liveOffsets); + } + } + + if (liveOffsets.Count == 0) + { + return; + } + + var mapping = arena.Compact(liveOffsets); + + foreach (var pos in activePositions) + { + var data = engine.Read(Name, pos); + if (data is null || data.Length != layout.FixedSize) + { + continue; + } + + var patched = RepointVariableSlots(data, layout, mapping); + if (patched is not null) + { + engine.TryUpdateInPlace(Name, pos, patched); + } + } + } + + /// Collects the overflow-block offsets referenced by a fixed-width record's variable slots. + private static void CollectVariableOffsets(byte[] record, FixedWidthRecordLayout layout, HashSet live) + => FixedWidthCodec.CollectVariableOffsets(record, layout, live); + + /// + /// Returns a copy of a fixed-width record with its variable slots re-pointed through the + /// compaction mapping, or null when no slot moved. + /// + private static byte[]? RepointVariableSlots(byte[] record, FixedWidthRecordLayout layout, Dictionary mapping) + => FixedWidthCodec.RepointVariableSlots(record, layout, mapping); + /// /// Rebuilds the primary key index after compaction. /// Positions change after compaction, so we need to rescan the file. diff --git a/src/SharpCoreDB/DataStructures/Table.FixedWidthMigration.cs b/src/SharpCoreDB/DataStructures/Table.FixedWidthMigration.cs new file mode 100644 index 00000000..3ea8ff23 --- /dev/null +++ b/src/SharpCoreDB/DataStructures/Table.FixedWidthMigration.cs @@ -0,0 +1,248 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.DataStructures; + +using SharpCoreDB.Storage.Hybrid; + +/// +/// B5: 1.x → 2.0 record-format migration. Converts a legacy table (variable-length records) to the +/// fixed-width record layout (out-of-line overflow arena): current rows are re-read through the +/// legacy codec, re-serialized as fixed-width records (variable values move into a fresh overflow +/// arena), and the primary-key / hash indexes are rebuilt on the new record positions. Page-based +/// tables are converted to Columnar storage in-process first. +/// +public partial class Table +{ + /// + /// Migrates this table from the legacy variable-length record format to the fixed-width record + /// layout. Returns the number of rows migrated (0 when the table is already fixed-width). + /// Requires a writable table; page-based tables are converted to Columnar storage first. + /// + public int MigrateToFixedWidth() + { + if (isReadOnly) + { + throw new InvalidOperationException("Cannot migrate a read-only table to the fixed-width record layout."); + } + + rwLock.EnterWriteLock(); + try + { + if (_fixedWidthRecords) + { + return 0; // already in the target format + } + + List> rows; + + if (StorageMode == StorageMode.PageBased) + { + // PageBased → Columnar conversion happens first (in-process). ScanPageBasedTable + // resolves the current rows without relying on the PK index. + rows = Select(); + ConvertToColumnarInPlace(); + } + else + { + if (StorageMode != StorageMode.Columnar) + { + throw new NotSupportedException( + $"Storage mode '{StorageMode}' cannot be migrated to the fixed-width record layout."); + } + + // B5 safety net: a table created with the fixed-width flag BEFORE the record format + // was persisted in metadata (B1–B4) is unmarked but already stores fixed-width + // records. Re-reading it as legacy would corrupt it, so adopt the format when the + // on-disk records provably match the fixed-width layout (constant length + variable + // slots resolve in the arena). Legacy records with fixed-size-only columns are + // byte-identical to fixed-width, so adopting is also correct for them. + if (RecordsMatchFixedWidthLayout()) + { + _fixedWidthRecords = true; + return 0; + } + + // Rebuild the PK index with the LEGACY codec so Select() filters stale versions + // correctly (the index may be empty right after metadata load). + if (PrimaryKeyIndex >= 0) + { + RebuildPrimaryKeyIndexFromDisk(); + } + + rows = Select(); + } + + // Switch the serializer to the fixed-width codec and start with a fresh arena. + var arenaPath = System.IO.Path.ChangeExtension(DataFile, ".ovf"); + if (File.Exists(arenaPath)) + { + File.Delete(arenaPath); + } + + _overflowArena = null; + _fixedWidthLayout = null; + _fixedWidthRecords = true; + + // 4. Re-serialize every row as a fixed-width record (variable values → fresh arena). + var records = new List(rows.Count); + foreach (var row in rows) + { + records.Add(SerializeRowFixedWidth(row)); + } + + // 5. Write the new records to a temp file and swap it in atomically (the same pattern + // as AppendOnlyEngine.CompactTable, so encryption handling is identical). + var tempPath = DataFile + ".fwmig.tmp"; + try + { + if (File.Exists(tempPath)) + { + File.Delete(tempPath); + } + + if (records.Count > 0) + { + storage.AppendBytesMultiple(tempPath, records); + } + else + { + File.WriteAllBytes(tempPath, Array.Empty()); + } + + File.Delete(DataFile); + File.Move(tempPath, DataFile); + } + catch + { + if (File.Exists(tempPath)) + { + try { File.Delete(tempPath); } catch { /* best-effort cleanup */ } + } + + throw; + } + + // 6. Rebuild the indexes against the new fixed-width records (DeserializeRow dispatches + // to the fixed-width codec now) and fix the cached row count. + RebuildPrimaryKeyIndex(); + foreach (var col in loadedIndexes.ToList()) + { + RebuildHashIndex(col); + } + + Interlocked.Exchange(ref _cachedRowCount, rows.Count); + + return rows.Count; + } + finally + { + rwLock.ExitWriteLock(); + } + } + + /// + /// Converts this table from page-based to columnar (append-only) storage in place: the + /// page-based engine and its .pages files are dropped, is set + /// to Columnar and switches to the .dat convention. The rows + /// themselves are written by the caller (the fixed-width rewrite). + /// + private void ConvertToColumnarInPlace() + { + // Dispose + drop the page-based engine (it owns the .pages files and their handles). + DisposeStorageEngine(); + + var pagesPath = DataFile; + var directory = System.IO.Path.GetDirectoryName(pagesPath) ?? "."; + var baseName = System.IO.Path.GetFileNameWithoutExtension(pagesPath); + + // The engine stores pages in table_{stableId}.pages (deterministic FNV-1a of the upper-cased + // table name) — the {name}.pages DDL file is just an empty placeholder. + uint stableTableId = ComputeStableTableId(Name); + foreach (var file in Directory.EnumerateFiles(directory, "*.pages")) + { + var fileName = System.IO.Path.GetFileName(file); + if (string.Equals(fileName, baseName + ".pages", System.StringComparison.OrdinalIgnoreCase) || + string.Equals(fileName, $"table_{stableTableId}.pages", System.StringComparison.OrdinalIgnoreCase)) + { + try { File.Delete(file); } catch { /* best-effort cleanup */ } + } + } + + StorageMode = StorageMode.Columnar; + DataFile = System.IO.Path.ChangeExtension(pagesPath, ".dat"); + } + + /// + /// Deterministic FNV-1a table id used by the page-based engine's file naming + /// (table_{id}.pages). Mirrors PageBasedEngine.ComputeStableTableId. + /// + private static uint ComputeStableTableId(string tableName) + { + const uint fnvOffset = 2166136261; + const uint fnvPrime = 16777619; + + uint hash = fnvOffset; + foreach (var b in System.Text.Encoding.UTF8.GetBytes(tableName.ToUpperInvariant())) + { + hash ^= b; + hash *= fnvPrime; + } + + return hash; + } + + /// + /// Probes the on-disk records to determine whether they already use the fixed-width layout. + /// Returns true when every record has exactly + /// bytes AND every non-NULL variable slot resolves to a block in the overflow arena. Legacy + /// records (variable-length strings/blobs) fail the length or the arena-resolution check, so + /// they never match; fixed-size-only legacy records are byte-identical and safely adopt. + /// + private bool RecordsMatchFixedWidthLayout() + { + var layout = GetFixedWidthLayout(); + var arena = GetOverflowArena(); + var engine = GetOrCreateStorageEngine(); + bool any = false; + + foreach (var (_, data) in engine.GetAllRecords(Name)) + { + any = true; + if (data is not { Length: var len } || len != layout.FixedSize) + { + return false; + } + + for (int i = 0; i < layout.ColumnCount; i++) + { + if (!layout.IsVariable[i]) + { + continue; + } + + var slot = layout.Offsets[i]; + if (slot + 5 > data.Length) + { + return false; + } + + if (data[slot] == 0) + { + continue; // NULL slot — valid in either format + } + + // The slot must be an arena offset, not a legacy string-length prefix. + var offset = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan(slot + 1, 4)); + if (arena.Read(offset) is null) + { + return false; + } + } + } + + return any; // true only when at least one record exists and all records match + } +} diff --git a/src/SharpCoreDB/DataStructures/Table.Indexing.cs b/src/SharpCoreDB/DataStructures/Table.Indexing.cs index d851e9fa..1f360017 100644 --- a/src/SharpCoreDB/DataStructures/Table.Indexing.cs +++ b/src/SharpCoreDB/DataStructures/Table.Indexing.cs @@ -269,6 +269,26 @@ public void EnsureIndexLoaded(string columnName) } } + /// + /// Loads every registered hash index up front so DML write paths (the append-only UPDATE + /// fallback and the DELETE index cleanup) can maintain them incrementally. An unloaded index + /// is rebuilt from the data file on next use — which, after an append update or logical + /// delete, still contains the stale record — so any write that creates stale versions must + /// ensure its registered indexes are loaded first. Cheap after the first load (cached). + /// + private void EnsureAllRegisteredIndexesLoaded() + { + if (this.registeredIndexes.Count == 0) + return; + + // Safe to iterate directly: the caller holds the write lock and EnsureIndexLoaded only + // mutates hashIndexes/loadedIndexes/staleIndexes, never the registeredIndexes registry. + foreach (var columnName in this.registeredIndexes.Keys) + { + EnsureIndexLoaded(columnName); + } + } + /// /// Checks if a hash index exists for the specified column. /// diff --git a/src/SharpCoreDB/DataStructures/Table.PageBasedScan.cs b/src/SharpCoreDB/DataStructures/Table.PageBasedScan.cs index 6c5cbe4b..08d9f6d7 100644 --- a/src/SharpCoreDB/DataStructures/Table.PageBasedScan.cs +++ b/src/SharpCoreDB/DataStructures/Table.PageBasedScan.cs @@ -114,6 +114,12 @@ private List> ScanPageBasedTable(string? where) /// private Dictionary? DeserializeRowFromSpan(byte[] data) { + // Fixed-width record layout (out-of-line overflow): variable slots reference the arena. + if (_fixedWidthRecords) + { + return data is { Length: > 0 } ? DeserializeRowFixedWidth(data.AsSpan()) : null; + } + if (data == null || data.Length == 0) { #if DEBUG diff --git a/src/SharpCoreDB/DataStructures/Table.Serialization.cs b/src/SharpCoreDB/DataStructures/Table.Serialization.cs index d608e4ba..15893b36 100644 --- a/src/SharpCoreDB/DataStructures/Table.Serialization.cs +++ b/src/SharpCoreDB/DataStructures/Table.Serialization.cs @@ -320,6 +320,269 @@ private bool AllUpdatedColumnsFit( /// Number of rows patched in place via (monitoring). public long TotalInPlacePatches => Interlocked.Read(ref _inPlacePatchCount); + /// + /// Fixed-width layout step 1: computes the actual per-column byte offsets in an existing + /// serialized row by walking the length-prefixed record (fixed-size columns contribute their + /// fixed encoded size; variable-length columns contribute 1 null flag + 4-byte length + + /// payload). Unlike , this resolves offsets AFTER a + /// variable-length column, so e.g. score in (name TEXT, email TEXT, age INT, score REAL) + /// can be patched in place even though its schema-level offset is "unstable". Returns null + /// when the record is corrupt / out of bounds — callers must fall back to full serialization. + /// + private int[]? ComputeActualColumnOffsets(byte[] row) + { + if (row is not { Length: > 0 }) + return null; + + var offsets = new int[Columns.Count]; + int offset = 0; + for (int i = 0; i < Columns.Count; i++) + { + if (offset >= row.Length) + return null; + + offsets[i] = offset; + + int fixedSize = GetFixedEncodedSize(ColumnTypes[i]); + if (fixedSize >= 0) + { + offset += fixedSize; + continue; + } + + // Variable-length: 1 null flag (+ 4-byte length + payload when not null). + if (row[offset] == 0) + { + offset += 1; + continue; + } + + if (offset + 5 > row.Length) + return null; + + int len = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(row.AsSpan(offset + 1, 4)); + if (len < 0 || offset + 5 + len > row.Length) + return null; + + offset += 5 + len; + } + + return offsets; + } + + /// + /// Fixed-width layout step 2: like but resolves the + /// updated column offsets from the actual record bytes (see ), + /// so fields after a variable-length column can also be patched in place. Returns null when any + /// updated field would change the record length (or the record is corrupt) — callers then fall + /// back to full serialization. + /// + private byte[]? TryOverwriteFieldsInPlaceActual(byte[] existingRow, Dictionary updates) + { + if (existingRow is not { Length: > 0 } || updates.Count == 0) + return null; + + var columnIndexCache = GetColumnIndexCache(); + var offsets = ComputeActualColumnOffsets(existingRow); + if (offsets is null) + return null; + + // Pass 1: every updated column must have a valid offset and fit in its existing slot. + foreach (var (column, value) in updates) + { + if (!columnIndexCache.TryGetValue(column, out int colIdx) || colIdx < 0 || colIdx >= offsets.Length) + return null; + + int offset = offsets[colIdx]; + if (offset < 0 || offset >= existingRow.Length) + return null; + + int newSize = GetEncodedSize(value, ColumnTypes[colIdx]); + int oldSize = ReadColumnEncodedSize(existingRow.AsSpan(), offset, ColumnTypes[colIdx]); + if (newSize > oldSize) + return null; + + // A variable-length field before the last column changes the byte position of every + // following column; overwriting it in place is only safe when its encoding keeps the + // exact same size. Fixed-size fields never change size, and the last column has no + // followers to shift. + if (GetFixedEncodedSize(ColumnTypes[colIdx]) < 0 && colIdx < Columns.Count - 1 && newSize != oldSize) + return null; + } + + // Pass 2: copy the row and overwrite only the updated fields. + var result = new byte[existingRow.Length]; + existingRow.CopyTo(result, 0); + var span = result.AsSpan(); + + foreach (var (column, value) in updates) + { + if (!columnIndexCache.TryGetValue(column, out int colIdx)) + return null; + + _ = WriteTypedValueToSpan(span.Slice(offsets[colIdx]), value, ColumnTypes[colIdx]); + } + + return result; + } + + #region Fixed-width record layout (out-of-line overflow, opt-in) + + private FixedWidthRecordLayout GetFixedWidthLayout() + { + _fixedWidthLayout ??= FixedWidthRecordLayout.Compute(ColumnTypes); + return _fixedWidthLayout; + } + + private OverflowArena GetOverflowArena() + { + if (_overflowArena is null) + { + var arenaPath = string.IsNullOrEmpty(DataFile) + ? System.IO.Path.ChangeExtension(Name + ".dat", ".ovf") + : System.IO.Path.ChangeExtension(DataFile, ".ovf"); + _overflowArena = new OverflowArena(storage, arenaPath); + + // B6: the free-list is in-memory, so a reopened arena treats every .ovf block as live. + // Derive the cross-session free-list from the records: free every block no fixed-width + // record references, so same-length value updates reuse the space in the new session. + RebuildOverflowArenaFreeListFromDisk(); + } + + return _overflowArena; + } + + /// + /// B6: rebuilds the overflow-arena free-list from disk after a reopen. Dead blocks (freed + /// within a session) stay physically in the .ovf until the next copy-on-compact, and the + /// in-memory free-list is per-session, so scanning the fixed-width records and freeing every + /// block no record references restores cross-session reuse without persisting the free-list. + /// + private void RebuildOverflowArenaFreeListFromDisk() + { + if (!_fixedWidthRecords || storage is null || _overflowArena is null || + string.IsNullOrEmpty(DataFile) || !File.Exists(DataFile)) + { + return; + } + + var layout = GetFixedWidthLayout(); + var live = new HashSet(); + foreach (var (_, data) in storage.ReadAllRecords(DataFile)) + { + if (data is { Length: > 0 }) + { + FixedWidthCodec.CollectVariableOffsets(data, layout, live); + } + } + + if (live.Count == 0) + { + return; + } + + foreach (var offset in _overflowArena.GetAllOffsets().ToList()) + { + if (!live.Contains(offset)) + { + _overflowArena.Free(offset); + } + } + } + + internal static byte[] EncodeVariablePayload(DataType type, object value) + { + return type switch + { + DataType.Blob => (byte[])value, + _ => System.Text.Encoding.UTF8.GetBytes(value?.ToString() ?? string.Empty), + }; + } + + internal static object DecodeVariablePayload(DataType type, byte[] payload) + { + return type switch + { + DataType.Blob => payload, + _ => System.Text.Encoding.UTF8.GetString(payload), + }; + } + + /// Serializes a row using the fixed-width record layout (variable values → overflow arena). + private byte[] SerializeRowFixedWidth(Dictionary row) + => FixedWidthCodec.SerializeRow(row, Columns, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena()); + + /// Deserializes a fixed-width record into a row dictionary (variable values read from the overflow arena). + private Dictionary DeserializeRowFixedWidth(ReadOnlySpan data) + => FixedWidthCodec.DeserializeRow(data, Columns, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena()); + + /// + /// Fixed-width in-place patch: overwrites only the updated slots in an existing fixed record + /// (variable values get a new overflow block and the slot offset is updated). The record length + /// is constant, so the patched record always fits — the write is an in-place overwrite (#6). + /// + private byte[]? TryOverwriteFixedWidthInPlace(byte[] existingRow, Dictionary updates) + { + if (existingRow.Length != GetFixedWidthLayout().FixedSize) + { + return null; + } + + var layout = GetFixedWidthLayout(); + var arena = GetOverflowArena(); + var columnIndexCache = GetColumnIndexCache(); + var result = new byte[existingRow.Length]; + existingRow.CopyTo(result, 0); + var span = result.AsSpan(); + + foreach (var (column, value) in updates) + { + if (!columnIndexCache.TryGetValue(column, out int colIdx) || colIdx < 0 || colIdx >= layout.ColumnCount) + { + return null; + } + + var slot = span.Slice(layout.Offsets[colIdx], layout.SlotSizes[colIdx]); + if (layout.IsVariable[colIdx]) + { + // B6: offset 0 is a VALID arena block (the first block's length prefix sits at 0), so + // -1 is the sentinel for "no block" (NULL slot) — a real offset 0 must be freed too, + // otherwise the first variable block leaks and the free-list cannot reuse it. + int oldOffset = slot[0] == 0 ? -1 : System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(slot[1..]); + if (value == null || value == DBNull.Value) + { + if (oldOffset >= 0) + { + arena.Free(oldOffset); + } + + slot[0] = 0; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(slot[1..], 0); + } + else + { + var payload = EncodeVariablePayload(ColumnTypes[colIdx], value); + var offset = arena.Write(payload); + if (oldOffset >= 0) + { + arena.Free(oldOffset); + } + + slot[0] = 1; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(slot[1..], (int)offset); + } + } + else + { + _ = WriteTypedValueToSpan(slot, value, ColumnTypes[colIdx]); + } + } + + return result; + } + + #endregion + /// /// WP13: computes the exact encoded size of a row so serialization can allocate the /// final array once (no ArrayPool.Rent + ToArray double allocation, no copy). @@ -365,6 +628,13 @@ private int ComputeExactRowSize(object[] values) [MethodImpl(MethodImplOptions.AggressiveOptimization)] private byte[] SerializeRowExact(Dictionary row) { + // Fixed-width record layout (out-of-line overflow): constant-size record, variable values + // stored in the table's overflow arena. + if (_fixedWidthRecords) + { + return SerializeRowFixedWidth(row); + } + byte[] buffer = new byte[ComputeExactRowSize(row)]; int bytesWritten = WriteRowOptimized(buffer.AsSpan(), row); return bytesWritten == buffer.Length @@ -751,7 +1021,7 @@ private int EstimateRowSize(Dictionary row) /// The data type of the value. /// Number of bytes written. [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private int WriteTypedValueToSpan(Span buffer, object value, DataType type) + internal static int WriteTypedValueToSpan(Span buffer, object value, DataType type) { if (value == DBNull.Value || value == null) { @@ -923,7 +1193,7 @@ private int WriteTypedValueToSpan(Span buffer, object value, DataType type /// Output: number of bytes consumed. /// The deserialized value. [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private object ReadTypedValueFromSpan(ReadOnlySpan buffer, DataType type, out int bytesRead) + internal static object ReadTypedValueFromSpan(ReadOnlySpan buffer, DataType type, out int bytesRead) { bytesRead = 1; @@ -1489,6 +1759,12 @@ private static bool TryCoerceValue(object value, DataType targetType, out object [MethodImpl(MethodImplOptions.AggressiveOptimization)] private Dictionary DeserializeRowWithSimd(ReadOnlySpan data) { + // Fixed-width record layout (out-of-line overflow): variable slots reference the arena. + if (_fixedWidthRecords) + { + return DeserializeRowFixedWidth(data); + } + if (data.IsEmpty) return new Dictionary(Columns.Count); diff --git a/src/SharpCoreDB/DataStructures/Table.StructScanning.cs b/src/SharpCoreDB/DataStructures/Table.StructScanning.cs index 925f9413..af4666d6 100644 --- a/src/SharpCoreDB/DataStructures/Table.StructScanning.cs +++ b/src/SharpCoreDB/DataStructures/Table.StructScanning.cs @@ -54,6 +54,21 @@ public partial class Table [MethodImpl(MethodImplOptions.AggressiveOptimization)] public IEnumerable ScanStructRows(bool enableCaching = false) { + // Fixed-width record layout (out-of-line overflow): the zero-alloc struct scan walks the + // variable-length record format, so fixed-width tables fall back to the dictionary path + // (correct, allocated). StructRow.FromDictionary is self-consistent (own bytes + schema). + if (_fixedWidthRecords) + { + var columns = Columns.ToArray(); + var types = ColumnTypes.ToArray(); + foreach (var row in Select()) + { + yield return StructRow.FromDictionary(row, columns, types); + } + + yield break; + } + // ✅ FIX: Validate upfront, then delegate to iterator methods ArgumentNullException.ThrowIfNull(this.storage); @@ -63,12 +78,18 @@ public IEnumerable ScanStructRows(bool enableCaching = false) if (this.StorageMode == StorageMode.Columnar) { // Columnar mode: Read entire file and iterate with position filtering - return ScanColumnarStructRowsInternal(schema, enableCaching); + foreach (var row in ScanColumnarStructRowsInternal(schema, enableCaching)) + { + yield return row; + } } else // PageBased { // PageBased mode: Use storage engine's GetAllRecords - return ScanPageBasedStructRowsInternal(schema, enableCaching); + foreach (var row in ScanPageBasedStructRowsInternal(schema, enableCaching)) + { + yield return row; + } } } @@ -172,7 +193,14 @@ private IEnumerable ScanStructRowsWhereCore(string? where, bool enabl private IEnumerable ScanStructRowsWhereCoreIterator(string? where, bool enableCaching) { - var schema = BuildVariableLengthSchema(); + // Fixed-width records: StructRow's variable-length schema can't walk the fixed-width + // format, so matched records are materialized through the dictionary path. The numeric-SIMD + // fast path below is still usable (raw constant-offset reads, no schema walk); anything else + // falls back to the arena-aware dictionary full scan (see ScanStructRows). + bool fixedWidth = _fixedWidthRecords; + string[]? fixedColumns = fixedWidth ? Columns.ToArray() : null; + DataType[]? fixedTypes = fixedWidth ? ColumnTypes.ToArray() : null; + var schema = fixedWidth ? default : BuildVariableLengthSchema(); var engine = GetOrCreateStorageEngine(); string? simpleColumn = null; @@ -180,20 +208,22 @@ private IEnumerable ScanStructRowsWhereCoreIterator(string? where, bo bool hasSimpleWhere = where is { Length: > 0 } && TryParseSimpleWhereClause(where, out simpleColumn, out simpleValue); - // Fast path 1: hash-index point lookup (mirrors SelectInternal). - var hashRows = TryScanHashIndex(simpleColumn, simpleValue, hasSimpleWhere, schema, engine, enableCaching); - if (hashRows is not null) + // Fast path 1: hash-index point lookup (mirrors SelectInternal). StructRow can only + // represent variable-length records, so fixed-width tables skip this path. + if (!fixedWidth && hasSimpleWhere && simpleColumn is not null && simpleValue is not null && + this.registeredIndexes.ContainsKey(simpleColumn)) { - foreach (var row in hashRows) + foreach (var row in ScanByHashIndexPoint(simpleColumn, simpleValue, schema, engine, enableCaching)) yield return row; yield break; } - // Fast path 2: primary-key lookup. - var pkRows = TryScanPrimaryKey(simpleColumn, simpleValue, hasSimpleWhere, schema, engine, enableCaching); - if (pkRows is not null) + // Fast path 2: primary-key lookup (variable-length layout only — StructRow schema walk). + if (!fixedWidth && hasSimpleWhere && simpleColumn is not null && simpleValue is not null && + this.PrimaryKeyIndex >= 0 && + string.Equals(simpleColumn, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) { - foreach (var row in pkRows) + foreach (var row in ScanByPrimaryKeyPoint(simpleValue, schema, engine, enableCaching)) yield return row; yield break; } @@ -201,67 +231,40 @@ private IEnumerable ScanStructRowsWhereCoreIterator(string? where, bo // Fast path 3: fixed-width numeric equality — SIMD batch filter over extracted values // (no deserialization, no boxing). Integer/Long use portable Vector; Real uses // direct per-record reads. - var numericRows = TryScanNumeric( - simpleColumn, simpleValue, hasSimpleWhere, schema, engine, enableCaching); - if (numericRows is not null) + if (hasSimpleWhere && simpleColumn is not null && simpleValue is not null && + TryGetFixedNumericWhereInfo(simpleColumn, out var numericOffset, out var numericType) && + TryParseNumericExpected(simpleValue, numericType, out var numericExpected)) { - foreach (var row in numericRows) + foreach (var row in ScanByNumericSimd( + numericOffset, numericType, numericExpected, schema, engine, enableCaching, + fixedWidth, fixedColumns, fixedTypes)) yield return row; yield break; } - // Fallback: full scan with a simple equality predicate (scalar, allocation-free per row). - foreach (var row in ScanStructRows(enableCaching)) + // Fixed-width fallback: arena-aware dictionary full scan (StructRow can't walk the format). + if (fixedWidth) { - bool matches = !hasSimpleWhere || simpleColumn is null || simpleValue is null || - MatchesSimpleWhere(row, schema, simpleColumn, simpleValue); - if (matches) + foreach (var row in Select(where)) { - yield return row; + yield return StructRow.FromDictionary(row, fixedColumns ?? [], fixedTypes ?? []); } - } - } - private IEnumerable? TryScanHashIndex( - string? simpleColumn, object? simpleValue, bool hasSimpleWhere, - VariableLengthSchema schema, IStorageEngine engine, bool enableCaching) - { - if (!hasSimpleWhere || simpleColumn is null || simpleValue is null || - !this.registeredIndexes.ContainsKey(simpleColumn)) - { - return null; + yield break; } - return ScanByHashIndexPoint(simpleColumn, simpleValue, schema, engine, enableCaching); - } - - private IEnumerable? TryScanPrimaryKey( - string? simpleColumn, object? simpleValue, bool hasSimpleWhere, - VariableLengthSchema schema, IStorageEngine engine, bool enableCaching) - { - if (!hasSimpleWhere || simpleColumn is null || simpleValue is null || - this.PrimaryKeyIndex < 0 || - !string.Equals(simpleColumn, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + // Fallback: full scan with a simple equality predicate (scalar, allocation-free per row). + // NOSONAR:S3267 - intentional: LINQ Where would allocate per row on the scan hot path. + foreach (var row in ScanStructRows(enableCaching)) { - return null; + if (!hasSimpleWhere || simpleColumn is null || simpleValue is null || + MatchesSimpleWhere(row, schema, simpleColumn, simpleValue)) + { + yield return row; + } } - - return ScanByPrimaryKeyPoint(simpleValue, schema, engine, enableCaching); } - private IEnumerable? TryScanNumeric( - string? simpleColumn, object? simpleValue, bool hasSimpleWhere, - VariableLengthSchema schema, IStorageEngine engine, bool enableCaching) - { - if (!hasSimpleWhere || simpleColumn is null || simpleValue is null || - !TryGetFixedNumericWhereInfo(simpleColumn, out var numericOffset, out var numericType) || - !TryParseNumericExpected(simpleValue, numericType, out var numericExpected)) - { - return null; - } - - return ScanByNumericSimd(numericOffset, numericType, numericExpected, schema, engine, enableCaching); - } /// Fast path 1: hash-index point lookup (mirrors SelectInternal). private IEnumerable ScanByHashIndexPoint( string simpleColumn, object simpleValue, VariableLengthSchema schema, IStorageEngine engine, bool enableCaching) @@ -313,90 +316,91 @@ private IEnumerable ScanByPrimaryKeyPoint( /// /// Fast path 3: fixed-width numeric equality — SIMD batch filter over extracted values /// (no deserialization, no boxing). Integer/Long use portable Vector<T>; Real uses - /// direct per-record reads. + /// direct per-record reads. Fixed-width tables materialize rows through the dictionary path. /// private IEnumerable ScanByNumericSimd( int numericOffset, DataType numericType, object numericExpected, - VariableLengthSchema schema, IStorageEngine engine, bool enableCaching) + VariableLengthSchema schema, IStorageEngine engine, bool enableCaching, + bool fixedWidth, string[]? fixedColumns, DataType[]? fixedTypes) { if (numericType == DataType.Integer || numericType == DataType.Long) { - foreach (var row in ScanNumericIntLong(numericOffset, numericType, numericExpected, schema, engine, enableCaching)) - yield return row; - yield break; - } + List? intValues = numericType == DataType.Integer ? new List(1024) : null; + List? longValues = numericType == DataType.Long ? new List(1024) : null; + var recordDatas = new List(1024); + var recordPositions = new List(1024); - // Real (double): direct per-record reads. - foreach (var (recordPosition, data) in engine.GetAllRecords(Name)) - { - if (data is not { Length: > 0 } || - !MatchesNumericDirect(data, numericOffset, numericType, numericExpected) || - !TryValidateCurrentVersion(data, schema, recordPosition)) + foreach (var (pos, rec) in engine.GetAllRecords(Name)) { - continue; - } - - yield return new StructRow(data.AsMemory(), schema, enableCaching); - } - } + if (rec is not { Length: > 0 } || !TryExtractNumericDirect(rec, numericOffset, numericType, out var val)) + { + continue; + } - /// - /// SIMD batch filter for Integer/Long expected values: collects raw constant-offset numeric - /// values, runs the portable Vector<T> equality filter and yields the matches. - /// - private IEnumerable ScanNumericIntLong( - int numericOffset, DataType numericType, object numericExpected, - VariableLengthSchema schema, IStorageEngine engine, bool enableCaching) - { - List? intValues = numericType == DataType.Integer ? new List(1024) : null; - List? longValues = numericType == DataType.Long ? new List(1024) : null; - var recordDatas = new List(1024); - var recordPositions = new List(1024); + if (intValues is not null) + { + intValues.Add((int)val); + } + else if (longValues is not null) + { + longValues.Add((long)val); + } - foreach (var (pos, rec) in engine.GetAllRecords(Name)) - { - if (rec is not { Length: > 0 } || !TryExtractNumericDirect(rec, numericOffset, numericType, out var val)) - { - continue; + recordDatas.Add(rec); + recordPositions.Add(pos); } + var matches = new List(16); if (intValues is not null) { - intValues.Add((int)val); + SimdFilterInt32Batch(CollectionsMarshal.AsSpan(intValues), (int)numericExpected, matches); } else if (longValues is not null) { - longValues.Add((long)val); + SimdFilterInt64Batch(CollectionsMarshal.AsSpan(longValues), (long)numericExpected, matches); } - recordDatas.Add(rec); - recordPositions.Add(pos); - } + for (int mi = 0; mi < matches.Count; mi++) + { + var rec = recordDatas[matches[mi]]; + if (!TryValidateCurrentVersion(rec, schema, recordPositions[matches[mi]], fixedWidth)) + { + continue; + } - var matches = new List(16); - if (intValues is not null) - { - SimdFilterInt32Batch(CollectionsMarshal.AsSpan(intValues), (int)numericExpected, matches); - } - else if (longValues is not null) - { - SimdFilterInt64Batch(CollectionsMarshal.AsSpan(longValues), (long)numericExpected, matches); + if (fixedWidth) + { + yield return StructRow.FromDictionary(DeserializeRowFixedWidth(rec.AsSpan()), fixedColumns ?? [], fixedTypes ?? []); + } + else + { + yield return new StructRow(rec.AsMemory(), schema, enableCaching); + } + } } - - for (int mi = 0; mi < matches.Count; mi++) + else { - var rec = recordDatas[matches[mi]]; - if (!TryValidateCurrentVersion(rec, schema, recordPositions[matches[mi]])) + // Real (double): direct per-record reads. + foreach (var (recordPosition, data) in engine.GetAllRecords(Name)) { - continue; - } + if (data is not { Length: > 0 } || + !MatchesNumericDirect(data, numericOffset, numericType, numericExpected) || + !TryValidateCurrentVersion(data, schema, recordPosition, fixedWidth)) + { + continue; + } - yield return new StructRow(rec.AsMemory(), schema, enableCaching); + if (fixedWidth) + { + yield return StructRow.FromDictionary(DeserializeRowFixedWidth(data.AsSpan()), fixedColumns ?? [], fixedTypes ?? []); + } + else + { + yield return new StructRow(data.AsMemory(), schema, enableCaching); + } + } } } - - - /// /// Zero-allocation enumerable for . Foreach on this concrete /// type uses (no heap allocation); treating it as /// IEnumerable<StructRow> (LINQ, boxing) uses a small class-based enumerator. @@ -515,16 +519,46 @@ private bool InitAndMoveNext() bool hasSimpleWhere = _where is { Length: > 0 } && TryParseSimpleWhereClause(_where, out simpleColumn, out simpleValue); - // Fast path 1: hash-index point lookup (mirrors SelectInternal). - if (TryActivateHashPhase(simpleColumn, simpleValue, hasSimpleWhere, out var hashMoved)) + // Fast path 1: hash-index point lookup (mirrors SelectInternal). Disabled for + // fixed-width tables (their records use the overflow format, not the walkable layout). + if (!_table._fixedWidthRecords && hasSimpleWhere && simpleColumn is not null && simpleValue is not null && + _table.registeredIndexes.ContainsKey(simpleColumn)) { - return hashMoved; + _table.EnsureIndexLoaded(simpleColumn); + if (_table.hashIndexes.TryGetValue(simpleColumn, out var hashIndex)) + { + var colIdx = _table.Columns.IndexOf(simpleColumn); + if (colIdx >= 0) + { + var key = ParseValueForHashLookup(simpleValue.ToString() ?? string.Empty, _table.ColumnTypes[colIdx]); + if (key is not null) + { + _positions = hashIndex.LookupPositions(key); + _posIndex = 0; + _phase = Phase.Hash; + return MoveNextHash(); + } + } + } } - // Fast path 2: primary-key lookup. - if (TryActivatePkPhase(simpleColumn, simpleValue, hasSimpleWhere, out var pkMoved)) + // Fast path 2: primary-key lookup. Disabled for fixed-width tables (same reason). + if (!_table._fixedWidthRecords && hasSimpleWhere && simpleColumn is not null && simpleValue is not null && + _table.PrimaryKeyIndex >= 0 && + string.Equals(simpleColumn, _table.Columns[_table.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) { - return pkMoved; + var pkStr = simpleValue.ToString() ?? string.Empty; + var search = _table.Index.Search(pkStr); + if (search.Found) + { + _pkPosition = search.Value; + _pkFound = true; + _phase = Phase.Pk; + return MoveNextPk(); + } + + _phase = Phase.Done; + return false; } // Fallback: numeric-SIMD batch filter / full scan (allocating by nature). @@ -533,67 +567,6 @@ private bool InitAndMoveNext() return MoveNextFallback(); } - private bool TryActivateHashPhase(string? simpleColumn, object? simpleValue, bool hasSimpleWhere, out bool movedNext) - { - movedNext = false; - if (!hasSimpleWhere || simpleColumn is null || simpleValue is null || - !_table.registeredIndexes.ContainsKey(simpleColumn)) - { - return false; - } - - _table.EnsureIndexLoaded(simpleColumn); - if (!_table.hashIndexes.TryGetValue(simpleColumn, out var hashIndex)) - { - return false; - } - - var colIdx = _table.Columns.IndexOf(simpleColumn); - if (colIdx < 0) - { - return false; - } - - var key = ParseValueForHashLookup(simpleValue.ToString() ?? string.Empty, _table.ColumnTypes[colIdx]); - if (key is null) - { - return false; - } - - _positions = hashIndex.LookupPositions(key); - _posIndex = 0; - _phase = Phase.Hash; - movedNext = MoveNextHash(); - return true; - } - - private bool TryActivatePkPhase(string? simpleColumn, object? simpleValue, bool hasSimpleWhere, out bool movedNext) - { - movedNext = false; - if (!hasSimpleWhere || simpleColumn is null || simpleValue is null || - _table.PrimaryKeyIndex < 0 || - !string.Equals(simpleColumn, _table.Columns[_table.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - var pkStr = simpleValue.ToString() ?? string.Empty; - var search = _table.Index.Search(pkStr); - if (search.Found) - { - _pkPosition = search.Value; - _pkFound = true; - _phase = Phase.Pk; - movedNext = MoveNextPk(); - } - else - { - _phase = Phase.Done; - } - - return true; - } - /// Advances the hash-index fast path. private bool MoveNextHash() { @@ -752,19 +725,33 @@ private static bool MatchesSimpleWhere(StructRow row, VariableLengthSchema schem /// /// Stale-version guard: when the table has a PK, the PK index must point to /// for the record to be the current version. - /// Returns true for tables without a PK (no version tracking). + /// Returns true for tables without a PK (no version tracking). For fixed-width records the + /// PK is read via the arena-aware dictionary deserialization (constant slot offsets). /// - private bool TryValidateCurrentVersion(ReadOnlySpan recordData, VariableLengthSchema schema, long recordPosition) + private bool TryValidateCurrentVersion(ReadOnlySpan recordData, VariableLengthSchema schema, long recordPosition, bool fixedWidth) { if (this.PrimaryKeyIndex < 0) { return true; } - var pkValue = ExtractPrimaryKeyValueFromSpan(recordData, schema); - if (pkValue is null) + string pkValue; + if (fixedWidth) { - return false; + var row = DeserializeRowFixedWidth(recordData); + pkValue = row.TryGetValue(this.Columns[this.PrimaryKeyIndex], out var v) && v is not null && v != DBNull.Value + ? v.ToString() ?? string.Empty + : string.Empty; + } + else + { + var pk = ExtractPrimaryKeyValueFromSpan(recordData, schema); + if (pk is null) + { + return false; + } + + pkValue = pk; } var search = this.Index.Search(pkValue); @@ -789,6 +776,19 @@ private bool TryGetFixedNumericWhereInfo(string column, out int valueOffset, out if (type != DataType.Integer && type != DataType.Long && type != DataType.Real) return false; + if (_fixedWidthRecords) + { + // Fixed-width layout: every column sits at a constant slot offset (null flag + payload), + // so the numeric column can be read directly regardless of preceding variable columns — + // no layout walk needed (B4). + var layout = GetFixedWidthLayout(); + if (colIdx >= layout.ColumnCount) + return false; + + valueOffset = layout.Offsets[colIdx]; + return true; + } + for (int i = 0; i < colIdx; i++) { (int size, bool isVariable) = GetColumnSizeAndVariability(this.ColumnTypes[i]); @@ -833,7 +833,7 @@ private static bool TryExtractNumericDirect( DataType type, out object value) { - value = null!; // NOSONAR:S8970 - required: out parameter is non-nullable, set on success paths + value = null!; // valueOffset points at the null flag; value data starts at +1. if (valueOffset + 1 >= recordData.Length) @@ -864,7 +864,7 @@ private static bool TryExtractNumericDirect( /// private static bool TryParseNumericExpected(object? value, DataType type, out object expected) { - expected = null!; // NOSONAR:S8970 - required: out parameter is non-nullable, set on success paths + expected = null!; string text = value?.ToString() ?? string.Empty; switch (type) @@ -883,6 +883,31 @@ private static bool TryParseNumericExpected(object? value, DataType type, out ob } } + /// + /// B4: fixed-width string early-WHERE — reads the variable column's constant slot + /// [null-flag(1)][arena-offset(4)], resolves the payload from the overflow arena and + /// compares it byte-wise against the pre-encoded expected UTF-8 (Binary collation). The + /// comparison is exact for Binary collation (no full-row deserialization for non-matches). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool MatchesFixedWidthStringDirect( + ReadOnlySpan recordData, + int slotOffset, + OverflowArena arena, + ReadOnlySpan expectedUtf8) + { + if (slotOffset + 5 > recordData.Length || recordData[slotOffset] == 0) + { + return false; // truncated record or NULL slot (NULL never equals a value) + } + + // NOTE: offset 0 is a VALID block offset (the first arena block's length prefix sits at 0), + // so only the flag byte above distinguishes NULL — never filter on the offset value itself. + var arenaOffset = BinaryPrimitives.ReadInt32LittleEndian(recordData.Slice(slotOffset + 1, 4)); + var payload = arena.Read(arenaOffset); + return payload is not null && payload.AsSpan().SequenceEqual(expectedUtf8); + } + #endregion #region Internal Scanning Implementation diff --git a/src/SharpCoreDB/DataStructures/Table.cs b/src/SharpCoreDB/DataStructures/Table.cs index 34c310ac..b3cac515 100644 --- a/src/SharpCoreDB/DataStructures/Table.cs +++ b/src/SharpCoreDB/DataStructures/Table.cs @@ -51,7 +51,12 @@ public Table(IStorage storage, bool isReadOnly = false, DatabaseConfig? config = { (this.storage, this.isReadOnly) = (storage, isReadOnly); _config = config; - + + // Fixed-width record layout (out-of-line overflow): opt-in, columnar mode only. + // When enabled, records have a constant size per schema and variable-length values live in + // the table's overflow arena — every UPDATE is an in-place overwrite. + _fixedWidthRecords = config?.FixedWidthRecordLayout ?? false; + // Apply compaction threshold from config if provided if (config is not null && config.ColumnarAutoCompactionThreshold > 0) { @@ -251,6 +256,34 @@ private Dictionary GetColumnIndexCache() // ✅ NEW: DatabaseConfig for passing optimizations through to storage engines private readonly DatabaseConfig? _config; + // Fixed-width record layout (out-of-line overflow, SQLite-model) — opt-in via + // DatabaseConfig.FixedWidthRecordLayout. Only takes effect for columnar/append-only tables. + private bool _fixedWidthRecords; + private FixedWidthRecordLayout? _fixedWidthLayout; + private OverflowArena? _overflowArena; + + /// + /// Gets or sets whether this table uses the fixed-width record layout (out-of-line overflow). + /// Persisted in table metadata so a database created with the flag reopens correctly. + /// + public bool IsFixedWidthRecords + { + get => _fixedWidthRecords; + set => _fixedWidthRecords = value; + } + + /// + /// B6: gets the number of overflow-arena blocks reused in place via the free-list (diagnostics). + /// Freed blocks of the same payload length are overwritten in place instead of appended, so the + /// .ovf stops growing during same-length variable-column updates. + /// + public int OverflowArenaBlockReuses => _overflowArena?.BlockReuses ?? 0; + + /// + /// B6: gets the number of freed arena blocks currently tracked for in-place reuse (diagnostics). + /// + public int OverflowArenaFreeBlockCount => _overflowArena?.FreeBlockCount ?? 0; + // ✅ NEW: Compaction tracking for columnar storage private long _deletedRowCount = 0; private long _updatedRowCount = 0; @@ -561,32 +594,51 @@ public void RebuildPrimaryKeyIndexFromDisk() // Parse just enough to get the primary key value try { - var row = new Dictionary(); - int offset = 0; + string? pkStr = null; - for (int i = 0; i < Columns.Count; i++) + if (_fixedWidthRecords) { - if (offset >= recordData.Length) + // Fixed-width layout: variable columns reference the overflow arena, so the + // record must be deserialized through the fixed-width codec (a raw walk would + // misread the 5-byte variable slots as length-prefixed values). + var fwRow = DeserializeRowFixedWidth(recordData.AsSpan()); + if (fwRow.TryGetValue(Columns[PrimaryKeyIndex], out var fwPk) && fwPk is not null) { - break; + pkStr = fwPk.ToString(); } + } + else + { + var row = new Dictionary(); + int offset = 0; - var value = ReadTypedValueFromSpan(recordData.AsSpan(offset), ColumnTypes[i], out int bytesRead); - - // Only store PK column, we don't need the rest for index rebuild - if (i == PrimaryKeyIndex) + for (int i = 0; i < Columns.Count; i++) { - row[Columns[i]] = value; + if (offset >= recordData.Length) + { + break; + } + + var value = ReadTypedValueFromSpan(recordData.AsSpan(offset), ColumnTypes[i], out int bytesRead); + + // Only store PK column, we don't need the rest for index rebuild + if (i == PrimaryKeyIndex) + { + row[Columns[i]] = value; + } + + offset += bytesRead; } - offset += bytesRead; + // Extract PK value + if (row.TryGetValue(Columns[PrimaryKeyIndex], out var pkValue) && pkValue != null) + { + pkStr = pkValue.ToString() ?? string.Empty; + } } - // Extract PK value and add to index - if (row.TryGetValue(Columns[PrimaryKeyIndex], out var pkValue) && pkValue != null) + if (pkStr is not null) { - var pkStr = pkValue.ToString() ?? string.Empty; - // Only add if this key doesn't exist yet (handles UPDATE versions - keep latest) var existing = Index.Search(pkStr); if (!existing.Found) diff --git a/src/SharpCoreDB/Database/Caching/Database.PlanCaching.cs b/src/SharpCoreDB/Database/Caching/Database.PlanCaching.cs index 9278b904..3f5215e5 100644 --- a/src/SharpCoreDB/Database/Caching/Database.PlanCaching.cs +++ b/src/SharpCoreDB/Database/Caching/Database.PlanCaching.cs @@ -51,6 +51,27 @@ private QueryPlanCache GetPlanCache() [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool IsPlanCachingEnabled() => config?.EnableCompiledPlanCache ?? true; + // B8: memoizes the normalized SQL per exact SQL text. The same statement text is + // normalized on every ExecuteQuery call otherwise (trim + whitespace collapse + string + // allocation) — for repeated point-lookups this cache turns 10K normalizations into 10K + // dictionary lookups. + private readonly System.Collections.Concurrent.ConcurrentDictionary _normalizedSqlCache = + new(System.StringComparer.Ordinal); + + /// + /// B8: returns the normalized SQL, memoized per exact input text. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private string GetNormalizedSql(string sql) + { + if (!(config?.NormalizeSqlForPlanCache ?? true)) + { + return sql; + } + + return _normalizedSqlCache.GetOrAdd(sql, static s => QueryPlanCache.NormalizeSql(s.Trim())); + } + /// /// Caches a query plan for DML operations (INSERT, UPDATE, DELETE). /// Normalizes SQL and parameters to maximize cache hit rate. @@ -65,9 +86,7 @@ private QueryPlanCache GetPlanCache() if (!IsPlanCachingEnabled()) return null; - var normalized = (config?.NormalizeSqlForPlanCache ?? true) - ? NormalizeSqlForCaching(sql) - : sql; + var normalized = GetNormalizedSql(sql); var key = BuildCacheKey(normalized, parameters, commandType); var cache = GetPlanCache(); @@ -97,9 +116,7 @@ private QueryPlanCache GetPlanCache() if (!IsPlanCachingEnabled() || planCache is null) return null; - var normalized = (config?.NormalizeSqlForPlanCache ?? true) - ? NormalizeSqlForCaching(sql) - : sql; + var normalized = GetNormalizedSql(sql); var key = BuildCacheKey(normalized, parameters, commandType); diff --git a/src/SharpCoreDB/Database/Core/Database.Core.cs b/src/SharpCoreDB/Database/Core/Database.Core.cs index 10c3d6ea..553db682 100644 --- a/src/SharpCoreDB/Database/Core/Database.Core.cs +++ b/src/SharpCoreDB/Database/Core/Database.Core.cs @@ -214,6 +214,8 @@ private void Load() { string? metaJson; bool metaExists; + // B5: set when auto-migration converts a legacy (1.x) table to the fixed-width layout. + bool migratedAnyTable = false; if (_storageProvider is not null) { @@ -402,6 +404,24 @@ private void Load() table.InitializeStorageEngine(); } + // B5 (1.x → 2.0 record-format migration): the persisted fixed-width flag is now + // authoritative — a legacy (1.x) table simply lacks it (variable-length records). + // Opening a legacy table as fixed-width would misread its records, so when the + // config opts into FixedWidthRecordLayout we AUTO-MIGRATE the legacy table instead. + // Both Columnar and PageBased tables are migrated (PageBased tables are converted + // to Columnar storage in-process first); read-only opens never rewrite data. + if (!table.IsFixedWidthRecords && config is { FixedWidthRecordLayout: true }) + { + var storageMode = table.StorageMode; + if (!isReadOnly && + (storageMode == SharpCoreDB.Storage.Hybrid.StorageMode.Columnar || + storageMode == SharpCoreDB.Storage.Hybrid.StorageMode.PageBased)) + { + table.MigrateToFixedWidth(); + migratedAnyTable = true; + } + } + // ✅ CRITICAL FIX: Complete initialization of new DDL properties // Ensure lists have correct length @@ -487,6 +507,13 @@ private void Load() #if DEBUG System.Diagnostics.Debug.WriteLine($"[Load] Total tables loaded: {tables.Count}"); #endif + + // B5: persist the new record-format flags (and rebuilt indexes) when auto-migration + // converted any legacy table during this load. + if (migratedAnyTable) + { + SaveMetadata(); + } } /// @@ -514,6 +541,7 @@ private void SaveMetadata() ForeignKeys = t.ForeignKeys, // Added for Phase 1.2 ColumnCollations = t.ColumnCollations, // ✅ COLLATE Phase 1: Persist per-column collation AutoIncrementCounters = t.AutoIncrementCounters, // ✅ AUTO INCREMENT: Persist counter state + IsFixedWidthRecords = t.IsFixedWidthRecords, // B5: persist the record format (1.x → 2.0) }).ToList(); var meta = new Dictionary { [PersistenceConstants.TablesKey] = tablesList }; @@ -819,6 +847,12 @@ public async ValueTask DisposeAsync() queryCache?.Clear(); ClearPlanCache(); + // ✅ B6: flush + release each table's storage engine (see the sync Dispose path). + foreach (var table in tables.Values.OfType()) + { + try { table.Dispose(); } catch { /* best-effort */ } + } + _disposed = true; GC.SuppressFinalize(this); } @@ -861,6 +895,14 @@ protected virtual void Dispose(bool disposing) pageCache?.Clear(false, null); queryCache?.Clear(); ClearPlanCache(); // ✅ Clear query plan cache on disposal + + // ✅ B6: flush + release each table's storage engine so pending page-based writes are + // persisted. A single INSERT/UPDATE never flushes the page cache, so without this a + // reopened PageBased table returned zero rows (data loss on dispose). + foreach (var table in tables.Values.OfType
()) + { + try { table.Dispose(); } catch { /* best-effort */ } + } } _disposed = true; @@ -871,8 +913,7 @@ protected virtual void Dispose(bool disposing) /// results (struct enumerable — foreach on the returned value is allocation-free). Avoids /// per-row Dictionary allocations and value boxing (~200 B → ~20 B per row). Supports the /// simple "SELECT [*|col] FROM t [WHERE col = @param|'literal'] [LIMIT n]" shape; more complex - /// queries throw . Parameterized queries reuse the plan - /// cache and the zero-reparse point-lookup fast path. + /// queries throw . /// public DataStructures.StructRowQueryEnumerable ExecuteQueryStruct(string sql, Dictionary? parameters = null) { diff --git a/src/SharpCoreDB/Database/Core/Database.FixedWidthMigration.cs b/src/SharpCoreDB/Database/Core/Database.FixedWidthMigration.cs new file mode 100644 index 00000000..5204a1ad --- /dev/null +++ b/src/SharpCoreDB/Database/Core/Database.FixedWidthMigration.cs @@ -0,0 +1,40 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB; + +using SharpCoreDB.DataStructures; + +/// +/// B5: 1.x → 2.0 record-format migration at the database level. Exposes the on-demand conversion +/// of a legacy (variable-length records) table to the fixed-width record layout and persists the +/// new record-format flag in metadata. +/// +public partial class Database +{ + /// + public int MigrateTableToFixedWidth(string tableName) + { + if (isReadOnly) + { + throw new InvalidOperationException("Cannot migrate a table in a read-only database."); + } + + if (!tables.TryGetValue(tableName, out var table)) + { + throw new InvalidOperationException($"Unknown table: {tableName}"); + } + + if (table is not Table concrete) + { + throw new NotSupportedException( + $"Table '{tableName}' does not support the fixed-width record layout (single-file tables use their own storage format)."); + } + + int migrated = concrete.MigrateToFixedWidth(); + SaveMetadata(); // persist the new record-format flag so reopen keeps the layout + return migrated; + } +} diff --git a/src/SharpCoreDB/Database/Execution/Database.Batch.cs b/src/SharpCoreDB/Database/Execution/Database.Batch.cs index 98e7f72e..20562f8b 100644 --- a/src/SharpCoreDB/Database/Execution/Database.Batch.cs +++ b/src/SharpCoreDB/Database/Execution/Database.Batch.cs @@ -514,6 +514,332 @@ private static bool IsInsertStatement(string sql) return trimmed.Length >= 11 && trimmed[..11].Equals(SqlInsertPrefix, StringComparison.OrdinalIgnoreCase); } + /// + /// Phase-2 fast parse: matches exactly the canonical single-row DML shape + /// UPDATE <table> SET <col> = <literal> WHERE <col> = <literal> + /// (and DELETE FROM <table> WHERE <col> = <literal>) with a quotes-aware + /// span scan — no regex. Any deviation (multi-column SET, other operators, top-level commas, + /// missing WHERE, trailing semicolons, unterminated strings) returns false so the caller falls + /// back to the general regex path. Returns raw literal text (quotes included) so the caller + /// still converts via . + /// + private static bool TryScanCanonicalDml( + string sql, + out string table, + out string setCol, + out string setValRaw, + out string whereCol, + out string whereValRaw) + { + table = string.Empty; + setCol = string.Empty; + setValRaw = string.Empty; + whereCol = string.Empty; + whereValRaw = string.Empty; + + var s = sql.AsSpan().Trim(); + if (s.Length == 0) + { + return false; + } + + int i = 0; + + // verb: UPDATE or DELETE + bool isUpdate; + if (s[i] is 'U' or 'u') + { + isUpdate = true; + if (!TryConsumeKeyword(s, ref i, "UPDATE") || !TryConsumeWhitespace(s, ref i)) + { + return false; + } + } + else if (s[i] is 'D' or 'd') + { + isUpdate = false; + if (!TryConsumeKeyword(s, ref i, "DELETE") || !TryConsumeWhitespace(s, ref i)) + { + return false; + } + + if (!TryConsumeKeyword(s, ref i, "FROM") || !TryConsumeWhitespace(s, ref i)) + { + return false; + } + } + else + { + return false; + } + + // table name: simple identifier up to the next whitespace + if (!TryReadSimpleIdent(s, ref i, out var tableSpan) || tableSpan.IsEmpty) + { + return false; + } + + table = tableSpan.ToString(); + + if (isUpdate) + { + if (!TryConsumeWhitespace(s, ref i) || !TryConsumeKeyword(s, ref i, "SET") || !TryConsumeWhitespace(s, ref i)) + { + return false; + } + + // single SET column: = + if (!TryReadSimpleIdent(s, ref i, out var setColSpan) || setColSpan.IsEmpty) + { + return false; + } + + setCol = setColSpan.ToString(); + + if (!TrySkipWsAndEquals(s, ref i)) + { + return false; + } + + // set literal: read quotes-aware up to a top-level WHERE keyword + if (!TryReadLiteralUntilWhere(s, ref i, out var setValSpan)) + { + return false; + } + + setValRaw = setValSpan.ToString(); + + // TryReadLiteralUntilWhere advanced i to just after "WHERE"; skip the whitespace + // before the WHERE column. + if (!TryConsumeWhitespace(s, ref i)) + { + return false; + } + } + + // WHERE = (to end of statement) + if (!TryReadSimpleIdent(s, ref i, out var whereColSpan) || whereColSpan.IsEmpty) + { + return false; + } + + whereCol = whereColSpan.ToString(); + + if (!TrySkipWsAndEquals(s, ref i)) + { + return false; + } + + int v0 = i; + bool inString = false; + char quote = '\0'; + while (i < s.Length) + { + char ch = s[i]; + if (inString) + { + if (ch == quote) + { + if (i + 1 < s.Length && s[i + 1] == quote) + { + i += 2; // escaped '' inside a string literal + continue; + } + + inString = false; + } + + i++; + continue; + } + + if (ch is '\'' or '"') + { + inString = true; + quote = ch; + i++; + continue; + } + + if (ch == ';') + { + return false; // trailing semicolon not part of the canonical shape + } + + i++; + } + + if (inString) + { + return false; // unterminated string literal + } + + var whereVal = s[v0..].Trim(); + if (whereVal.IsEmpty) + { + return false; + } + + whereValRaw = whereVal.ToString(); + return true; + } + + private static bool TryConsumeWhitespace(ReadOnlySpan s, ref int i) + { + int start = i; + while (i < s.Length && char.IsWhiteSpace(s[i])) + { + i++; + } + + return i > start; + } + + private static bool TryConsumeKeyword(ReadOnlySpan s, ref int i, string keyword) + { + if (i + keyword.Length > s.Length) + { + return false; + } + + if (!s.Slice(i, keyword.Length).Equals(keyword.AsSpan(), StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + // word boundary: next char must be whitespace (or end) + if (i + keyword.Length < s.Length && !char.IsWhiteSpace(s[i + keyword.Length])) + { + return false; + } + + i += keyword.Length; + return true; + } + + private static bool TryReadSimpleIdent(ReadOnlySpan s, ref int i, out ReadOnlySpan ident) + { + ident = default; + int start = i; + while (i < s.Length && !char.IsWhiteSpace(s[i]) && s[i] != '=' && s[i] != '(' && s[i] != ')') + { + i++; + } + + if (i == start) + { + return false; + } + + ident = s[start..i]; + return true; + } + + private static bool TrySkipWsAndEquals(ReadOnlySpan s, ref int i) + { + while (i < s.Length && char.IsWhiteSpace(s[i])) + { + i++; + } + + if (i >= s.Length || s[i] != '=') + { + return false; + } + + i++; + while (i < s.Length && char.IsWhiteSpace(s[i])) + { + i++; + } + + return true; + } + + /// + /// Reads a value literal from up to a top-level WHERE keyword + /// (or the end of the span), respecting single/double-quoted strings. Returns false when the + /// value contains a top-level comma (multi-column SET), an unterminated string or a trailing + /// semicolon, or when no WHERE keyword follows. On success is left after + /// the WHERE keyword. + /// + private static bool TryReadLiteralUntilWhere(ReadOnlySpan s, ref int i, out ReadOnlySpan value) + { + value = default; + int v0 = i; + bool inString = false; + char quote = '\0'; + + while (i < s.Length) + { + char ch = s[i]; + if (inString) + { + if (ch == quote) + { + if (i + 1 < s.Length && s[i + 1] == quote) + { + i += 2; + continue; + } + + inString = false; + } + + i++; + continue; + } + + if (ch is '\'' or '"') + { + inString = true; + quote = ch; + i++; + continue; + } + + if (ch == ',') + { + return false; // multi-column SET clause -> not canonical + } + + if (ch == ';') + { + return false; + } + + if (char.IsWhiteSpace(ch)) + { + // boundary check for the WHERE keyword (must follow whitespace) + int j = i; + while (j < s.Length && char.IsWhiteSpace(s[j])) + { + j++; + } + + if (j + 5 <= s.Length && + s.Slice(j, 5).Equals("WHERE".AsSpan(), StringComparison.OrdinalIgnoreCase) && + (j + 5 == s.Length || char.IsWhiteSpace(s[j + 5]))) + { + // value ends at the last non-whitespace before the WHERE keyword + int end = i; + while (end > v0 && char.IsWhiteSpace(s[end - 1])) + { + end--; + } + + value = s[v0..end]; + i = j + 5; // position after "WHERE" + return !value.IsEmpty; + } + } + + i++; + } + + return false; // canonical UPDATE must have a WHERE clause + } + /// /// Attempts to parse an UPDATE statement for batch execution. /// Extracts the table name, WHERE clause, and SET column-value pairs. @@ -529,6 +855,29 @@ private bool TryParseUpdateForBatch(string sql, out string tableName, out string where = string.Empty; updates = []; + // Phase-2 fast path: canonical single-column shape + // `UPDATE
SET = WHERE = ` — regex-free. + if (TryScanCanonicalDml(sql, out var fastTable, out var setCol, out var setValRaw, out var whereCol, out var whereValRaw)) + { + if (!tables.TryGetValue(fastTable, out var fastTableMeta)) + { + return false; + } + + int colIdx = fastTableMeta.Columns.IndexOf(setCol); + if (colIdx < 0) + { + return false; + } + + var parsed = SqlParser.ParseValue(setValRaw, fastTableMeta.ColumnTypes[colIdx]); + updates[setCol] = parsed ?? DBNull.Value; + tableName = fastTable; + where = whereCol + " = " + whereValRaw; + return true; + } + + // Fallback: general regex path for non-canonical UPDATE statements. var span = sql.AsSpan().Trim(); if (span.Length < 6 || !span[..6].Equals("UPDATE", StringComparison.OrdinalIgnoreCase)) return false; @@ -579,6 +928,21 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string tableName = string.Empty; where = string.Empty; + // Phase-2 fast path: canonical single-row shape + // `DELETE FROM
WHERE = ` — regex-free. + if (TryScanCanonicalDml(sql, out var fastTable, out _, out _, out var whereCol, out var whereValRaw)) + { + if (!tables.ContainsKey(fastTable)) + { + return false; + } + + tableName = fastTable; + where = whereCol + " = " + whereValRaw; + return true; + } + + // Fallback: general regex path for non-canonical DELETE statements. var span = sql.AsSpan().Trim(); if (span.Length < 6 || !span[..6].Equals("DELETE", StringComparison.OrdinalIgnoreCase)) return false; @@ -825,26 +1189,34 @@ private static List> RowsToDictionaryList(SharpCoreDB { try { + // B9: the only caller already ran IsInsertStatement, so the statement starts with + // "INSERT INTO" (after optional leading whitespace) — skip the redundant full-span + // IndexOf scan. var insertSql = sql.AsSpan(); - var insertIdx = insertSql.IndexOf(SqlInsertPrefix, StringComparison.OrdinalIgnoreCase); - if (insertIdx < 0) return null; - - insertSql = insertSql.Slice(insertIdx); - var tableStart = (SqlInsertPrefix.Length + 1); - - // Find table name end - int tableEnd = -1; - for (int i = tableStart; i < insertSql.Length; i++) + int idx = 0; + while (idx < insertSql.Length && char.IsWhiteSpace(insertSql[idx])) { - if (insertSql[i] == ' ' || insertSql[i] == '(') - { - tableEnd = i; - break; - } + idx++; } - if (tableEnd == -1) return null; - var tableName = insertSql.Slice(tableStart, tableEnd - tableStart).Trim().ToString(); + insertSql = insertSql.Slice(idx); + const int KeywordLen = 11; // "INSERT INTO".Length + const int PrefixLen = 12; // "INSERT INTO ".Length + if (insertSql.Length < PrefixLen || + !insertSql[..KeywordLen].Equals(SqlInsertPrefix, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + // Find table name end (whitespace or opening parenthesis). + var nameSpan = insertSql.Slice(KeywordLen).TrimStart(); + int tableEnd = nameSpan.IndexOfAny(' ', '('); + if (tableEnd < 0) + { + return null; + } + + var tableName = nameSpan[..tableEnd].ToString(); if (!tables.ContainsKey(tableName)) return null; diff --git a/src/SharpCoreDB/Database/TableMetadataDto.cs b/src/SharpCoreDB/Database/TableMetadataDto.cs index 08d29dac..67f8a20e 100644 --- a/src/SharpCoreDB/Database/TableMetadataDto.cs +++ b/src/SharpCoreDB/Database/TableMetadataDto.cs @@ -35,6 +35,12 @@ public sealed class TableMetadataDto /// Gets or sets the storage mode. public SharpCoreDB.Storage.Hybrid.StorageMode StorageMode { get; set; } + /// + /// Gets or sets whether the table uses the fixed-width record layout (out-of-line overflow). + /// B5: persisted so a reopened database keeps the record format without needing the config flag. + /// + public bool IsFixedWidthRecords { get; set; } + /// Gets or sets auto-increment flags per column. public List? IsAuto { get; set; } diff --git a/src/SharpCoreDB/DatabaseConfig.cs b/src/SharpCoreDB/DatabaseConfig.cs index d3171e6b..3ca71f85 100644 --- a/src/SharpCoreDB/DatabaseConfig.cs +++ b/src/SharpCoreDB/DatabaseConfig.cs @@ -19,6 +19,17 @@ public class DatabaseConfig /// public bool NoEncryptMode { get; init; } = false; + /// + /// Gets a value indicating whether new directory-mode tables use the fixed-width record layout + /// with out-of-line overflow (the SQLite-model). Fixed-size columns live at constant record + /// offsets; variable-length (TEXT/BLOB) values are stored in a per-table overflow arena, so the + /// record length is constant per schema and every UPDATE is an in-place overwrite. + /// ⚠️ OPT-IN FORMAT: only enable on databases whose columnar tables are created with the same + /// flag, and never share such databases with tooling built before this option. Existing tables + /// are unaffected. + /// + public bool FixedWidthRecordLayout { get; init; } = false; + /// /// Gets a value indicating whether SQLite integer type affinity is used for DDL type mapping. /// When (opt-in), INTEGER maps to (Int64), diff --git a/src/SharpCoreDB/DatabaseExtensions.cs b/src/SharpCoreDB/DatabaseExtensions.cs index 5578dedb..073ae8c8 100644 --- a/src/SharpCoreDB/DatabaseExtensions.cs +++ b/src/SharpCoreDB/DatabaseExtensions.cs @@ -712,6 +712,28 @@ public int MigrateLegacyUlids() return converted; } + /// + public int MigrateTableToFixedWidth(string tableName) + { + if (_options.IsReadOnly) + { + throw new InvalidOperationException("Cannot migrate a table in a read-only database."); + } + + if (!_tables.TryGetValue(tableName, out var table)) + { + throw new InvalidOperationException($"Unknown table: {tableName}"); + } + + if (table is not SingleFileTable sft) + { + throw new NotSupportedException( + $"Table '{tableName}' does not support the fixed-width record layout."); + } + + return sft.MigrateToFixedWidth(); + } + /// /// Rewrites every ULID value of a single-file table from the legacy encoding to the spec encoding. /// @@ -835,6 +857,10 @@ private void LoadTables() if (metadata != null) { var table = new SingleFileTable(tableName, _storageProvider, metadata.Value); + // B6: forward the database config's fixed-width flag on reopen too — a legacy JSON + // table opened with FixedWidthRecordLayout is auto-migrated on first load. (The + // on-disk binary format is still authoritative for reading regardless of config.) + table.SetFixedWidthRecords(_options.DatabaseConfig?.FixedWidthRecordLayout ?? false); _tables[tableName] = table; } } @@ -1084,6 +1110,9 @@ private void ExecuteCreateTableInternal(string sql) table.ColumnCheckExpressions = columnCheckExpressions; table.TableCheckConstraints = tableCheckConstraints; table.UniqueConstraints = uniqueConstraints; + // B6: forward the database config's fixed-width flag so new single-file tables store + // binary fixed-width records (with the overflow block) instead of JSON rows. + table.SetFixedWidthRecords(_options.DatabaseConfig?.FixedWidthRecordLayout ?? false); _tables[tableName] = table; // Register table schema with the directory manager so it persists on disk diff --git a/src/SharpCoreDB/Interfaces/IDatabase.cs b/src/SharpCoreDB/Interfaces/IDatabase.cs index 9a3f56f2..cd7dd7d0 100644 --- a/src/SharpCoreDB/Interfaces/IDatabase.cs +++ b/src/SharpCoreDB/Interfaces/IDatabase.cs @@ -339,4 +339,18 @@ SharpCoreDB.DataStructures.StructRowQueryEnumerable ExecuteQueryStruct(string sq /// Thrown when the database is read-only or a row /// cannot be located while migrating. int MigrateLegacyUlids(); + + /// + /// B5: migrates a legacy (1.x, variable-length records) table to the fixed-width record layout + /// (out-of-line overflow arena, 2.0). Current rows are re-read with the legacy codec, + /// re-serialized as fixed-width records and the primary-key / hash indexes are rebuilt. The + /// migrated record format is persisted in metadata so it survives reopen without the config flag. + /// + /// The table to migrate. + /// The number of rows migrated (0 when the table is already fixed-width or empty). + /// Thrown when the database or table is read-only, + /// or the table does not exist. + /// Thrown for non-columnar tables (page-based) and + /// single-file tables, which do not support the fixed-width record layout. + int MigrateTableToFixedWidth(string tableName); } diff --git a/src/SharpCoreDB/Interfaces/IStorage.cs b/src/SharpCoreDB/Interfaces/IStorage.cs index abf8965f..03b27a39 100644 --- a/src/SharpCoreDB/Interfaces/IStorage.cs +++ b/src/SharpCoreDB/Interfaces/IStorage.cs @@ -104,6 +104,15 @@ public interface IStorage /// bool OverwriteRecordAt(string path, long offset, byte[] data); + /// + /// Like for the case where the caller guarantees + /// has the same byte length as the stored record payload (an in-place + /// field patch built from the existing row). Implementations may skip the length-prefix + /// read/verification; the default routes to . + /// + bool OverwriteRecordAtSameLength(string path, long offset, byte[] data) => + OverwriteRecordAt(path, offset, data); + /// /// Appends multiple binary data blocks to a file in a single batch operation (used for batch inserts). /// diff --git a/src/SharpCoreDB/Interfaces/IStorageEngine.cs b/src/SharpCoreDB/Interfaces/IStorageEngine.cs index 49a7a11e..6c2262cc 100644 --- a/src/SharpCoreDB/Interfaces/IStorageEngine.cs +++ b/src/SharpCoreDB/Interfaces/IStorageEngine.cs @@ -55,6 +55,16 @@ public interface IStorageEngine : IDisposable /// bool TryUpdateInPlace(string tableName, long storageReference, byte[] newData); + /// + /// Same contract as for the common case where the caller has + /// already read the existing record and guarantees has the exact same + /// byte length as the stored payload (e.g. an in-place field patch built from the existing row). + /// Engines that can use the guarantee skip the extra length-prefix read; the default routes to + /// so existing implementations keep working unchanged. + /// + bool TryUpdateInPlaceSameLength(string tableName, long storageReference, byte[] newData) => + TryUpdateInPlace(tableName, storageReference, newData); + /// /// Deletes a record at the specified storage reference. /// diff --git a/src/SharpCoreDB/Interfaces/ITable.cs b/src/SharpCoreDB/Interfaces/ITable.cs index 6e4fba80..3fe637b6 100644 --- a/src/SharpCoreDB/Interfaces/ITable.cs +++ b/src/SharpCoreDB/Interfaces/ITable.cs @@ -50,6 +50,14 @@ public interface ITable /// bool HasInternalRowId { get; } + /// + /// B5: gets whether this table uses the fixed-width record layout (out-of-line overflow arena). + /// Legacy (1.x) tables store variable-length records and report false until migrated + /// with (or auto-migrated on reopen when the + /// database config opts into FixedWidthRecordLayout). + /// + bool IsFixedWidthRecords => false; + /// /// Gets whether columns are auto-generated. /// @@ -199,12 +207,46 @@ void ApplySchema(TableSchemaDefinition schema) { } /// The updates to apply. void Update(string? where, Dictionary updates); + /// + /// Updates rows matching and returns the number of affected rows. + /// The default implementation preserves the historic two-pass behavior (Select for the count, + /// then Update) so third-party implementations keep working without + /// changes. Core implementations (, ) override + /// this with a single-pass path so the SQL UPDATE path does not materialize every matching row + /// just to count it. + /// + /// The where clause string. + /// The updates to apply. + /// The number of affected rows. + int UpdateAffectedCount(string? where, Dictionary updates) + { + var count = Select(where, null, true, false).Count; + Update(where, updates); + return count; + } + /// /// Deletes rows from the table. /// /// The where clause string. void Delete(string? where); + /// + /// Deletes rows matching and returns the affected (pre-delete) rows. + /// The default implementation preserves the historic two-pass behavior (Select to capture the + /// rows, then Delete) so third-party implementations keep working without + /// changes. Core implementations (, ) override + /// this with a single-pass path so the SQL DELETE path does not materialize the same rows twice. + /// + /// The where clause string. + /// The rows that were deleted (pre-delete values). + List> DeleteAffectedRows(string? where) + { + var rows = Select(where, null, true, false); + Delete(where); + return rows; + } + /// /// Finds a single row by primary key value, bypassing SQL parsing. /// Returns null if not found. diff --git a/src/SharpCoreDB/Services/SqlParser.Core.cs b/src/SharpCoreDB/Services/SqlParser.Core.cs index 0bf61b8f..52387934 100644 --- a/src/SharpCoreDB/Services/SqlParser.Core.cs +++ b/src/SharpCoreDB/Services/SqlParser.Core.cs @@ -306,21 +306,86 @@ private bool TryExecuteSimpleSelect( if (!this.tables.TryGetValue(simple.TableName, out var table)) return false; - string whereStr; - if (!TryBuildSimpleWhereStr(simple, parameters, out whereStr)) - return false; + if (simple.WhereColumn is not null) + { + // B8: direct hash-index point lookup for `WHERE indexed_col = @param|literal`. This + // skips building a WHERE string and re-parsing it inside SelectInternal — the single + // biggest overhead difference vs the Direct API (FindByIndex) on point reads. + if (TryResolveWhereValue(simple, parameters, out var whereValue) && whereValue is not null) + { + if (table is DataStructures.Table concrete && + concrete.TrySelectIndexedPointLookup(simple.WhereColumn, whereValue, out var indexRows)) + { + if (simple.Offset.HasValue && simple.Offset.Value > 0) + indexRows = [.. indexRows.Skip(simple.Offset.Value)]; - var rows = table.Select(whereStr, simple.OrderByColumn, simple.OrderByAscending, noEncrypt: false); + if (simple.Limit.HasValue && simple.Limit.Value > 0) + indexRows = [.. indexRows.Take(simple.Limit.Value)]; - // Apply LIMIT/OFFSET exactly like the legacy ExecuteSelectQuery path. - if (simple.Offset.HasValue && simple.Offset.Value > 0) - rows = [.. rows.Skip(simple.Offset.Value)]; + results = concrete.DeduplicateByPrimaryKey(indexRows); + return true; + } + } - if (simple.Limit.HasValue && simple.Limit.Value > 0) - rows = [.. rows.Take(simple.Limit.Value)]; + // Fallback: build the WHERE string exactly like the legacy binder and let the table + // scan/index machinery resolve it (non-indexed columns, non-binary collations, …). + if (!TryBuildSimpleWhereStr(simple, parameters, out var whereStr)) + return false; - results = table is Table concreteTable ? concreteTable.DeduplicateByPrimaryKey(rows) : rows; - return true; + var rows = table.Select(whereStr, simple.OrderByColumn, simple.OrderByAscending, noEncrypt: false); + + // Apply LIMIT/OFFSET exactly like the legacy ExecuteSelectQuery path. + if (simple.Offset.HasValue && simple.Offset.Value > 0) + rows = [.. rows.Skip(simple.Offset.Value)]; + + if (simple.Limit.HasValue && simple.Limit.Value > 0) + rows = [.. rows.Take(simple.Limit.Value)]; + + results = table is DataStructures.Table concreteTable ? concreteTable.DeduplicateByPrimaryKey(rows) : rows; + return true; + } + + // No WHERE (full scan) — the legacy parser handles this shape. + return false; + } + + /// + /// B8: resolves the simple-SELECT WHERE value as an object (parameter value or literal) + /// for the direct hash-index point lookup. + /// + private static bool TryResolveWhereValue( + SimpleSelectPlan simple, + Dictionary? parameters, + out object? value) + { + value = null; + + if (simple.WhereParameter is not null) + { + if (parameters is null || parameters.Count == 0) + return false; + + if (!TryResolveParameterValue(parameters, simple.WhereParameter, out value)) + return false; + + return value is not null && value != DBNull.Value; + } + + if (simple.WhereLiteral is not null) + { + var literal = simple.WhereLiteral; + if (literal.Length >= 2 && + ((literal[0] == '\'' && literal[^1] == '\'') || + (literal[0] == '"' && literal[^1] == '"'))) + { + literal = literal[1..^1]; + } + + value = literal; + return true; + } + + return false; } /// diff --git a/src/SharpCoreDB/Services/SqlParser.DML.cs b/src/SharpCoreDB/Services/SqlParser.DML.cs index 668455d5..d0ffe9d7 100644 --- a/src/SharpCoreDB/Services/SqlParser.DML.cs +++ b/src/SharpCoreDB/Services/SqlParser.DML.cs @@ -1548,10 +1548,10 @@ private void ExecuteUpdate(string sql, IWAL? wal) } } - // Count affected rows before update for change tracking - var affectedCount = table.Select(whereClause, orderBy: null, asc: true, noEncrypt: false).Count; - - table.Update(whereClause, updates); + // Issue #8: single-pass — UpdateAffectedCount applies the update AND returns the affected + // count, so change-tracking no longer needs a separate full Select pass (the old code + // materialized every matching row just to count them). + var affectedCount = table.UpdateAffectedCount(whereClause, updates); _lastChanges = affectedCount; _totalChanges += affectedCount; @@ -1588,8 +1588,10 @@ private void ExecuteDelete(string sql, IWAL? wal) var whereClause = deleteMatch.Groups[2].Value.Trim(); - // Capture rows before deletion for RETURNING and change tracking - var affectedRows = table.Select(whereClause, orderBy: null, asc: true, noEncrypt: false); + // Issue #8: single-pass delete — DeleteAffectedRows deletes AND returns the affected rows, + // so RETURNING + affected-count no longer need a separate full Select pass (the old code + // materialized matching rows twice: once here and once inside Table.Delete). + var affectedRows = table.DeleteAffectedRows(whereClause); var affectedCount = affectedRows.Count; if (returningColumns is not null) @@ -1597,7 +1599,6 @@ private void ExecuteDelete(string sql, IWAL? wal) _pendingQueryResults = ProjectReturningRows(affectedRows, returningColumns); } - table.Delete(whereClause); _lastChanges = affectedCount; _totalChanges += affectedCount; diff --git a/src/SharpCoreDB/Services/Storage.Append.cs b/src/SharpCoreDB/Services/Storage.Append.cs index 96638e1e..cb3b9e98 100644 --- a/src/SharpCoreDB/Services/Storage.Append.cs +++ b/src/SharpCoreDB/Services/Storage.Append.cs @@ -42,6 +42,18 @@ public partial class Storage private readonly Dictionary> bufferedAppends = new(); private readonly Dictionary cachedFileLengths = new(); // ✅ NEW: Cache file lengths + // ✅ B7: Write-behind log for in-place overwrites made inside a transaction. The original + // bytes stay on disk until commit (nothing is overwritten early), so rollback is simply + // dropping this buffer — no undo data needs to be stored. On commit the buffered records + // are written once per file. Previously every update inside ExecuteBatchSQL fell back to + // append because OverwriteRecordAt refused to write inside a transaction. + private readonly ConcurrentDictionary> bufferedOverwrites = new(StringComparer.Ordinal); + + // Base file length captured at the first buffered operation of the transaction. In-place + // overwrites are only safe below this boundary (records already flushed to disk); offsets + // at or above it belong to still-buffered appends and must fall back to append. + private readonly Dictionary bufferedFileBaseLengths = new(StringComparer.Ordinal); + // ✅ NEW: Tracks which buffered files still need the 8-byte magic header written on flush // (only for brand-new files created while encryption is enabled). private readonly HashSet headerPendingFiles = new(StringComparer.Ordinal); @@ -206,6 +218,10 @@ private void EnsureAppendInitialized(string path, bool encryptWrites) long fileLength = fileExists ? new FileInfo(path).Length : 0; long initialLength = fileLength; + // B7: remember the flushed boundary for this file so in-place overwrites inside + // the transaction only touch records that already exist on disk. + bufferedFileBaseLengths[path] = fileLength; + // ✅ Known Issue 1 FIX: Brand-new encrypted files (absent OR empty, since DDL // pre-creates empty .dat files) start after the 8-byte magic header so buffered // record positions match the real on-disk offsets after FlushBufferedAppends. @@ -239,6 +255,12 @@ private static void WriteEncryptedHeader(FileStream fs) // and the temp directory can be deleted after the database is disposed. private readonly ConcurrentDictionary _readHandleCache = new(); + // B7: cached write handles for in-place record overwrites (OverwriteRecordAt). Without this, + // every in-place UPDATE inside a transaction opened a fresh FileStream per call — measurably + // slower than the buffered-append path (10k updates: 0.26s → 1.3s). A handle per table file + // brings the overwrite path back to a single open per file. + private readonly ConcurrentDictionary _writeHandleCache = new(); + /// /// Returns (or opens) a cached for random-access reads on . /// @@ -252,6 +274,57 @@ private SafeFileHandle GetOrOpenReadHandle(string path) => FileShare.ReadWrite | FileShare.Delete, FileOptions.None)); + /// + /// B7: returns (or opens) a cached for random-access in-place + /// overwrites on . Sharing flags match the read handle so readers see + /// overwritten bytes immediately. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private SafeFileHandle GetOrOpenWriteHandle(string path) => + _writeHandleCache.GetOrAdd(path, static p => + File.OpenHandle( + p, + FileMode.Open, + FileAccess.Write, + FileShare.ReadWrite | FileShare.Delete, + FileOptions.None)); + + /// + /// B7: performs an in-place overwrite of a length-prefixed record. Table files (.dat) use the + /// cached write handle; overflow-arena files (.ovf) open a short-lived stream because the + /// arena owns its own append/reuse streams and a lingering handle would conflict with them. + /// + private void WriteRecordInPlace(string path, long offset, ReadOnlySpan lengthPrefix, ReadOnlySpan record) + { + if (path.EndsWith(".ovf", StringComparison.OrdinalIgnoreCase)) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.None); + fs.Position = offset; + fs.Write(lengthPrefix); + fs.Write(record); + } + else + { + SafeFileHandle writeHandle = GetOrOpenWriteHandle(path); + RandomAccess.Write(writeHandle, lengthPrefix, offset); + RandomAccess.Write(writeHandle, record, offset + 4); + } + } + + /// + /// Closes all cached write handles (paired with ). + /// + public void CloseWriteHandles() + { + foreach (var (key, handle) in _writeHandleCache) + { + if (_writeHandleCache.TryRemove(key, out _)) + { + handle.Dispose(); + } + } + } + /// /// Closes and removes all cached read handles. /// Call this when the database is disposed so temp directories can be deleted on Windows. @@ -265,6 +338,8 @@ public void CloseReadHandles() handle.Dispose(); } } + + CloseWriteHandles(); } /// @@ -301,7 +376,9 @@ public long AppendBytes(string path, byte[] data) } // Normal append (not in transaction) - write immediately - using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read, 4096, FileOptions.WriteThrough); + // B7: FileShare.ReadWrite|Delete so the cached in-place-overwrite write handle and the + // append path can coexist (a FileShare.Read open would fail while the write handle is open). + using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.WriteThrough); long position = fs.Position; // ✅ Known Issue 1 FIX: brand-new encrypted files (position 0) receive the 8-byte @@ -345,12 +422,7 @@ public bool OverwriteRecordAt(string path, long offset, byte[] data) { ArgumentNullException.ThrowIfNull(data); - // In-place overwrites of already-flushed records cannot be buffered/rolled back with the - // append-only transaction machinery — fall back to append semantics in a transaction. - if (IsInTransaction) - { - return false; - } + bool inTransaction = IsInTransaction; bool encryptWrites = ShouldEncryptWrites(path); byte[] record = EncryptRecord(data, encryptWrites); @@ -392,13 +464,62 @@ public bool OverwriteRecordAt(string path, long offset, byte[] data) return false; } - // Overwrite length prefix + payload in place; the file length is unchanged so all - // following records keep their offsets. - BinaryPrimitives.WriteInt32LittleEndian(lengthBuffer, recordLength); - using var fs = new FileStream(path, FileMode.Open, FileAccess.Write, FileShare.Read, 4096, FileOptions.None); - fs.Position = offset; - fs.Write(lengthBuffer); - fs.Write(record.AsSpan()); + // B7: inside a transaction, buffer the overwrite (write-behind) instead of writing to + // disk per row; outside one, write it immediately. Nothing is written to disk before + // commit in the transactional case, so rollback needs no undo data. + return BufferOrWriteOverwriteInPlace(path, offset, record); + } + catch (IOException) + { + return false; + } + } + + /// + /// B7: buffers (inside a transaction) or writes (outside one) an in-place overwrite of a + /// length-prefixed record whose payload is (already encrypted when + /// applicable). The caller guarantees the new payload length equals the stored payload length, + /// so no length-prefix read/verification is needed. + /// + private bool BufferOrWriteOverwriteInPlace(string path, long offset, byte[] record) + { + bool inTransaction = IsInTransaction; + int recordLength = record.Length; + + try + { + if (inTransaction) + { + // Only records already flushed to disk (offset below the buffered-appends boundary) + // can be overwritten in place; still-buffered records fall back to append. + if (!bufferedFileBaseLengths.TryGetValue(path, out long baseLength)) + { + baseLength = File.Exists(path) ? new FileInfo(path).Length : 0; + bufferedFileBaseLengths[path] = baseLength; + } + + if (offset + 4 + recordLength > baseLength) + { + return false; + } + + lock (appendLock) + { + if (!bufferedOverwrites.TryGetValue(path, out var overwrites)) + { + overwrites = new Dictionary(); + bufferedOverwrites[path] = overwrites; + } + + overwrites[offset] = record; + } + } + else + { + Span lengthBuffer = stackalloc byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(lengthBuffer, recordLength); + WriteRecordInPlace(path, offset, lengthBuffer, record); + } } catch (IOException) { @@ -415,7 +536,27 @@ public bool OverwriteRecordAt(string path, long offset, byte[] data) return true; } + /// + /// Overwrites a length-prefixed record in place at when the caller + /// guarantees the new plaintext payload has the same length as the stored one (e.g. an in-place + /// field patch built from the existing record bytes). Skips the length-prefix read/verification + /// that performs — one less per-row syscall in the batch-DML + /// hot path. + /// + /// The table data file path. + /// The physical file offset of the record's 4-byte length prefix. + /// The plaintext record data to write (same length as the stored payload). + /// True when the record was overwritten/buffered in place. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public bool OverwriteRecordAtSameLength(string path, long offset, byte[] data) + { + ArgumentNullException.ThrowIfNull(data); + + bool encryptWrites = ShouldEncryptWrites(path); + byte[] record = EncryptRecord(data, encryptWrites); + return BufferOrWriteOverwriteInPlace(path, offset, record); + } /// [MethodImpl(MethodImplOptions.AggressiveOptimization)] @@ -440,7 +581,7 @@ public long[] AppendBytesMultiple(string path, List dataBlocks) // Normal batch append (not in transaction) - write immediately var positions = new long[dataBlocks.Count]; - using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read, 65536, FileOptions.WriteThrough); + using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 65536, FileOptions.WriteThrough); Span lengthBuffer = stackalloc byte[4]; @@ -490,6 +631,9 @@ internal void FlushBufferedAppends() { if (bufferedAppends.Count == 0) { + // Buffered in-place overwrites are flushed by CommitSync/CommitAsync, NOT by + // intermediate flushes (FlushTransactionBuffer) — an intermediate flush must not + // make rollback impossible. return; } @@ -509,7 +653,7 @@ internal void FlushBufferedAppends() if (appends.Count == 0) continue; - using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read, 65536); + using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, 65536); // ✅ Known Issue 1 FIX: Write the 8-byte magic header when this was a // brand-new file created while encryption is enabled. @@ -532,6 +676,21 @@ internal void FlushBufferedAppends() bufferedAppends.Clear(); cachedFileLengths.Clear(); headerPendingFiles.Clear(); + bufferedFileBaseLengths.Clear(); + } + } + + /// + /// B7: flushes buffered appends AND buffered in-place overwrites. Only the true commit path + /// (CommitSync/CommitAsync) calls this — intermediate flushes keep overwrites buffered so + /// rollback stays possible. + /// + internal void FlushBufferedAppendsAndOverwrites() + { + lock (appendLock) + { + FlushBufferedAppends(); + FlushBufferedOverwrites(); } } @@ -595,21 +754,89 @@ public void FlushTransactionBuffer() /// /// Clears all buffered appends during transaction rollback. + /// B7: in-place overwrites made inside the transaction are restored first (undo log), so a + /// rollback returns the table file to its pre-transaction state. /// internal void ClearBufferedAppends() { lock (appendLock) { + RestoreBufferedOverwrites(); + bufferedAppends.Clear(); cachedFileLengths.Clear(); // ✅ Clear cache too headerPendingFiles.Clear(); // ✅ Clear pending header markers on rollback + bufferedFileBaseLengths.Clear(); } } + /// + /// B7: discards the buffered in-place overwrites on rollback. Because overwrites are + /// write-behind (nothing was written to disk), the file already holds the original bytes — + /// no restore work is needed. + /// + private void RestoreBufferedOverwrites() + { + bufferedOverwrites.Clear(); + } + + /// + /// B7: writes every buffered in-place overwrite to disk. Called when the transaction is + /// committed (after the buffered appends are flushed). + /// + private void FlushBufferedOverwrites() + { + foreach (var (path, overwrites) in bufferedOverwrites) + { + if (overwrites.Count == 0) + { + continue; + } + + try + { + Span lengthPrefix = stackalloc byte[4]; + foreach (var (offset, record) in overwrites) + { + BinaryPrimitives.WriteInt32LittleEndian(lengthPrefix, record.Length); + WriteRecordInPlace(path, offset, lengthPrefix, record); + } + } + catch (IOException) + { + // The in-place overwrite is best-effort; the append path remains authoritative. + } + } + + bufferedOverwrites.Clear(); + } + /// [MethodImpl(MethodImplOptions.AggressiveOptimization)] public byte[]? ReadBytesFrom(string path, long offset) { + // B7: inside a transaction, a buffered in-place overwrite takes precedence over the disk + // version (the overwrite is written to disk only at commit). The buffer holds the payload + // only (its length is the record's stored length). + if (!bufferedOverwrites.IsEmpty && + bufferedOverwrites.TryGetValue(path, out var buffered) && + buffered.TryGetValue(offset, out var newRecord) && + newRecord.Length > 0) + { + if (newRecord.Length <= MaxRecordSize) + { + byte[] bufferedPayload = new byte[newRecord.Length]; + Buffer.BlockCopy(newRecord, 0, bufferedPayload, 0, newRecord.Length); + + if (UseRecordEncryption && FileHasEncryptedHeader(path)) + { + return DecryptRecord(bufferedPayload); + } + + return bufferedPayload; + } + } + // PERF: Use cached SafeFileHandle + RandomAccess instead of opening a new // FileStream for every point-lookup call. Reusing a handle drops kernel // overhead from ~50-100 µs to a single pread/ReadFile syscall (~1-5 µs). diff --git a/src/SharpCoreDB/Services/Storage.Core.cs b/src/SharpCoreDB/Services/Storage.Core.cs index d3a5071b..d27dc7ba 100644 --- a/src/SharpCoreDB/Services/Storage.Core.cs +++ b/src/SharpCoreDB/Services/Storage.Core.cs @@ -84,7 +84,7 @@ public async Task CommitAsync() } // ✅ CRITICAL FIX: Flush buffered appends BEFORE closing transaction! - FlushBufferedAppends(); + FlushBufferedAppendsAndOverwrites(); // Flush all buffered writes to disk this.transactionBuffer.Flush(); @@ -108,7 +108,7 @@ public void CommitSync() throw new InvalidOperationException("No active transaction to commit"); } - FlushBufferedAppends(); + FlushBufferedAppendsAndOverwrites(); this.transactionBuffer.Flush(); } } @@ -129,10 +129,9 @@ public bool IsInTransaction [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - lock (this.transactionLock) - { - return this.transactionBuffer.IsInTransaction; - } + // B7: lock-free bool read (atomic in .NET). The transaction lock guards writes; a + // stale-by-one-frame read is harmless on this hot path (per-row update check). + return this.transactionBuffer.IsInTransaction; } } diff --git a/src/SharpCoreDB/Services/Storage.ReadWrite.cs b/src/SharpCoreDB/Services/Storage.ReadWrite.cs index a5bf981a..075ff46d 100644 --- a/src/SharpCoreDB/Services/Storage.ReadWrite.cs +++ b/src/SharpCoreDB/Services/Storage.ReadWrite.cs @@ -86,7 +86,15 @@ public void Write(string path, string data) return null; } - byte[] fileData = File.ReadAllBytes(path); + byte[] fileData; + // B7: open with FileShare.ReadWrite so the cached write handle (in-place overwrites) can + // coexist with full-file reads. File.ReadAllBytes defaults to FileShare.Read, which fails + // while a write handle is open. + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + { + fileData = new byte[fs.Length]; + fs.ReadExactly(fileData); + } var effectiveNoEncrypt = noEncrypt || this.noEncryption; if (effectiveNoEncrypt) diff --git a/src/SharpCoreDB/SingleFileTable.cs b/src/SharpCoreDB/SingleFileTable.cs index 7ff161fd..325307e1 100644 --- a/src/SharpCoreDB/SingleFileTable.cs +++ b/src/SharpCoreDB/SingleFileTable.cs @@ -50,9 +50,23 @@ private static JsonSerializerOptions CreateJsonOptions() private readonly DatabaseConfig? _config; private readonly Lock _tableLock = new(); private readonly string _dataBlockName = $"table:{tableName}:data"; + private readonly string _overflowBlockName = $"table:{tableName}:overflow"; private List> _rowCache = []; private bool _cacheLoaded; + // Fixed-width record layout (out-of-line overflow): binary records in the data block with + // variable-length values in the overflow block. The on-disk format is detected on load; the + // config flag only selects the format for NEW tables and triggers JSON → binary migration. + private bool _fixedWidthRecords; + private FixedWidthRecordLayout? _fixedWidthLayout; + private SingleFileOverflowArena? _overflowArena; + + // Issue A1: primary-key hash index for O(1) point lookups (FindByPrimaryKey / + // SELECT … WHERE pk = value / UpdateByPrimaryKey / DeleteByPrimaryKey). Keyed by the ordinal + // string form of the PK column value (the same comparison FindByPrimaryKey already used). + // Maintained incrementally on every row mutation and rebuilt on cache load / rollback. + private readonly Dictionary>> _pkIndex = new(StringComparer.Ordinal); + // Comparison operators in precedence order for simple-condition fast-path parsing // (must match EvaluateSingleCondition's ordering: >= before >, etc.). private static readonly string[] SingleFileConditionOperators = [">=", "<=", "!=", "<>", "=", ">", "<"]; @@ -90,6 +104,7 @@ public SingleFileTable(string tableName, IStorageProvider storageProvider, Datab : this(tableName, storageProvider) { _config = config; + _fixedWidthRecords = config?.FixedWidthRecordLayout ?? false; } /// @@ -214,6 +229,7 @@ public void Insert(Dictionary row) { ApplyDefaults(row); _rowCache.Add(row); + IndexRow(row); _isDirty = true; } @@ -240,6 +256,7 @@ public long[] InsertBatch(List> rows) var row = rows[i]; ApplyDefaults(row); _rowCache.Add(row); + IndexRow(row); positions[i] = _rowCache.Count - 1; } @@ -278,6 +295,14 @@ public List> Select(string? where, string? orderBy, b lock (_tableLock) { + // Issue A1 fast path: an exact `pk = value` equality resolves through the primary-key + // hash index (O(1)) instead of a full cache scan. Candidates are still verified with + // the full predicate so semantics are identical to the scan path. + if (TryGetPkLookupResults(where, orderBy, asc, out var pkResults)) + { + return pkResults; + } + // PERF: evaluate WHERE/ORDER BY against the cached rows (read-only) and // materialize (defensive-copy) only the surviving rows. Previously every // row was copied up-front, so a point lookup on a large cache copied the @@ -293,6 +318,31 @@ public List> Select(string? where, string? orderBy, b } } + private bool TryGetPkLookupResults(string? where, string? orderBy, bool asc, out List> results) + { + results = []; + var condition = NormalizeWhereCondition(where); + + if (!IsPkIndexLookupSafe() || !TryParsePkEquality(condition, out var pkValue) || pkValue is null) + { + return false; + } + + results = _pkIndex.TryGetValue(pkValue, out var candidates) + ? candidates.Where(row => EvaluateCondition(row, condition)) + .Select(row => new Dictionary(row)).ToList() + : []; + + if (!string.IsNullOrWhiteSpace(orderBy)) + { + results = asc + ? [.. results.OrderBy(row => GetOrderKey(row, orderBy))] + : [.. results.OrderByDescending(row => GetOrderKey(row, orderBy))]; + } + + return true; + } + private static string? NormalizeWhereCondition(string? where) { // Strip leading WHERE keyword if present @@ -337,7 +387,10 @@ private static IEnumerable> ApplyOrderBy( => row.TryGetValue(orderBy, out var value) ? value : null; /// - public void Update(string? where, Dictionary updates) + public void Update(string? where, Dictionary updates) => UpdateAffectedCount(where, updates); + + /// + public int UpdateAffectedCount(string? where, Dictionary updates) { ArgumentNullException.ThrowIfNull(updates); EnsureCacheLoaded(); @@ -349,18 +402,35 @@ public void Update(string? where, Dictionary updates) condition = condition[6..].Trim(); } + bool updatesTouchPk = PrimaryKeyIndex >= 0 && + updates.Keys.Any(k => string.Equals(k, Columns[PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)); + + int affected = 0; lock (_tableLock) { foreach (var row in _rowCache) { if (string.IsNullOrWhiteSpace(condition) || EvaluateCondition(row, condition)) { + string? oldPkKey = updatesTouchPk ? GetPkKey(row) : null; + foreach (var update in updates) { row[update.Key] = update.Value; } + if (oldPkKey is not null) + { + var newPkKey = GetPkKey(row); + if (!string.Equals(oldPkKey, newPkKey, StringComparison.Ordinal)) + { + UnindexRow(row, oldPkKey); + IndexRow(row); + } + } + _isDirty = true; + affected++; } } } @@ -370,6 +440,8 @@ public void Update(string? where, Dictionary updates) { FlushCache(); } + + return affected; } /// @@ -384,6 +456,7 @@ public void UpdateBatch(Dictionary> updates) if (PrimaryKeyIndex < 0) return; var pkColumn = Columns[PrimaryKeyIndex]; + bool updatesTouchPk = updates.Keys.Any(k => string.Equals(k?.ToString(), pkColumn, StringComparison.OrdinalIgnoreCase)); lock (_tableLock) { @@ -399,11 +472,23 @@ public void UpdateBatch(Dictionary> updates) continue; } + string? oldPkKey = updatesTouchPk ? GetPkKey(row) : null; + foreach (var update in rowUpdates) { row[update.Key] = update.Value; } + if (oldPkKey is not null) + { + var newPkKey = GetPkKey(row); + if (!string.Equals(oldPkKey, newPkKey, StringComparison.Ordinal)) + { + UnindexRow(row, oldPkKey); + IndexRow(row); + } + } + _isDirty = true; } } @@ -432,10 +517,21 @@ public void Delete(string? where) if (string.IsNullOrWhiteSpace(condition)) { _rowCache.Clear(); + _pkIndex.Clear(); } else { - _rowCache.RemoveAll(row => EvaluateCondition(row, condition)); + // Remove matching rows while keeping the primary-key index in sync + // (reverse iteration avoids index-shift issues). + for (int i = _rowCache.Count - 1; i >= 0; i--) + { + var row = _rowCache[i]; + if (EvaluateCondition(row, condition)) + { + UnindexRow(row); + _rowCache.RemoveAt(i); + } + } } _isDirty = true; @@ -448,6 +544,51 @@ public void Delete(string? where) } } + /// + public List> DeleteAffectedRows(string? where) + { + EnsureCacheLoaded(); + + // Strip leading WHERE keyword if present + var condition = where?.Trim(); + if (condition is not null && condition.StartsWith("WHERE ", StringComparison.OrdinalIgnoreCase)) + { + condition = condition[6..].Trim(); + } + + lock (_tableLock) + { + List> toDelete; + if (string.IsNullOrWhiteSpace(condition)) + { + toDelete = [.. _rowCache]; + } + else + { + toDelete = _rowCache.Where(row => EvaluateCondition(row, condition)).ToList(); + } + + if (toDelete.Count > 0) + { + foreach (var row in toDelete) + { + UnindexRow(row); + _rowCache.Remove(row); + } + + _isDirty = true; + } + + // ✅ CRITICAL FIX: Only flush if not in transaction + if (AutoFlush && _isDirty && !_isInTransaction) + { + FlushCache(); + } + + return toDelete; + } + } + /// /// /// ✅ FIX (Known Issue 3): Point lookups now work in single-file mode via the in-memory @@ -462,19 +603,14 @@ public void Delete(string? where) return null; } - var pkColumn = Columns[PrimaryKeyIndex]; var keyStr = key?.ToString(); lock (_tableLock) { - foreach (var row in _rowCache) + // Issue A1: O(1) primary-key hash index (was an O(N) cache scan). + if (keyStr is not null && _pkIndex.TryGetValue(keyStr, out var rows) && rows.Count > 0) { - if (row.TryGetValue(pkColumn, out var pkValue) && - pkValue is not null && - string.Equals(pkValue.ToString(), keyStr, StringComparison.Ordinal)) - { - return new Dictionary(row); - } + return new Dictionary(rows[0]); } } @@ -499,28 +635,34 @@ public bool UpdateByPrimaryKey(object key, Dictionary updates) return false; } - var pkColumn = Columns[PrimaryKeyIndex]; var keyStr = key?.ToString(); bool found = false; lock (_tableLock) { - foreach (var row in _rowCache) + // Issue A1: O(1) primary-key hash index (was an O(N) cache scan). + if (keyStr is not null && _pkIndex.TryGetValue(keyStr, out var rows) && rows.Count > 0) { - if (!row.TryGetValue(pkColumn, out var pkValue) || pkValue is null || - !string.Equals(pkValue.ToString(), keyStr, StringComparison.Ordinal)) - { - continue; - } + var row = rows[0]; + string? oldPkKey = GetPkKey(row); foreach (var update in updates) { row[update.Key] = update.Value; } + if (oldPkKey is not null) + { + var newPkKey = GetPkKey(row); + if (!string.Equals(oldPkKey, newPkKey, StringComparison.Ordinal)) + { + UnindexRow(row, oldPkKey); + IndexRow(row); + } + } + _isDirty = true; found = true; - break; } } @@ -545,25 +687,19 @@ public bool DeleteByPrimaryKey(object key) return false; } - var pkColumn = Columns[PrimaryKeyIndex]; var keyStr = key?.ToString(); bool found = false; lock (_tableLock) { - for (int i = 0; i < _rowCache.Count; i++) + // Issue A1: O(1) primary-key hash index (was an O(N) cache scan). + if (keyStr is not null && _pkIndex.TryGetValue(keyStr, out var rows) && rows.Count > 0) { - var row = _rowCache[i]; - if (!row.TryGetValue(pkColumn, out var pkValue) || pkValue is null || - !string.Equals(pkValue.ToString(), keyStr, StringComparison.Ordinal)) - { - continue; - } - - _rowCache.RemoveAt(i); + var row = rows[0]; + UnindexRow(row, keyStr); + _rowCache.Remove(row); _isDirty = true; found = true; - break; } } @@ -587,17 +723,76 @@ public void FlushCache() } List> serializableRows; + List> rowsToWrite; lock (_tableLock) { + rowsToWrite = _rowCache.ToList(); serializableRows = _rowCache.Select(ToSerializableRow).ToList(); _isDirty = false; } - // Serialize to byte array to get exact length - var jsonBytes = JsonSerializer.SerializeToUtf8Bytes(serializableRows, JsonOptions); + if (!_fixedWidthRecords) + { + // Legacy JSON row format (write using WriteBlockAsync to properly track data length). + var jsonBytes = JsonSerializer.SerializeToUtf8Bytes(serializableRows, JsonOptions); + _storageProvider.WriteBlockAsync(_dataBlockName, jsonBytes).GetAwaiter().GetResult(); + return; + } + + // B6: binary fixed-width records + out-of-line overflow arena. + var layout = _fixedWidthLayout ??= FixedWidthRecordLayout.Compute(ColumnTypes); + var arena = _overflowArena ??= new SingleFileOverflowArena(); + + var records = new List(rowsToWrite.Count); + foreach (var row in rowsToWrite) + { + records.Add(FixedWidthCodec.SerializeRow(row, Columns, ColumnTypes, layout, arena)); + } + + // Sweep: values that changed (or rows that were deleted) leave their old blocks + // unreferenced — free them so the free-list can reuse them in place on the next flush. + var liveOffsets = new HashSet(); + foreach (var record in records) + { + FixedWidthCodec.CollectVariableOffsets(record, layout, liveOffsets); + } + + arena.FreeUnreferenced(liveOffsets); + + // Copy-on-compact the arena when its dead space grows (freed blocks that were not reused + // in place). The records' variable slots are re-pointed through the compaction mapping. + if (arena.TotalCount >= 32 && arena.LiveCount * 4 < arena.TotalCount) + { + var mapping = arena.Compact(liveOffsets); + if (mapping.Count > 0) + { + var repointed = new List(records.Count); + foreach (var record in records) + { + repointed.Add(FixedWidthCodec.RepointVariableSlots(record, layout, mapping) ?? record); + } - // Write using WriteBlockAsync to properly track data length - _storageProvider.WriteBlockAsync(_dataBlockName, jsonBytes).GetAwaiter().GetResult(); + records = repointed; + } + } + + int total = 0; + foreach (var record in records) + { + total += 4 + record.Length; + } + + var buffer = new byte[total]; + int position = 0; + foreach (var record in records) + { + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(position, 4), record.Length); + record.CopyTo(buffer, position + 4); + position += 4 + record.Length; + } + + _storageProvider.WriteBlockAsync(_dataBlockName, buffer).GetAwaiter().GetResult(); + _storageProvider.WriteBlockAsync(_overflowBlockName, arena.Serialize()).GetAwaiter().GetResult(); } /// @@ -676,6 +871,43 @@ public void CreateBTreeIndex(string indexName, string columnName, bool isUnique /// public void SetDatabase(Database database) { } + /// + /// B6: gets whether this table uses the fixed-width record layout. The on-disk format is + /// detected when the cache is first loaded, so this is accurate even without the config flag. + /// + public bool IsFixedWidthRecords + { + get + { + EnsureCacheLoaded(); + return _fixedWidthRecords; + } + } + + /// Sets the fixed-width record layout flag (used by DDL to forward the database config). + internal void SetFixedWidthRecords(bool value) => _fixedWidthRecords = value; + + /// + /// B6: converts this table from the legacy JSON row format to the binary fixed-width record + /// layout (out-of-line overflow arena). Returns the number of rows written. + /// + public int MigrateToFixedWidth() + { + lock (_tableLock) + { + EnsureCacheLoaded(); + if (_fixedWidthRecords) + { + return 0; + } + + _fixedWidthRecords = true; + _isDirty = true; + FlushCache(); + return _rowCache.Count; + } + } + private readonly Dictionary _columnUsage = new(StringComparer.OrdinalIgnoreCase); private void EnsureCacheLoaded() @@ -696,35 +928,109 @@ private void EnsureCacheLoaded() // are both transparently handled. GetReadStream returns the raw on-disk bytes // when encryption is off, which would hand compressed data (Brotli/GZip marker // bytes) to the JSON parser on reopen — breaking SELECT after reopen. - var jsonBytes = _storageProvider.ReadBlockAsync(_dataBlockName, CancellationToken.None).GetAwaiter().GetResult(); - if (jsonBytes is null || jsonBytes.Length == 0) + var dataBytes = _storageProvider.ReadBlockAsync(_dataBlockName, CancellationToken.None).GetAwaiter().GetResult(); + if (dataBytes is null || dataBytes.Length == 0) { _rowCache = []; _cacheLoaded = true; + RebuildPkIndex(); return; } - // Trim trailing null bytes - var endIndex = jsonBytes.Length; - while (endIndex > 0 && jsonBytes[endIndex - 1] == 0) + // Detect the on-disk format on the RAW bytes: the legacy JSON row array vs binary + // fixed-width records. Trailing-null trimming is ONLY valid for the JSON format — a + // binary record can legitimately end with 0x00 bytes (a variable slot whose arena + // offset's most significant bytes are zero), so binary blocks are parsed untrimmed. + if (IsFixedWidthDataBlock(dataBytes)) { - endIndex--; + // Binary fixed-width records: the on-disk format is authoritative (even when the + // config flag is off, reading must use the binary codec). + _fixedWidthRecords = true; + var overflowBytes = _storageProvider.ReadBlockAsync(_overflowBlockName, CancellationToken.None).GetAwaiter().GetResult(); + _overflowArena = SingleFileOverflowArena.Deserialize(overflowBytes); + var layout = _fixedWidthLayout ??= FixedWidthRecordLayout.Compute(ColumnTypes); + var binaryRows = new List>(); + + long position = 0; + while (position + 4 <= dataBytes.Length) + { + int length = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(dataBytes.AsSpan((int)position, 4)); + if (length <= 0 || position + 4 + length > dataBytes.Length) + { + break; // truncated / corrupt + } + + binaryRows.Add(FixedWidthCodec.DeserializeRow( + dataBytes.AsSpan((int)position + 4, length), Columns, ColumnTypes, layout, _overflowArena)); + position += 4 + length; + } + + _rowCache = binaryRows; } - - if (endIndex == 0) + else { - _rowCache = []; - _cacheLoaded = true; - return; + // Legacy JSON row format (trim historical trailing null padding first). + var endIndex = dataBytes.Length; + while (endIndex > 0 && dataBytes[endIndex - 1] == 0) + { + endIndex--; + } + + if (endIndex == 0) + { + _rowCache = []; + _cacheLoaded = true; + RebuildPkIndex(); + return; + } + + var trimmedJsonBytes = dataBytes.AsSpan(0, endIndex); + var rows = JsonSerializer.Deserialize>>(trimmedJsonBytes, JsonOptions); + _rowCache = rows?.Select(FromSerializableRow).ToList() ?? []; + + // Config opts into fixed-width: convert the in-memory rows to binary on next flush. + if (_fixedWidthRecords) + { + _isDirty = true; + } } - - var trimmedJsonBytes = jsonBytes.AsSpan(0, endIndex); - var rows = JsonSerializer.Deserialize>>(trimmedJsonBytes, JsonOptions); - _rowCache = rows?.Select(FromSerializableRow).ToList() ?? []; + _cacheLoaded = true; + RebuildPkIndex(); } } + /// + /// Detects whether the data block holds binary fixed-width records (every record has exactly + /// the fixed-width slot size) rather than the legacy JSON row array. The on-disk format is + /// authoritative on reopen regardless of the config flag. + /// + private bool IsFixedWidthDataBlock(ReadOnlySpan data) + { + if (ColumnTypes is not { Count: > 0 }) + { + return false; + } + + var layout = _fixedWidthLayout ??= FixedWidthRecordLayout.Compute(ColumnTypes); + long position = 0; + bool any = false; + + while (position + 4 <= data.Length) + { + int length = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(data.Slice((int)position, 4)); + if (length != layout.FixedSize || position + 4 + length > data.Length) + { + return false; + } + + any = true; + position += 4 + length; + } + + return any; + } + private void LoadSchemaFromProvider(string tableName) { if (_storageProvider is not SingleFileStorageProvider provider) @@ -834,6 +1140,7 @@ internal void RollbackTransaction() if (_transactionSnapshot is not null) { _rowCache = _transactionSnapshot.Select(row => new Dictionary(row)).ToList(); + RebuildPkIndex(); } _transactionSnapshot = null; @@ -842,6 +1149,137 @@ internal void RollbackTransaction() } } + /// Returns the ordinal string key for a row's primary key value, or null when the + /// table has no PK / the value is null. + private string? GetPkKey(Dictionary row) + { + if (PrimaryKeyIndex < 0 || PrimaryKeyIndex >= Columns.Count) + return null; + + if (row.TryGetValue(Columns[PrimaryKeyIndex], out var pkValue) && pkValue is not null) + { + return pkValue.ToString(); + } + + return null; + } + + /// Adds a row to the primary-key index (no-op without a PK or null PK). + private void IndexRow(Dictionary row) + { + var key = GetPkKey(row); + if (key is null) + return; + + if (!_pkIndex.TryGetValue(key, out var list)) + { + list = new List>(1); + _pkIndex[key] = list; + } + + list.Add(row); + } + + /// Removes a row from the primary-key index. is the key to + /// remove under (defaults to the row's current PK key) — pass the OLD key when the row's PK + /// value has already been changed. + private void UnindexRow(Dictionary row, string? key = null) + { + key ??= GetPkKey(row); + if (key is null) + return; + + if (_pkIndex.TryGetValue(key, out var list)) + { + list.Remove(row); + if (list.Count == 0) + { + _pkIndex.Remove(key); + } + } + } + + /// Rebuilds the primary-key index from the current row cache (cache load / rollback). + private void RebuildPkIndex() + { + _pkIndex.Clear(); + foreach (var row in _rowCache) + { + IndexRow(row); + } + } + + /// + /// Tries to parse an exact pk = value equality. Returns false for compound / range / + /// special-syntax conditions (the caller falls back to the full scan). + /// + private bool TryParsePkEquality(string condition, out string? value) + { + value = null; + if (PrimaryKeyIndex < 0 || string.IsNullOrWhiteSpace(condition)) + return false; + + var trimmed = condition.Trim(); + if (trimmed.Contains(" AND ", StringComparison.OrdinalIgnoreCase) || + trimmed.Contains(" OR ", StringComparison.OrdinalIgnoreCase) || + trimmed.Contains(" IN ", StringComparison.OrdinalIgnoreCase) || + trimmed.Contains("LIKE", StringComparison.OrdinalIgnoreCase) || + trimmed.Contains("BETWEEN", StringComparison.OrdinalIgnoreCase) || + trimmed.Contains(" IS ", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + int eq = trimmed.IndexOf('='); + if (eq <= 0 || trimmed.IndexOf('=', eq + 1) >= 0) + return false; + + var col = trimmed[..eq].Trim().Trim('"', '[', ']', '`'); + if (!string.Equals(col, Columns[PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + return false; + + var val = trimmed[(eq + 1)..].Trim(); + if (val.Length == 0) + return false; + + if ((val.StartsWith('\'') && val.EndsWith('\'')) || + (val.StartsWith('"') && val.EndsWith('"'))) + { + val = val[1..^1]; + } + + // Normalize numeric literals to the row's canonical ToString() form so `pk = 05` + // resolves the same index key ("5") as `pk = 5` — matching the numeric comparison the + // typed WHERE predicate performs. + switch (ColumnTypes[PrimaryKeyIndex]) + { + case DataType.Integer when int.TryParse(val, out var intVal): + val = intVal.ToString(System.Globalization.CultureInfo.InvariantCulture); + break; + case DataType.Long when long.TryParse(val, out var longVal): + val = longVal.ToString(System.Globalization.CultureInfo.InvariantCulture); + break; + } + + value = val; + return true; + } + + /// + /// Whether the PK column's string form is a stable, canonical representation that can be + /// compared against SQL literals for index lookups. String, Integer, Long, Boolean, Guid and + /// Ulid are canonical; DateTime / Real / Decimal / Blob are not (culture / format), so those + /// fall back to the full scan. + /// + private bool IsPkIndexLookupSafe() + { + if (PrimaryKeyIndex < 0 || PrimaryKeyIndex >= ColumnTypes.Count) + return false; + + return ColumnTypes[PrimaryKeyIndex] is DataType.String or DataType.Integer or DataType.Long + or DataType.Boolean or DataType.Guid or DataType.Ulid; + } + private void ApplyDefaults(Dictionary row) { for (int i = 0; i < Columns.Count; i++) diff --git a/src/SharpCoreDB/Storage/ColumnStore.Aggregates.cs b/src/SharpCoreDB/Storage/ColumnStore.Aggregates.cs index 5b725f96..e73128f4 100644 --- a/src/SharpCoreDB/Storage/ColumnStore.Aggregates.cs +++ b/src/SharpCoreDB/Storage/ColumnStore.Aggregates.cs @@ -222,7 +222,15 @@ private static int SumInt32ParallelSIMD(int[] data) long partialSum = 0; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vsum = Vector512.Zero; + for (; i <= end - Vector512.Count; i += Vector512.Count) + vsum = Vector512.Add(vsum, Vector512.LoadUnsafe(ref data[i])); + for (int j = 0; j < Vector512.Count; j++) + partialSum += vsum[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vsum = Vector256.Zero; for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -250,7 +258,15 @@ private static long SumInt64ParallelSIMD(long[] data) long partialSum = 0; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vsum = Vector512.Zero; + for (; i <= end - Vector512.Count; i += Vector512.Count) + vsum = Vector512.Add(vsum, Vector512.LoadUnsafe(ref data[i])); + for (int j = 0; j < Vector512.Count; j++) + partialSum += vsum[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vsum = Vector256.Zero; for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -278,7 +294,15 @@ private static double SumDoubleParallelSIMD(double[] data) double partialSum = 0; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vsum = Vector512.Zero; + for (; i <= end - Vector512.Count; i += Vector512.Count) + vsum = Vector512.Add(vsum, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + partialSum += vsum[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vsum = Vector256.Zero; for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -306,7 +330,15 @@ private static int MinInt32ParallelSIMD(int[] data) int partialMin = int.MaxValue; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vmin = Vector512.Create(int.MaxValue); + for (; i <= end - Vector512.Count; i += Vector512.Count) + vmin = Vector512.Min(vmin, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmin[j] < partialMin) partialMin = vmin[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vmin = Vector256.Create(int.MaxValue); for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -335,7 +367,15 @@ private static long MinInt64ParallelSIMD(long[] data) long partialMin = long.MaxValue; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vmin = Vector512.Create(long.MaxValue); + for (; i <= end - Vector512.Count; i += Vector512.Count) + vmin = Vector512.Min(vmin, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmin[j] < partialMin) partialMin = vmin[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vmin = Vector256.Create(long.MaxValue); for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -364,7 +404,15 @@ private static double MinDoubleParallelSIMD(double[] data) double partialMin = double.MaxValue; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vmin = Vector512.Create(double.MaxValue); + for (; i <= end - Vector512.Count; i += Vector512.Count) + vmin = Vector512.Min(vmin, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmin[j] < partialMin) partialMin = vmin[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vmin = Vector256.Create(double.MaxValue); for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -393,7 +441,15 @@ private static int MaxInt32ParallelSIMD(int[] data) int partialMax = int.MinValue; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vmax = Vector512.Create(int.MinValue); + for (; i <= end - Vector512.Count; i += Vector512.Count) + vmax = Vector512.Max(vmax, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmax[j] > partialMax) partialMax = vmax[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vmax = Vector256.Create(int.MinValue); for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -422,7 +478,15 @@ private static long MaxInt64ParallelSIMD(long[] data) long partialMax = long.MinValue; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vmax = Vector512.Create(long.MinValue); + for (; i <= end - Vector512.Count; i += Vector512.Count) + vmax = Vector512.Max(vmax, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmax[j] > partialMax) partialMax = vmax[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vmax = Vector256.Create(long.MinValue); for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -451,7 +515,15 @@ private static double MaxDoubleParallelSIMD(double[] data) double partialMax = double.MinValue; int i = start; - if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && (end - start) >= Vector512.Count) + { + var vmax = Vector512.Create(double.MinValue); + for (; i <= end - Vector512.Count; i += Vector512.Count) + vmax = Vector512.Max(vmax, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmax[j] > partialMax) partialMax = vmax[j]; + } + else if (Vector256.IsHardwareAccelerated && (end - start) >= Vector256.Count) { var vmax = Vector256.Create(double.MinValue); for (; i <= end - Vector256.Count; i += Vector256.Count) @@ -475,7 +547,15 @@ private static int SumInt32SIMDDirect(int[] data) long sum = 0; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vsum = Vector512.Zero; + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vsum = Vector512.Add(vsum, Vector512.LoadUnsafe(ref data[i])); + for (int j = 0; j < Vector512.Count; j++) + sum += vsum[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vsum = Vector256.Zero; for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -494,7 +574,15 @@ private static long SumInt64SIMDDirect(long[] data) long sum = 0; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vsum = Vector512.Zero; + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vsum = Vector512.Add(vsum, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + sum += vsum[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vsum = Vector256.Zero; for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -513,7 +601,15 @@ private static double SumDoubleSIMDDirect(double[] data) double sum = 0; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vsum = Vector512.Zero; + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vsum = Vector512.Add(vsum, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + sum += vsum[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vsum = Vector256.Zero; for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -533,7 +629,15 @@ private static int MinInt32SIMDDirect(int[] data) int min = int.MaxValue; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vmin = Vector512.Create(int.MaxValue); + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vmin = Vector512.Min(vmin, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmin[j] < min) min = vmin[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vmin = Vector256.Create(int.MaxValue); for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -553,7 +657,15 @@ private static long MinInt64SIMDDirect(long[] data) long min = long.MaxValue; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vmin = Vector512.Create(long.MaxValue); + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vmin = Vector512.Min(vmin, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmin[j] < min) min = vmin[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vmin = Vector256.Create(long.MaxValue); for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -573,7 +685,15 @@ private static double MinDoubleSIMDDirect(double[] data) double min = double.MaxValue; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vmin = Vector512.Create(double.MaxValue); + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vmin = Vector512.Min(vmin, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmin[j] < min) min = vmin[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vmin = Vector256.Create(double.MaxValue); for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -593,7 +713,15 @@ private static int MaxInt32SIMDDirect(int[] data) int max = int.MinValue; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vmax = Vector512.Create(int.MinValue); + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vmax = Vector512.Max(vmax, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmax[j] > max) max = vmax[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vmax = Vector256.Create(int.MinValue); for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -613,7 +741,15 @@ private static long MaxInt64SIMDDirect(long[] data) long max = long.MinValue; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vmax = Vector512.Create(long.MinValue); + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vmax = Vector512.Max(vmax, Vector512.LoadUnsafe(ref data[i])); + for (int j = 0; j < Vector512.Count; j++) + if (vmax[j] > max) max = vmax[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vmax = Vector256.Create(long.MinValue); for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -633,7 +769,15 @@ private static double MaxDoubleSIMDDirect(double[] data) double max = double.MinValue; int i = 0; - if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) + if (Vector512.IsHardwareAccelerated && data.Length >= Vector512.Count) + { + var vmax = Vector512.Create(double.MinValue); + for (; i <= data.Length - Vector512.Count; i += Vector512.Count) + vmax = Vector512.Max(vmax, Vector512.Create(data.AsSpan(i))); + for (int j = 0; j < Vector512.Count; j++) + if (vmax[j] > max) max = vmax[j]; + } + else if (Vector256.IsHardwareAccelerated && data.Length >= Vector256.Count) { var vmax = Vector256.Create(double.MinValue); for (; i <= data.Length - Vector256.Count; i += Vector256.Count) @@ -648,4 +792,4 @@ private static double MaxDoubleSIMDDirect(double[] data) } #endregion -} +} \ No newline at end of file diff --git a/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs b/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs index c9f34758..bac79e4e 100644 --- a/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs +++ b/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs @@ -132,6 +132,26 @@ public bool TryUpdateInPlace(string tableName, long storageReference, byte[] new return overwritten; } + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public bool TryUpdateInPlaceSameLength(string tableName, long storageReference, byte[] newData) + { + ArgumentNullException.ThrowIfNull(newData); + + // Caller guarantees newData has the same payload length as the stored record (in-place field + // patch), so the storage layer skips the length-prefix read/verification. + var filePath = GetTableFilePath(tableName); + bool overwritten = storage.OverwriteRecordAtSameLength(filePath, storageReference, newData); + + if (overwritten) + { + Interlocked.Increment(ref totalUpdates); + Interlocked.Add(ref bytesWritten, newData.Length); + } + + return overwritten; + } + /// diff --git a/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs b/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs index 1044d003..85627817 100644 --- a/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs +++ b/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs @@ -477,11 +477,15 @@ protected virtual void Dispose(bool disposing) { if (disposing) { + // ✅ CRITICAL FIX: Flush dirty pages before closing the page managers. A single + // INSERT / UPDATE never calls FlushDirtyPages (only CommitAsync/Flush do), so without + // this the page cache is lost on dispose and reopened tables return zero rows. foreach (var manager in tableManagers.Values) { + manager.FlushDirtyPages(); manager.Dispose(); } - + tableManagers.Clear(); } } diff --git a/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs b/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs new file mode 100644 index 00000000..01ed3a13 --- /dev/null +++ b/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs @@ -0,0 +1,232 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Storage.Scdb; + +using SharpCoreDB.DataStructures; +using System.Buffers.Binary; +using System.Collections.Generic; + +/// +/// In-memory overflow arena for single-file (.scdb) fixed-width tables. The arena is serialized to +/// a dedicated provider block (table:{name}:overflow) as a contiguous stream of +/// [length(4)][payload] entries; block offsets are the byte positions of the length prefixes +/// (identical semantics to the directory-mode ). Freed blocks keep their +/// slot so offsets stay valid, are reused in place when a new payload has the exact same length +/// (free-list), and are reclaimed by a copy-on-compact pass when the dead space grows. +/// +public sealed class SingleFileOverflowArena : IOverflowArena +{ + private readonly Dictionary _blocks = new(); + private readonly Dictionary> _freeByLength = new(); + private readonly Dictionary _contentIndex = new(System.StringComparer.Ordinal); + private long _nextOffset; + private int _blockReuses; + + /// Gets the number of times a freed block was reused in place (diagnostics). + public int BlockReuses => _blockReuses; + + /// Gets the number of freed blocks currently tracked for in-place reuse (diagnostics). + public int FreeBlockCount + { + get + { + int total = 0; + foreach (var list in _freeByLength.Values) + { + total += list.Count; + } + + return total; + } + } + + /// Gets the number of currently live blocks. + public int LiveCount + { + get + { + var live = new HashSet(_blocks.Keys); + foreach (var list in _freeByLength.Values) + { + foreach (var offset in list) + { + live.Remove(offset); + } + } + + return live.Count; + } + } + + /// Gets the total number of blocks (live + freed). + public int TotalCount => _blocks.Count; + + /// + /// Frees every block not referenced by the current rows' records. The single-file table + /// re-serializes its whole row cache on every flush, so unreferenced blocks (values that + /// changed or rows that were deleted) become free for exact-length in-place reuse. + /// + public void FreeUnreferenced(IReadOnlyCollection liveOffsets) + { + var live = liveOffsets as HashSet ?? new HashSet(liveOffsets); + foreach (var offset in _blocks.Keys.ToList()) + { + if (!live.Contains(offset)) + { + Free(offset); + } + } + } + + /// + public long Write(byte[] payload) + { + ArgumentNullException.ThrowIfNull(payload); + + // B6: idempotent re-serialization — the single-file table flushes its whole row cache, so + // an unchanged value must not allocate a new block. A live block with the exact same + // payload content is reused (values are immutable payloads, so sharing is safe here; the + // single-file arena never frees a shared block). + var contentKey = System.Text.Encoding.Latin1.GetString(payload); + if (_contentIndex.TryGetValue(contentKey, out var dedupedOffset)) + { + return dedupedOffset; + } + + if (_freeByLength.TryGetValue(payload.Length, out var offsets)) + { + while (offsets.Count > 0) + { + var offset = offsets[^1]; + offsets.RemoveAt(offsets.Count - 1); + if (offsets.Count == 0) + { + _freeByLength.Remove(payload.Length); + } + + _blocks[offset] = payload; + _contentIndex[contentKey] = offset; + _blockReuses++; + return offset; + } + } + + var newOffset = _nextOffset; + _blocks[newOffset] = payload; + _contentIndex[contentKey] = newOffset; + _nextOffset += 4 + payload.Length; + return newOffset; + } + + /// + public byte[]? Read(long offset) => _blocks.TryGetValue(offset, out var payload) ? payload : null; + + /// + public void Free(long offset) + { + if (!_blocks.Remove(offset, out var payload)) + { + return; // already freed (or unknown) — never double-track + } + + var contentKey = System.Text.Encoding.Latin1.GetString(payload); + if (_contentIndex.TryGetValue(contentKey, out var indexedOffset) && indexedOffset == offset) + { + _contentIndex.Remove(contentKey); // only drop the index when this block was its sole owner + } + + if (!_freeByLength.TryGetValue(payload.Length, out var offsets)) + { + offsets = []; + _freeByLength[payload.Length] = offsets; + } + + offsets.Add(offset); + } + + /// Serializes all blocks (live and freed) as a contiguous [length][payload] stream. + public byte[] Serialize() + { + if (_blocks.Count == 0) + { + return []; + } + + var buffer = new byte[checked((int)_nextOffset)]; + foreach (var (offset, payload) in _blocks) + { + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan((int)offset, 4), payload.Length); + payload.CopyTo(buffer, (int)offset + 4); + } + + return buffer; + } + + /// + /// Loads the arena from a serialized provider block. Every block in the file is treated as + /// live; freed blocks that were not reused before a flush are harmless dead weight until the + /// next copy-on-compact pass (no record references them). + /// + public static SingleFileOverflowArena Deserialize(byte[]? data) + { + var arena = new SingleFileOverflowArena(); + if (data is null || data.Length == 0) + { + return arena; + } + + long position = 0; + while (position + 4 <= data.Length) + { + int length = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan((int)position, 4)); + if (length < 0 || position + 4 + length > data.Length) + { + break; // truncated / corrupt + } + + var payload = data.AsSpan((int)position + 4, length).ToArray(); + arena._blocks[position] = payload; + arena._contentIndex[System.Text.Encoding.Latin1.GetString(payload)] = position; + position += 4 + length; + } + + arena._nextOffset = position; + return arena; + } + + /// + /// Copy-on-compact: rewrites the live blocks (those in ) into a + /// fresh arena and returns the old → new offset mapping. Callers must re-point the fixed-width + /// records that reference the moved blocks. Freed blocks are reclaimed and the free-list cleared. + /// + public Dictionary Compact(IReadOnlyCollection activeOffsets) + { + var mapping = new Dictionary(activeOffsets.Count); + var newBlocks = new Dictionary(activeOffsets.Count); + long offset = 0; + + foreach (var activeOffset in activeOffsets) + { + if (_blocks.TryGetValue(activeOffset, out var payload)) + { + newBlocks[offset] = payload; + mapping[activeOffset] = offset; + offset += 4 + payload.Length; + } + } + + _blocks.Clear(); + _freeByLength.Clear(); + _contentIndex.Clear(); + foreach (var (newOffset, payload) in newBlocks) + { + _blocks[newOffset] = payload; + _contentIndex[System.Text.Encoding.Latin1.GetString(payload)] = newOffset; + } + + _nextOffset = offset; + return mapping; + } +} diff --git a/tests/SharpCoreDB.Tests/BatchCanonicalParseTests.cs b/tests/SharpCoreDB.Tests/BatchCanonicalParseTests.cs new file mode 100644 index 00000000..05729392 --- /dev/null +++ b/tests/SharpCoreDB.Tests/BatchCanonicalParseTests.cs @@ -0,0 +1,178 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.IO; +using Xunit; + +/// +/// Phase-2 canonical batch-DML fast parse: pin the observable behaviour of +/// ExecuteBatchSQL for canonical single-row UPDATE/DELETE statements AND for tricky +/// non-canonical shapes that must fall back to the general regex path (embedded commas, +/// keywords inside string literals, multi-column SET, non-= operators, whitespace in literals). +/// Both routes must produce identical results. +/// +public sealed class BatchCanonicalParseTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public BatchCanonicalParseTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_BatchCanonical_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + private static string? Scalar(IDatabase db, string sql, string column) + { + var rows = db.ExecuteQuery(sql); + if (rows.Count == 0) + { + return null; + } + + foreach (var key in rows[0].Keys) + { + if (key.Equals(column, StringComparison.OrdinalIgnoreCase)) + { + return rows[0][key]?.ToString(); + } + } + + return null; + } + + private static double? Num(IDatabase db, string sql, string column) + { + var rows = db.ExecuteQuery(sql); + if (rows.Count == 0) + { + return null; + } + + foreach (var key in rows[0].Keys) + { + if (key.Equals(column, StringComparison.OrdinalIgnoreCase) && rows[0][key] is not null) + { + return Convert.ToDouble(rows[0][key], System.Globalization.CultureInfo.InvariantCulture); + } + } + + return null; + } + + [Fact] + public void CanonicalSingleSetUpdate_UpdatesRow() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, email TEXT, age INTEGER, score REAL)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + db.ExecuteSQL("INSERT INTO t VALUES ('User1', 'u1@x', 30, 0.1)"); + + db.ExecuteBatchSQL(["UPDATE t SET score = 99.5 WHERE name = 'User1'"]); + + Assert.Equal(99.5, Num(db, "SELECT score FROM t WHERE name = 'User1'", "score")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SetValue_WithCommaAndWhereKeywordInsideString_IsHandled() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, data TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a,b')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'c')"); + + // data value contains a comma and the word WHERE inside the string literal. + db.ExecuteBatchSQL(["UPDATE t SET data = 'x, WHERE y' WHERE id = 1"]); + + Assert.Equal("x, WHERE y", Scalar(db, "SELECT data FROM t WHERE id = 1", "data")); + Assert.Equal("c", Scalar(db, "SELECT data FROM t WHERE id = 2", "data")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void MultiColumnSet_FallsBackAndUpdatesAll() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT, b INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'x', 10)"); + + db.ExecuteBatchSQL(["UPDATE t SET a = 'y', b = 42 WHERE id = 1"]); + + Assert.Equal("y", Scalar(db, "SELECT a FROM t WHERE id = 1", "a")); + Assert.Equal(42, Num(db, "SELECT b FROM t WHERE id = 1", "b")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void CanonicalDelete_RemovesRow() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, age INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES ('a', 1)"); + db.ExecuteSQL("INSERT INTO t VALUES ('b', 2)"); + + db.ExecuteBatchSQL(["DELETE FROM t WHERE name = 'a'"]); + + Assert.Null(Scalar(db, "SELECT name FROM t WHERE name = 'a'", "name")); + Assert.Equal("b", Scalar(db, "SELECT name FROM t WHERE name = 'b'", "name")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void WhereLiteral_WithSpaces_IsHandled() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, score REAL)"); + db.ExecuteSQL("INSERT INTO t VALUES ('User 1', 1.0)"); + + db.ExecuteBatchSQL(["UPDATE t SET score = 2.0 WHERE name = 'User 1'"]); + Assert.Equal(2.0, Num(db, "SELECT score FROM t WHERE name = 'User 1'", "score")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } +} + diff --git a/tests/SharpCoreDB.Tests/DmlSinglePassTests.cs b/tests/SharpCoreDB.Tests/DmlSinglePassTests.cs new file mode 100644 index 00000000..a4047877 --- /dev/null +++ b/tests/SharpCoreDB.Tests/DmlSinglePassTests.cs @@ -0,0 +1,252 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using System; +using System.IO; +using Xunit; + +/// +/// Issue #7/#8: single-pass DML paths. +/// - DELETE/UPDATE SQL no longer materialize matching rows twice (once for RETURNING/affected-count +/// and once inside the table operation) — the table operation itself returns the affected rows/count. +/// - Simple `pk = value` WHERE clauses are resolved through the primary-key B-tree directly +/// (single search + one read) in the single-row, batch and full-table DELETE/UPDATE paths. +/// These tests pin the observable behavior (affected rows/count and correctness for range / +/// non-indexed / non-PK WHERE clauses that must bypass the fast path). +/// +public sealed class DmlSinglePassTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public DmlSinglePassTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_DmlSinglePass_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + [Fact] + public void SqlDelete_ByPrimaryKey_AffectedCountIsOne() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + + db.ExecuteSQL("DELETE FROM t WHERE id = 1"); + + Assert.Equal(1, db.GetLastChanges()); + Assert.Single(db.ExecuteQuery("SELECT * FROM t")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SqlDelete_ByPrimaryKey_NonExistentKey_AffectsZeroRows() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + + db.ExecuteSQL("DELETE FROM t WHERE id = 999"); + + Assert.Equal(0, db.GetLastChanges()); + Assert.Single(db.ExecuteQuery("SELECT * FROM t")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SqlDelete_RangeWhere_DeletesAllMatchingRows() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + db.ExecuteSQL("INSERT INTO t VALUES (3, 'c')"); + + // `id > 1` must NOT hit the PK point-lookup fast path — it goes through the generic + // machinery and deletes every matching row. + db.ExecuteSQL("DELETE FROM t WHERE id > 1"); + + Assert.Equal(2, db.GetLastChanges()); + var remaining = db.ExecuteQuery("SELECT * FROM t"); + Assert.Single(remaining); + Assert.Equal(1, Convert.ToInt32(remaining[0]["id"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SqlDelete_NonIndexedColumn_FallsBackToFullScan() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + db.ExecuteSQL("INSERT INTO t VALUES (3, 'b')"); + + db.ExecuteSQL("DELETE FROM t WHERE name = 'b'"); + + Assert.Equal(2, db.GetLastChanges()); + var remaining = db.ExecuteQuery("SELECT * FROM t"); + Assert.Single(remaining); + Assert.Equal(1, Convert.ToInt32(remaining[0]["id"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SqlDelete_Returning_ReturnsPreDeleteRows() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + + var result = db.ExecuteQuery("DELETE FROM t WHERE id = 1 RETURNING id, name"); + + Assert.Single(result); + Assert.Equal(1, result[0]["id"]); + Assert.Equal("a", result[0]["name"]); + Assert.Single(db.ExecuteQuery("SELECT * FROM t")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SqlUpdate_ByPrimaryKey_AffectedCountIsOne() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + + db.ExecuteSQL("UPDATE t SET name = 'z' WHERE id = 1"); + + Assert.Equal(1, db.GetLastChanges()); + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("z", row[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void SqlUpdate_RangeWhere_AffectedCountIsTwo() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + db.ExecuteSQL("INSERT INTO t VALUES (3, 'c')"); + + db.ExecuteSQL("UPDATE t SET name = 'x' WHERE id > 1"); + + Assert.Equal(2, db.GetLastChanges()); + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal("a", rows[0]["name"]); + Assert.Equal("x", rows[1]["name"]); + Assert.Equal("x", rows[2]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void ExecuteBatchSQL_DeleteByPrimaryKey_DeletesRows() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + for (int i = 1; i <= 5; i++) + { + db.ExecuteSQL($"INSERT INTO t VALUES ({i}, 'n{i}')"); + } + + db.ExecuteBatchSQL(["DELETE FROM t WHERE id = 1", "DELETE FROM t WHERE id = 3"]); + + var remaining = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal(3, remaining.Count); + Assert.Equal(2, Convert.ToInt32(remaining[0]["id"])); + Assert.Equal(4, Convert.ToInt32(remaining[1]["id"])); + Assert.Equal(5, Convert.ToInt32(remaining[2]["id"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void ExecuteBatchSQL_UpdateByPrimaryKey_UpdatesRows() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + for (int i = 1; i <= 3; i++) + { + db.ExecuteSQL($"INSERT INTO t VALUES ({i}, 'n{i}')"); + } + + db.ExecuteBatchSQL(["UPDATE t SET name = 'x' WHERE id = 2", "UPDATE t SET name = 'y' WHERE id = 3"]); + + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal("n1", rows[0]["name"]); + Assert.Equal("x", rows[1]["name"]); + Assert.Equal("y", rows[2]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } +} diff --git a/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs b/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs new file mode 100644 index 00000000..401f8e01 --- /dev/null +++ b/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs @@ -0,0 +1,268 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.DataStructures; +using SharpCoreDB.Interfaces; +using System; +using System.IO; +using Xunit; + +/// +/// B5: 1.x → 2.0 record-format migration. A legacy (variable-length records) database opened with +/// is auto-migrated to the fixed-width layout; +/// an explicit API is also provided. The record +/// format is persisted in metadata so reopen keeps the layout without the config flag. +/// +public sealed class FixedWidthMigrationTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public FixedWidthMigrationTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_FixedWidthMigration_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + private IDatabase CreateLegacyDb() => _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig()); + + private IDatabase CreateFixedWidthDb() => _factory.Create( + _dirPath, "pw", isReadOnly: false, config: new DatabaseConfig { FixedWidthRecordLayout = true }); + + private IDatabase CreateReadOnlyFixedWidthDb() => _factory.Create( + _dirPath, "pw", isReadOnly: true, config: new DatabaseConfig { FixedWidthRecordLayout = true }); + + private static bool IsFixedWidth(IDatabase db, string tableName) + => db.TryGetTable(tableName, out var t) && t.IsFixedWidthRecords; + + private string DatPath(string table) => Path.Combine(_dirPath, $"{table}.dat"); + + private string OvfPath(string table) => Path.ChangeExtension(DatPath(table), ".ovf"); + + [Fact] + public void ReopenWithFixedWidthConfig_AutoMigratesLegacyTable() + { + // 1.x database: variable-length records. + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha', 1.5)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta', 2.5)"); + db.ExecuteSQL("UPDATE t SET name = 'ALPHA-2' WHERE id = 1"); // stale row in .dat + Assert.False(IsFixedWidth(db, "t")); + } + finally { (db as IDisposable)?.Dispose(); } + + // 2.0 reopen with the config flag → auto-migrate. + db = null; + try + { + db = CreateFixedWidthDb(); + Assert.True(IsFixedWidth(db, "t")); + Assert.True(File.Exists(OvfPath("t"))); + + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("ALPHA-2", row[0]["name"]); + Assert.Equal(2.5, Convert.ToDouble(db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["score"])); + + // Fixed-width behavior after migration: in-place UPDATE, no .dat growth. + long sizeAfterMigrate = new FileInfo(DatPath("t")).Length; + db.ExecuteSQL("UPDATE t SET name = 'a much longer name value than the original' WHERE id = 2"); + Assert.Equal(sizeAfterMigrate, new FileInfo(DatPath("t")).Length); + } + finally { (db as IDisposable)?.Dispose(); } + + // Third open WITHOUT the config flag → persisted record format is authoritative. + db = null; + try + { + db = CreateLegacyDb(); + Assert.True(IsFixedWidth(db, "t")); + Assert.Equal("a much longer name value than the original", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void ExplicitApi_MigrateTableToFixedWidth_PersistsFormat() + { + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta')"); + + int migrated = db.MigrateTableToFixedWidth("t"); + Assert.Equal(2, migrated); + Assert.True(IsFixedWidth(db, "t")); + + Assert.Equal("alpha", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + + // Reopen without the config flag → still fixed-width (flag persisted in metadata). + db = null; + try + { + db = CreateLegacyDb(); + Assert.True(IsFixedWidth(db, "t")); + Assert.Equal("beta", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void ExplicitApi_OnAlreadyFixedWidth_ReturnsZero() + { + IDatabase? db = null; + try + { + db = CreateFixedWidthDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + Assert.Equal(0, db.MigrateTableToFixedWidth("t")); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void EmptyTable_Migrates_SetsFormat() + { + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + Assert.Equal(0, db.MigrateTableToFixedWidth("t")); + Assert.True(IsFixedWidth(db, "t")); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void ReadOnlyOpen_WithFixedWidthConfig_StaysLegacy_DataReadable() + { + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + } + finally { (db as IDisposable)?.Dispose(); } + + db = null; + try + { + db = CreateReadOnlyFixedWidthDb(); + // Read-only opens never rewrite data: the table must stay legacy and stay readable. + Assert.False(IsFixedWidth(db, "t")); + Assert.Equal("alpha", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void PageBasedTable_ExplicitMigration_ConvertsToColumnarAndFixedWidth() + { + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT) STORAGE = PAGE_BASED"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta')"); + Assert.True(File.Exists(Path.Combine(_dirPath, "t.pages"))); + Assert.False(IsFixedWidth(db, "t")); + + int migrated = db.MigrateTableToFixedWidth("t"); + Assert.Equal(2, migrated); + Assert.True(IsFixedWidth(db, "t")); + Assert.True(File.Exists(DatPath("t"))); + Assert.False(File.Exists(Path.Combine(_dirPath, "t.pages"))); + + Assert.Equal("alpha", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + Assert.Equal("beta", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + + // Reopen without the config flag → persisted fixed-width columnar table, data intact. + db = null; + try + { + db = CreateLegacyDb(); + Assert.True(IsFixedWidth(db, "t")); + Assert.True(File.Exists(DatPath("t"))); + Assert.False(File.Exists(Path.Combine(_dirPath, "t.pages"))); + Assert.Equal("alpha", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void PageBasedTable_AutoMigrationOnReopen() + { + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT) STORAGE = PAGE_BASED"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta')"); + } + finally { (db as IDisposable)?.Dispose(); } + + db = null; + try + { + db = CreateFixedWidthDb(); // config opts into fixed-width → auto-migrate (incl. PageBased) + Assert.True(IsFixedWidth(db, "t")); + Assert.True(File.Exists(DatPath("t"))); + Assert.False(File.Exists(Path.Combine(_dirPath, "t.pages"))); + Assert.Equal("beta", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void Migration_DropsStaleVersions() + { + IDatabase? db = null; + try + { + db = CreateLegacyDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'v1')"); + db.ExecuteSQL("UPDATE t SET name = 'v2' WHERE id = 1"); + db.ExecuteSQL("UPDATE t SET name = 'v3' WHERE id = 1"); + + db.MigrateTableToFixedWidth("t"); + + // Only the current version remains after migration. + var rows = db.ExecuteQuery("SELECT * FROM t"); + Assert.Single(rows); + Assert.Equal("v3", rows[0]["name"]); + + Assert.True(db.TryGetTable("t", out var table)); + var concrete = Assert.IsType
(table); + Assert.Equal(1, concrete.Select().Count); + } + finally { (db as IDisposable)?.Dispose(); } + } +} diff --git a/tests/SharpCoreDB.Tests/FixedWidthPatchTests.cs b/tests/SharpCoreDB.Tests/FixedWidthPatchTests.cs new file mode 100644 index 00000000..ecce0bc2 --- /dev/null +++ b/tests/SharpCoreDB.Tests/FixedWidthPatchTests.cs @@ -0,0 +1,194 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using System; +using System.IO; +using Xunit; + +/// +/// Fixed-width layout step (SQLite-style UPDATE): when the row's storage position is known +/// (PK B-tree or hash index), the columnar UPDATE path patches only the updated fields at their +/// actual offsets in the existing record instead of deserializing → mutating → re-serializing the +/// whole row. A fixed-size field keeps the record length unchanged, so the write is an in-place +/// overwrite (Issue #6) and the data file does not grow. Variable-length fields that change size +/// fall back to the append path (correctness unchanged). +/// +public sealed class FixedWidthPatchTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public FixedWidthPatchTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_FixedWidth_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + private long DataFileSize(string table) => new FileInfo(Path.Combine(_dirPath, $"{table}.dat")).Length; + + [Fact] + public void Update_FixedFieldAfterVariableColumns_PatchesInPlace_FileDoesNotGrow() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + // No primary key — positions come from the hash index. `score` (REAL) and `age` + // (INTEGER) sit AFTER two variable-length TEXT columns, so the patch must discover + // their real offsets by walking the record. + db.ExecuteSQL("CREATE TABLE t (name TEXT, email TEXT, age INTEGER, score REAL, data TEXT)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + db.ExecuteSQL("INSERT INTO t VALUES ('User0', 'u0@test.com', 20, 0.0, 'payload-0')"); + db.ExecuteSQL("INSERT INTO t VALUES ('User1', 'u1@test.com', 30, 1.0, 'payload-1')"); + + long sizeAfterInsert = DataFileSize("t"); + Assert.True(sizeAfterInsert > 0); + + // 50 in-place updates of fixed-size fields (length never changes → no append, no growth). + // InvariantCulture: interpolated doubles must use '.' so the SQL parser reads the + // decimal point (the dev machine locale uses ',' otherwise). + for (int i = 0; i < 50; i++) + { + var scoreValue = (0.5 + i).ToString(System.Globalization.CultureInfo.InvariantCulture); + db.ExecuteSQL($"UPDATE t SET score = {scoreValue} WHERE name = 'User0'"); + } + + db.ExecuteSQL("UPDATE t SET age = 42 WHERE name = 'User1'"); + + Assert.Equal(sizeAfterInsert, DataFileSize("t")); + + var row0 = db.ExecuteQuery("SELECT * FROM t WHERE name = 'User0'"); + Assert.Single(row0); + Assert.Equal(49.5, Convert.ToDouble(row0[0]["score"])); + Assert.Equal(20, Convert.ToInt32(row0[0]["age"])); + + var row1 = db.ExecuteQuery("SELECT * FROM t WHERE name = 'User1'"); + Assert.Single(row1); + Assert.Equal(42, Convert.ToInt32(row1[0]["age"])); + Assert.Equal(1.0, Convert.ToDouble(row1[0]["score"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_ByPrimaryKey_FixedField_PatchesInPlace_FileDoesNotGrow() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, email TEXT, score REAL)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'User0', 'u0@test.com', 0.0)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'User1', 'u1@test.com', 1.0)"); + + long sizeAfterInsert = DataFileSize("t"); + + db.ExecuteSQL("UPDATE t SET score = 77.5 WHERE id = 1"); + + Assert.Equal(sizeAfterInsert, DataFileSize("t")); + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal(77.5, Convert.ToDouble(row[0]["score"])); + Assert.Equal("User0", row[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_VariableFieldGrows_FallsBackToAppend_ValueCorrect() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'short')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'other')"); + + // name grows: the patch cannot fit the field, so the update must fall back to the + // append path — the value must still be correct and the row readable. + db.ExecuteSQL("UPDATE t SET name = 'this is a much longer name value' WHERE id = 1"); + + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("this is a much longer name value", row[0]["name"]); + Assert.Equal(2, db.ExecuteQuery("SELECT * FROM t").Count); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_CompoundWhere_FallsBackToSelectInternal_StillCorrect() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, age INTEGER, score REAL)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + db.ExecuteSQL("INSERT INTO t VALUES ('User0', 20, 1.0)"); + db.ExecuteSQL("INSERT INTO t VALUES ('User0', 30, 2.0)"); + db.ExecuteSQL("INSERT INTO t VALUES ('User1', 40, 3.0)"); + + // Compound WHERE: must NOT resolve through the hash index only — every matching row + // gets the same score. + db.ExecuteSQL("UPDATE t SET score = 9.0 WHERE name = 'User0' AND age > 25"); + + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY age"); + Assert.Equal(1.0, Convert.ToDouble(rows[0]["score"])); + Assert.Equal(9.0, Convert.ToDouble(rows[1]["score"])); + Assert.Equal(3.0, Convert.ToDouble(rows[2]["score"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void ExecuteBatchSQL_Update_FixedFieldAfterVariableColumns_Correct() + { + var db = _factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, email TEXT, age INTEGER, score REAL)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + for (int i = 0; i < 4; i++) + { + db.ExecuteSQL($"INSERT INTO t VALUES ('User{i}', 'u{i}@test.com', {10 + i}, {i}.0)"); + } + + db.ExecuteBatchSQL([ + "UPDATE t SET score = 50.5 WHERE name = 'User1'", + "UPDATE t SET age = 99 WHERE name = 'User2'" + ]); + + // ORDER BY age after the updates: User0=10, User1=11, User3=13, User2=99. + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY age"); + Assert.Equal(50.5, Convert.ToDouble(rows[1]["score"])); + Assert.Equal(99, Convert.ToInt32(rows[3]["age"])); + Assert.Equal(2.0, Convert.ToDouble(rows[3]["score"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } +} diff --git a/tests/SharpCoreDB.Tests/FixedWidthRecordLayoutTests.cs b/tests/SharpCoreDB.Tests/FixedWidthRecordLayoutTests.cs new file mode 100644 index 00000000..0bd0bde4 --- /dev/null +++ b/tests/SharpCoreDB.Tests/FixedWidthRecordLayoutTests.cs @@ -0,0 +1,439 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; + +/// +/// Issue B1: out-of-line overflow (SQLite-model) — opt-in fixed-width record layout +/// (). Fixed-size columns live at constant +/// record offsets; TEXT/BLOB values are stored in the table's overflow arena, so the record length +/// is constant per schema and every UPDATE (fixed OR variable column) is an in-place overwrite — +/// the .dat file does not grow. +/// +public sealed class FixedWidthRecordLayoutTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public FixedWidthRecordLayoutTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_FixedWidthLayout_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + private IDatabase CreateFixedWidthDb() => _factory.Create( + _dirPath, "pw", isReadOnly: false, config: new DatabaseConfig { FixedWidthRecordLayout = true }); + + private string DatPath(string table) => Path.Combine(_dirPath, $"{table}.dat"); + + private string OvfPath(string table) => Path.ChangeExtension(DatPath(table), ".ovf"); + + [Fact] + public void RoundTrip_AllColumnTypes_PointAndFullScan() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL, flag BOOLEAN, created DATETIME)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha', 1.5, 1, '2024-01-01')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta', 2.5, 0, '2024-02-02')"); + + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 2"); + Assert.Single(row); + Assert.Equal("beta", row[0]["name"]); + Assert.Equal(2.5, Convert.ToDouble(row[0]["score"])); + Assert.Equal(false, Convert.ToBoolean(row[0]["flag"])); + + var all = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal(2, all.Count); + Assert.Equal("alpha", all[0]["name"]); + Assert.Equal("beta", all[1]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_FixedColumn_DoesNotGrowDataFile() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, val INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 0)"); + + long sizeAfterInsert = new FileInfo(DatPath("t")).Length; + + for (int i = 0; i <= 99; i++) + { + db.ExecuteSQL($"UPDATE t SET val = {i} WHERE id = 1"); + } + + Assert.Equal(sizeAfterInsert, new FileInfo(DatPath("t")).Length); + Assert.Equal(99, Convert.ToInt32(db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["val"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_VariableColumn_Grow_DoesNotGrowDataFile_ValueCorrect() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'short')"); + + long sizeAfterInsert = new FileInfo(DatPath("t")).Length; + Assert.True(File.Exists(OvfPath("t"))); // variable values go to the overflow arena + + // Growing the string must NOT grow the data file — the record stays fixed-width and the + // new payload goes to the arena. + db.ExecuteSQL("UPDATE t SET name = 'a much longer name value than the original' WHERE id = 1"); + + Assert.Equal(sizeAfterInsert, new FileInfo(DatPath("t")).Length); + Assert.Equal("a much longer name value than the original", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_VariableColumn_Shrink_ValueCorrect() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a long original value')"); + db.ExecuteSQL("UPDATE t SET name = 'x' WHERE id = 1"); + + Assert.Equal("x", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Delete_ByPrimaryKey_RemovesRow() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + + db.ExecuteSQL("DELETE FROM t WHERE id = 1"); + + Assert.Empty(db.ExecuteQuery("SELECT * FROM t WHERE id = 1")); + Assert.Single(db.ExecuteQuery("SELECT * FROM t")); + Assert.Equal("b", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Reopen_WithSameConfig_SurvivesArena() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha', 1.5)"); + db.ExecuteSQL("UPDATE t SET name = 'updated name that is longer' WHERE id = 1"); + } + finally + { + (db as IDisposable)?.Dispose(); + } + + var db2 = CreateFixedWidthDb(); + try + { + var row = db2.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("updated name that is longer", row[0]["name"]); + Assert.Equal(1.5, Convert.ToDouble(row[0]["score"])); + } + finally + { + (db2 as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Arena_Compacts_ReclaimsSpace_DataCorrect() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + // Insert row 2 FIRST so its variable block sits at arena offset 0 — the first arena + // block. Its offset stays live through the updates below and must survive compaction + // (regression: offset 0 is a valid block offset, only the slot flag distinguishes NULL). + db.ExecuteSQL("INSERT INTO t VALUES (2, 'seed-2')"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'seed-1')"); + + // Many variable updates: each appends a new arena block (the previous one is freed), + // so the .ovf grows until compaction reclaims it. + for (int i = 0; i < 300; i++) + { + db.ExecuteSQL($"UPDATE t SET name = 'value-{i}-with-enough-length' WHERE id = 1"); + } + + long arenaBefore = new FileInfo(OvfPath("t")).Length; + Assert.True(arenaBefore > 0); + + // Force compaction deterministically via the Table API (B3: arena + .dat together). + Assert.True(db.TryGetTable("t", out var table)); + var concrete = Assert.IsType(table); + concrete.CompactStorage(); + + long arenaAfter = new FileInfo(OvfPath("t")).Length; + Assert.True(arenaAfter < arenaBefore, $"arena did not shrink: {arenaAfter} >= {arenaBefore}"); + + // Data still correct after compaction + arena re-point. + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("value-299-with-enough-length", row[0]["name"]); + Assert.Equal("seed-2", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + + // Reopen: the remapped arena offsets + compacted .dat must survive a fresh load. + (db as IDisposable)?.Dispose(); + db = CreateFixedWidthDb(); + Assert.Equal("value-299-with-enough-length", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + Assert.Equal("seed-2", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void StructRow_Api_FallsBackToDictionary() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta')"); + + var rows = db.ExecuteQueryStruct("SELECT * FROM t WHERE id = 2").ToList(); + Assert.Single(rows); + Assert.Equal("beta", rows[0].GetValueBoxed(1).ToString()); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void NumericEarlyWhere_ConstantOffset_ColumnAfterVariable_PerfPath() + { + var db = CreateFixedWidthDb(); + try + { + // Numeric column after a variable-length column: only the fixed-width layout can read + // it at a constant slot offset — the variable-length walk would reject the preceding + // TEXT column. B4 re-enables the numeric early-WHERE for fixed-width tables. + db.ExecuteSQL("CREATE TABLE t (name TEXT, score INTEGER, id INTEGER PRIMARY KEY)"); + db.ExecuteSQL("INSERT INTO t VALUES ('alpha', 10, 1)"); + db.ExecuteSQL("INSERT INTO t VALUES ('beta', 30, 2)"); + db.ExecuteSQL("INSERT INTO t VALUES ('gamma', 30, 3)"); + db.ExecuteSQL("INSERT INTO t VALUES ('delta', 40, 4)"); + + var rows = db.ExecuteQuery("SELECT * FROM t WHERE score = 30 ORDER BY id"); + Assert.Equal(2, rows.Count); + Assert.Equal("beta", rows[0]["name"]); + Assert.Equal("gamma", rows[1]["name"]); + + Assert.Single(db.ExecuteQuery("SELECT * FROM t WHERE score = 40")); + Assert.Empty(db.ExecuteQuery("SELECT * FROM t WHERE score = 99")); + Assert.Empty(db.ExecuteQuery("SELECT * FROM t WHERE score = NULL")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void StringEarlyWhere_ConstantOffset_ArenaCompare() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'Alice')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'Bob')"); + db.ExecuteSQL("INSERT INTO t VALUES (3, NULL)"); + + // Simple equality on a string column → B4 early-WHERE: constant slot offset + arena + // payload compare (Binary collation). NULL never equals a value. + var rows = db.ExecuteQuery("SELECT * FROM t WHERE name = 'Alice'"); + Assert.Single(rows); + Assert.Equal(1, Convert.ToInt32(rows[0]["id"])); + + Assert.Empty(db.ExecuteQuery("SELECT * FROM t WHERE name = 'alice'")); + + // IS NULL is not a simple equality → full-scan EvaluateWhere fallback stays correct. + var nulls = db.ExecuteQuery("SELECT * FROM t WHERE name IS NULL"); + Assert.Single(nulls); + Assert.Equal(3, Convert.ToInt32(nulls[0]["id"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void NoCaseCollation_StringWhere_FallsBackCorrectly() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT COLLATE NOCASE)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'Alice')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'Bob')"); + + // NOCASE collation → the binary early-WHERE must NOT engage → full scan + + // collation-aware EvaluateWhere stays correct. + var rows = db.ExecuteQuery("SELECT * FROM t WHERE name = 'alice'"); + Assert.Single(rows); + Assert.Equal(1, Convert.ToInt32(rows[0]["id"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void StructRow_NumericWhere_FixedWidth_UsesSimdFastPath() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, score INTEGER, id INTEGER PRIMARY KEY)"); + db.ExecuteSQL("INSERT INTO t VALUES ('alpha', 10, 1)"); + db.ExecuteSQL("INSERT INTO t VALUES ('beta', 30, 2)"); + db.ExecuteSQL("INSERT INTO t VALUES ('gamma', 30, 3)"); + + // StructRow API: numeric equality on a non-indexed column → the numeric-SIMD batch + // fast path now works for fixed-width tables (constant-offset raw reads). + Assert.True(db.TryGetTable("t", out var table)); + var concrete = Assert.IsType(table); + var rows = concrete.ScanStructRowsWhere("score = 30").ToList(); + Assert.Equal(2, rows.Count); + Assert.Equal("beta", rows[0].GetValueBoxed(0).ToString()); + Assert.Equal("gamma", rows[1].GetValueBoxed(0).ToString()); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void ArenaFreeList_ReusesEqualLengthBlocks_NoGrowth() + { + var db = CreateFixedWidthDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'AAAAAAAA')"); // 8-byte arena payload + + // The first same-length update appends (the insert's block is freed into the free-list). + db.ExecuteSQL("UPDATE t SET name = 'BBBBBBBB' WHERE id = 1"); + long sizeAfterFirstUpdate = new FileInfo(OvfPath("t")).Length; + + // All subsequent same-length updates reuse the freed block in place → the arena no + // longer grows (B6 free-list; the storage layer requires identical plaintext length). + string[] names = { "CCCCCCCC", "DDDDDDDD", "EEEEEEEE", "FFFFFFFF", "GGGGGGGG" }; + for (int i = 0; i < 200; i++) + { + db.ExecuteSQL($"UPDATE t SET name = '{names[i % names.Length]}' WHERE id = 1"); + } + + Assert.Equal(sizeAfterFirstUpdate, new FileInfo(OvfPath("t")).Length); + Assert.Equal("GGGGGGGG", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + + // Diagnostics: the free-list actually performed in-place block reuse. + Assert.True(db.TryGetTable("t", out var table)); + var concrete = Assert.IsType(table); + Assert.True(concrete.OverflowArenaBlockReuses > 0, "expected at least one in-place arena block reuse"); + Assert.Equal(1, concrete.OverflowArenaFreeBlockCount); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void ArenaFreeList_SurvivesReopen() + { + // Session 1: one same-length update frees the first block (offset 0) and appends a new one. + long ovfAfterSession1; + IDatabase? db = null; + try + { + db = CreateFixedWidthDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'AAAAAAAA')"); + db.ExecuteSQL("UPDATE t SET name = 'BBBBBBBB' WHERE id = 1"); + ovfAfterSession1 = new FileInfo(OvfPath("t")).Length; + Assert.Equal("BBBBBBBB", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + + // Session 2: the free-list is derived from the records on arena load, so a same-length + // update reuses the freed block instead of appending → the arena does NOT grow. + db = null; + try + { + db = CreateFixedWidthDb(); + db.ExecuteSQL("UPDATE t SET name = 'CCCCCCCC' WHERE id = 1"); + Assert.Equal(ovfAfterSession1, new FileInfo(OvfPath("t")).Length); + Assert.Equal("CCCCCCCC", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + + Assert.True(db.TryGetTable("t", out var table)); + var concrete = Assert.IsType(table); + Assert.True(concrete.OverflowArenaBlockReuses > 0, "expected a cross-session in-place arena block reuse"); + } + finally { (db as IDisposable)?.Dispose(); } + } +} diff --git a/tests/SharpCoreDB.Tests/SingleFileFixedWidthTests.cs b/tests/SharpCoreDB.Tests/SingleFileFixedWidthTests.cs new file mode 100644 index 00000000..d8c86668 --- /dev/null +++ b/tests/SharpCoreDB.Tests/SingleFileFixedWidthTests.cs @@ -0,0 +1,196 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using SharpCoreDB.Storage; +using System; +using System.IO; +using Xunit; + +/// +/// B6: single-file (.scdb) fixed-width record layout. With +/// the single-file table stores binary fixed-width records in its data block (variable values in a +/// dedicated overflow block) instead of JSON rows, so value-only updates keep the data block +/// constant-size. The on-disk format is detected on reopen; legacy JSON tables migrate on demand. +/// +public sealed class SingleFileFixedWidthTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _scdbPath; + + public SingleFileFixedWidthTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _scdbPath = Path.Combine(Path.GetTempPath(), $"SCDB_FixedWidth_{Guid.NewGuid():N}.scdb"); + } + + public void Dispose() + { + try { if (File.Exists(_scdbPath)) File.Delete(_scdbPath); } catch { } + } + + private static DatabaseOptions FixedWidthOptions() => new() + { + StorageMode = StorageMode.SingleFile, + EnableMemoryMapping = true, + AutoVacuum = true, + AutoVacuumMode = VacuumMode.Quick, + DatabaseConfig = new DatabaseConfig { FixedWidthRecordLayout = true }, + }; + + private static DatabaseOptions JsonOptions() => new() + { + StorageMode = StorageMode.SingleFile, + EnableMemoryMapping = true, + AutoVacuum = true, + AutoVacuumMode = VacuumMode.Quick, + }; + + private IDatabase CreateFixedWidthDb() => _factory.CreateWithOptions(_scdbPath, "pw", FixedWidthOptions()); + + private IDatabase CreateJsonDb() => _factory.CreateWithOptions(_scdbPath, "pw", JsonOptions()); + + private static bool IsFixedWidth(IDatabase db, string tableName) + => db.TryGetTable(tableName, out var t) && t.IsFixedWidthRecords; + + [Fact] + public void RoundTrip_AllColumnTypes_Reopen() + { + IDatabase? db = null; + try + { + db = CreateFixedWidthDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL, flag BOOLEAN, created DATETIME)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha', 1.5, 1, '2024-01-01')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta', 2.5, 0, '2024-02-02')"); + Assert.True(IsFixedWidth(db, "t")); + + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 2"); + Assert.Single(row); + Assert.Equal("beta", row[0]["name"]); + Assert.Equal(2.5, Convert.ToDouble(row[0]["score"])); + Assert.Equal(false, Convert.ToBoolean(row[0]["flag"])); + } + finally { (db as IDisposable)?.Dispose(); } + + // Reopen: the on-disk format is detected (no config flag needed for reading). + db = null; + try + { + db = CreateJsonDb(); // config flag OFF + Assert.True(IsFixedWidth(db, "t")); + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("alpha", row[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void SameLengthUpdate_DoesNotGrowDataBlock() + { + IDatabase? db = null; + try + { + db = CreateFixedWidthDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'AAAAAAAA')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'BBBBBBBB')"); + + // The first same-length update appends one arena block (the free-list is empty); from + // then on freed blocks are reused in place, so the file must stop growing. + db.ExecuteSQL("UPDATE t SET name = 'CCCCCCCC' WHERE id = 1"); + long sizeAfterFirstUpdate = new FileInfo(_scdbPath).Length; + + string[] names = { "DDDDDDDD", "EEEEEEEE" }; + for (int i = 0; i < 100; i++) + { + db.ExecuteSQL($"UPDATE t SET name = '{names[i % names.Length]}' WHERE id = 1"); + } + + Assert.Equal("EEEEEEEE", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + Assert.Equal(sizeAfterFirstUpdate, new FileInfo(_scdbPath).Length); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void ExplicitMigration_JsonToFixedWidth() + { + // Legacy single-file table (JSON rows). + IDatabase? db = null; + try + { + db = CreateJsonDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta')"); + Assert.False(IsFixedWidth(db, "t")); + + int migrated = db.MigrateTableToFixedWidth("t"); + Assert.Equal(2, migrated); + Assert.True(IsFixedWidth(db, "t")); + + Assert.Equal("alpha", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + + // Reopen without any config → binary format detected, data intact. + db = null; + try + { + db = CreateJsonDb(); + Assert.True(IsFixedWidth(db, "t")); + Assert.Equal("beta", db.ExecuteQuery("SELECT * FROM t WHERE id = 2")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void ReopenWithFixedWidthConfig_AutoMigratesJson() + { + IDatabase? db = null; + try + { + db = CreateJsonDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha')"); + } + finally { (db as IDisposable)?.Dispose(); } + + db = null; + try + { + db = CreateFixedWidthDb(); // config opts into fixed-width → auto-migrate on load + Assert.True(IsFixedWidth(db, "t")); + Assert.Equal("alpha", db.ExecuteQuery("SELECT * FROM t WHERE id = 1")[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void PkIndexWorksOnBinaryFormat() + { + IDatabase? db = null; + try + { + db = CreateFixedWidthDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b')"); + + var row = db.FindByPrimaryKey("t", 2); + Assert.NotNull(row); + Assert.Equal("b", row!["name"]); + + Assert.Single(db.ExecuteQuery("SELECT * FROM t WHERE id = 1")); + } + finally { (db as IDisposable)?.Dispose(); } + } +} diff --git a/tests/SharpCoreDB.Tests/SingleFilePkIndexTests.cs b/tests/SharpCoreDB.Tests/SingleFilePkIndexTests.cs new file mode 100644 index 00000000..9bee18f9 --- /dev/null +++ b/tests/SharpCoreDB.Tests/SingleFilePkIndexTests.cs @@ -0,0 +1,190 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + +/// +/// Issue A1: single-file (.scdb) primary-key hash index. FindByPrimaryKey and +/// SELECT … WHERE pk = value resolve through the PK index (O(1)) instead of an O(N) cache +/// scan; the index is maintained on INSERT / UPDATE (including PK changes) / DELETE and rebuilt +/// when the row cache is loaded (reopen). +/// +public sealed class SingleFilePkIndexTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _scdbPath; + + public SingleFilePkIndexTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _scdbPath = Path.Combine(Path.GetTempPath(), $"SCDB_PkIndex_{Guid.NewGuid():N}.scdb"); + } + + public void Dispose() + { + try { if (File.Exists(_scdbPath)) File.Delete(_scdbPath); } catch { } + } + + private IDatabase CreateScdb() => _factory.CreateWithOptions(_scdbPath, "pw", DatabaseOptions.CreateSingleFileDefault()); + + private static void Seed(IDatabase db) + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a', 1.0)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b', 2.0)"); + db.ExecuteSQL("INSERT INTO t VALUES (3, 'c', 3.0)"); + } + + [Fact] + public void FindByPrimaryKey_ReturnsRow() + { + var db = CreateScdb(); + try + { + Seed(db); + var row = db.FindByPrimaryKey("t", 2); + Assert.NotNull(row); + Assert.Equal("b", row!["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void FindByPrimaryKey_NonExistent_ReturnsNull() + { + var db = CreateScdb(); + try + { + Seed(db); + Assert.Null(db.FindByPrimaryKey("t", 999)); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Select_ByPrimaryKey_ReturnsRow() + { + var db = CreateScdb(); + try + { + Seed(db); + var rows = db.ExecuteQuery("SELECT * FROM t WHERE id = 2"); + Assert.Single(rows); + Assert.Equal("b", rows[0]["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Select_ByPrimaryKey_NonExistent_ReturnsEmpty() + { + var db = CreateScdb(); + try + { + Seed(db); + Assert.Empty(db.ExecuteQuery("SELECT * FROM t WHERE id = 999")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_ByPrimaryKey_MaintainsIndex() + { + var db = CreateScdb(); + try + { + Seed(db); + db.ExecuteSQL("UPDATE t SET name = 'zzz' WHERE id = 2"); + + Assert.Equal("zzz", db.FindByPrimaryKey("t", 2)!["name"]); + Assert.Single(db.ExecuteQuery("SELECT * FROM t WHERE id = 2")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_ChangingPk_Reindexes() + { + var db = CreateScdb(); + try + { + Seed(db); + db.ExecuteSQL("UPDATE t SET id = 99 WHERE id = 2"); + + Assert.Null(db.FindByPrimaryKey("t", 2)); + Assert.Equal("b", db.FindByPrimaryKey("t", 99)!["name"]); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Delete_ByPrimaryKey_MaintainsIndex() + { + var db = CreateScdb(); + try + { + Seed(db); + db.ExecuteSQL("DELETE FROM t WHERE id = 2"); + + Assert.Null(db.FindByPrimaryKey("t", 2)); + Assert.Single(db.ExecuteQuery("SELECT * FROM t WHERE id = 1")); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Index_SurvivesReopen() + { + var db = CreateScdb(); + try + { + Seed(db); + } + finally + { + (db as IDisposable)?.Dispose(); + } + + var db2 = CreateScdb(); + try + { + Assert.Equal("b", db2.FindByPrimaryKey("t", 2)!["name"]); + Assert.Single(db2.ExecuteQuery("SELECT * FROM t WHERE id = 3")); + } + finally + { + (db2 as IDisposable)?.Dispose(); + } + } +} diff --git a/tests/SharpCoreDB.Tests/SingleFileWriteTests.cs b/tests/SharpCoreDB.Tests/SingleFileWriteTests.cs new file mode 100644 index 00000000..4fa0571f --- /dev/null +++ b/tests/SharpCoreDB.Tests/SingleFileWriteTests.cs @@ -0,0 +1,101 @@ +// +// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + +/// +/// Issue A2: single-file (.scdb) write-path behavior. +/// reuses a table's existing block offset when the new row-cache JSON fits the block's allocated +/// pages, so a fixed-length (same-size) update overwrites the block in place — the .scdb file must +/// not grow and the updated value must survive reopen. +/// +public sealed class SingleFileWriteTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _scdbPath; + + public SingleFileWriteTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _scdbPath = Path.Combine(Path.GetTempPath(), $"SCDB_Write_{Guid.NewGuid():N}.scdb"); + } + + public void Dispose() + { + try { if (File.Exists(_scdbPath)) File.Delete(_scdbPath); } catch { } + } + + private IDatabase CreateScdb() => _factory.CreateWithOptions(_scdbPath, "pw", DatabaseOptions.CreateSingleFileDefault()); + + private long FileSize => new FileInfo(_scdbPath).Length; + + [Fact] + public void SameLengthUpdate_OverwritesInPlace_FileDoesNotGrow() + { + var db = CreateScdb(); + try + { + // Single-digit integer column: every update serializes to the same JSON byte length, + // so the block is overwritten at its existing offset (no relocation / no growth). + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, val INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 0)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 0)"); + + long sizeAfterInsert = FileSize; + Assert.True(sizeAfterInsert > 0); + + for (int i = 0; i <= 9; i++) + { + db.ExecuteSQL($"UPDATE t SET val = {i} WHERE id = 1"); + } + + Assert.Equal(sizeAfterInsert, FileSize); + + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal(9, Convert.ToInt32(row[0]["val"])); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void Update_ValueSurvivesReopen() + { + var db = CreateScdb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, val INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 0)"); + db.ExecuteSQL("UPDATE t SET val = 42 WHERE id = 1"); + } + finally + { + (db as IDisposable)?.Dispose(); + } + + var db2 = CreateScdb(); + try + { + var row = db2.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal(42, Convert.ToInt32(row[0]["val"])); + } + finally + { + (db2 as IDisposable)?.Dispose(); + } + } +} diff --git a/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs b/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs index 738721a7..5ce9ef8d 100644 --- a/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs +++ b/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs @@ -7,6 +7,7 @@ namespace SharpCoreDB.Tests; using Microsoft.Extensions.DependencyInjection; using SharpCoreDB.Interfaces; +using SharpCoreDB.Services; using System; using System.Collections.Generic; using System.IO; @@ -126,4 +127,306 @@ public void SqlUpdate_VariableWidth_GrowsWhenStoredLengthChanges_StillCorrect() (db as IDisposable)?.Dispose(); } } + + [Fact] + public void BatchSqlUpdate_NoPrimaryKey_HashIndexLookup_PatchesInPlace() + { + // Regression: ExecuteBatchSQL groups UPDATEs into UpdateMultiple, which previously + // resolved rows without their storage positions. Without a PK the columnar write path + // could not find the record slot → it appended a new version per update (stale rows + + // compaction storm). The position now comes from the hash-index lookup, so fixed-size + // fields (score REAL) are patched in place and the row count stays stable. + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL(@"CREATE TABLE docs ( + name TEXT NOT NULL, + email TEXT, + age INTEGER, + score REAL, + data TEXT + )"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + var rows = new List>(200); + for (int i = 0; i < 200; i++) + { + rows.Add(new Dictionary + { + ["name"] = $"User{i}", + ["email"] = $"user{i}@test.com", + ["age"] = 20 + i % 60, + ["score"] = i * 0.1, + ["data"] = $"payload-{i}", + }); + } + db.InsertBatch("docs", rows); + long sizeAfterInsert = DataFileSize("docs"); + + var stmts = new List(200); + for (int i = 0; i < 200; i++) + { + stmts.Add(string.Format(System.Globalization.CultureInfo.InvariantCulture, + "UPDATE docs SET score = {0:F1} WHERE name = 'User{1}'", i * 99.9, i)); + } + db.ExecuteBatchSQL(stmts); + + // All 200 rows still present, no duplicates from stale appends. + Assert.Equal(200, db.ExecuteQuery("SELECT * FROM docs").Count); + + // REAL score is a fixed-size field → patched in place, file does not grow. + Assert.Equal(sizeAfterInsert, DataFileSize("docs")); + + // Values are actually updated and visible through the hash-index lookup. + var updated = db.ExecuteQuery("SELECT * FROM docs WHERE name = @n", + new Dictionary { ["@n"] = "User42" }); + Assert.Single(updated); + Assert.Equal(42 * 99.9, updated[0]["score"]); + + var all = db.ExecuteQuery("SELECT * FROM docs"); + Assert.Equal(200, all.Count); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void BatchSqlUpdate_NoPrimaryKey_PageBased_AppliesUpdates() + { + // Same scenario on the PageBased engine: without the position pass-through, updates on a + // PK-less table were silently dropped (no PK → no engine.Update). They must now be applied. + var config = new DatabaseConfig + { + NoEncryptMode = true, + StorageEngineType = StorageEngineType.PageBased, + EnableHashIndexes = true, + UseMemoryMapping = true, + WalDurabilityMode = DurabilityMode.Async, + }; + + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw", isReadOnly: false, config: config); + try + { + db.ExecuteSQL(@"CREATE TABLE docs ( + name TEXT NOT NULL, + score REAL + )"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + var rows = new List>(50); + for (int i = 0; i < 50; i++) + { + rows.Add(new Dictionary + { + ["name"] = $"User{i}", + ["score"] = i * 1.5, + }); + } + db.InsertBatch("docs", rows); + + var stmts = new List(50); + for (int i = 0; i < 50; i++) + { + stmts.Add(string.Format(System.Globalization.CultureInfo.InvariantCulture, + "UPDATE docs SET score = {0:F1} WHERE name = 'User{1}'", i * 77.7, i)); + } + db.ExecuteBatchSQL(stmts); + + Assert.Equal(50, db.ExecuteQuery("SELECT * FROM docs").Count); + var updated = db.ExecuteQuery("SELECT * FROM docs WHERE name = @n", + new Dictionary { ["@n"] = "User7" }); + Assert.Single(updated); + Assert.Equal(7 * 77.7, (double)updated[0]["score"], precision: 6); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void BatchSqlUpdate_NoPrimaryKey_Columnar_FileStable() + { + // Regression: ExecuteBatchSQL on a PK-less table resolves matching rows through the + // hash index but previously discarded the storage position. The columnar write path + // then could not patch in place → every update appended a new version (file growth, + // stale rows, compaction storm). With the position passed through, fixed-size fields + // are patched in place and the file size stays constant. + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL(@"CREATE TABLE docs ( + name TEXT NOT NULL, + email TEXT, + age INTEGER, + score REAL, + data TEXT + )"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + var rows = new List>(200); + for (int i = 0; i < 200; i++) + { + rows.Add(new Dictionary + { + ["name"] = $"User{i}", + ["email"] = $"user{i}@test.com", + ["age"] = 20 + i % 60, + ["score"] = i * 0.1, + ["data"] = $"payload-{i}", + }); + } + db.InsertBatch("docs", rows); + long sizeAfterInsert = DataFileSize("docs"); + Assert.True(sizeAfterInsert > 0); + + var stmts = new List(200); + for (int i = 0; i < 200; i++) + { + stmts.Add(string.Format(System.Globalization.CultureInfo.InvariantCulture, + "UPDATE docs SET score = {0:F1} WHERE name = 'User{1}'", i * 99.9, i)); + } + db.ExecuteBatchSQL(stmts); + + // No stale versions appended: same row count and unchanged file size. + Assert.Equal(200, db.ExecuteQuery("SELECT * FROM docs").Count); + Assert.Equal(sizeAfterInsert, DataFileSize("docs")); + + // Values are actually updated and visible through the hash-index lookup. + var updated = db.ExecuteQuery("SELECT * FROM docs WHERE name = @n", + new Dictionary { ["@n"] = "User42" }); + Assert.Single(updated); + Assert.Equal(42 * 99.9, (double)updated[0]["score"], precision: 6); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void BatchSqlUpdate_Rollback_RestoresOriginalValues() + { + // B7 regression: in-place overwrites inside a transaction are write-behind. Rollback + // must drop the buffered overwrites so the on-disk records stay byte-for-byte original. + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT NOT NULL, score REAL)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + + var rows = new List>(50); + for (int i = 0; i < 50; i++) + { + rows.Add(new Dictionary { ["name"] = $"User{i}", ["score"] = i * 1.5 }); + } + db.InsertBatch("t", rows); + + db.BeginStorageTransactionOnly(); + try + { + db.ExecuteBatchSQL(new[] { "UPDATE t SET score = 999.9 WHERE name = 'User42'" }); + + // Inside the transaction the new value is visible (buffered overwrite). + var inside = db.ExecuteQuery("SELECT * FROM t WHERE name = @n", + new Dictionary { ["@n"] = "User42" }); + Assert.Single(inside); + Assert.Equal(999.9, (double)inside[0]["score"], precision: 6); + } + finally + { + db.RollbackStorageTransaction(); + } + + // After rollback the pre-transaction value is restored and no rows were lost. + Assert.Equal(50, db.ExecuteQuery("SELECT * FROM t").Count); + var after = db.ExecuteQuery("SELECT * FROM t WHERE name = @n", + new Dictionary { ["@n"] = "User42" }); + Assert.Single(after); + Assert.Equal(42 * 1.5, (double)after[0]["score"], precision: 6); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void BatchSqlUpdate_FastPatch_NotNullViolationThrows() + { + // B7 fast patch: NOT NULL must still be validated on the changed values even though the + // full row is never deserialized. + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT NOT NULL, score REAL NOT NULL)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + db.ExecuteSQL("INSERT INTO t VALUES ('a', 1.0)"); + + var ex = Assert.Throws(() => + db.ExecuteBatchSQL(new[] { "UPDATE t SET score = NULL WHERE name = 'a'" })); + Assert.Contains("cannot be NULL", ex.Message, StringComparison.OrdinalIgnoreCase); + + // The row is unchanged. + var rows = db.ExecuteQuery("SELECT * FROM t"); + Assert.Single(rows); + Assert.Equal(1.0, (double)rows[0]["score"], precision: 6); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void BatchSqlUpdate_WithCheckConstraint_FallsBackAndApplies() + { + // B7 fast patch is disabled when a CHECK constraint exists (the constraint may read + // non-updated columns) — the full-row fallback must still apply the update correctly. + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL(@"CREATE TABLE t (name TEXT NOT NULL, score REAL CHECK (score >= 0))"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + db.ExecuteSQL("INSERT INTO t VALUES ('a', 1.0)"); + + db.ExecuteBatchSQL(new[] { "UPDATE t SET score = 42.5 WHERE name = 'a'" }); + var rows = db.ExecuteQuery("SELECT * FROM t"); + Assert.Single(rows); + Assert.Equal(42.5, (double)rows[0]["score"], precision: 6); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } + + [Fact] + public void BatchSqlUpdate_WhereColumnTouched_FallsBackAndApplies() + { + // B7 fast patch only applies when no indexed column changes. Updating the WHERE column + // itself must still work through the full-row path (including hash-index maintenance). + var db = (SharpCoreDB.Database)_factory.Create(_dirPath, "pw"); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT NOT NULL, score REAL)"); + db.ExecuteSQL("CREATE INDEX idx_t_name ON t(name)"); + db.ExecuteSQL("INSERT INTO t VALUES ('a', 1.0)"); + + db.ExecuteBatchSQL(new[] { "UPDATE t SET name = 'b' WHERE name = 'a'" }); + + Assert.Equal(0, db.ExecuteQuery("SELECT * FROM t WHERE name = @n", + new Dictionary { ["@n"] = "a" }).Count); + var rows = db.ExecuteQuery("SELECT * FROM t WHERE name = @n", + new Dictionary { ["@n"] = "b" }); + Assert.Single(rows); + Assert.Equal(1.0, (double)rows[0]["score"], precision: 6); + } + finally + { + (db as IDisposable)?.Dispose(); + } + } } diff --git a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/FixedWidthBenchmark.cs b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/FixedWidthBenchmark.cs new file mode 100644 index 00000000..656ea124 --- /dev/null +++ b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/FixedWidthBenchmark.cs @@ -0,0 +1,287 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. +// + +using System.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; + +namespace SharpCoreDB.Benchmarks.Comparative; + +/// +/// Before/after benchmark for the fixed-width record layout work (B1–B6): +/// the same workload runs against a legacy (variable-length records) database and a +/// fixed-width database, and the storage growth and elapsed time are compared. +/// Run with: dotnet run --project tests/benchmarks/SharpCoreDB.Benchmarks.Comparative -- --fixedwidth +/// +internal static class FixedWidthBenchmark +{ + private const int SameLengthUpdates = 10_000; + private const int VariableUpdates = 1_000; + private const int SelectRows = 100_000; + private const int SelectRounds = 30; + + public static void Run() + { + Console.WriteLine("╔══════════════════════════════════════════════════════════════╗"); + Console.WriteLine("║ Fixed-Width vs Legacy (variable-length) — storage & speed ║"); + Console.WriteLine("╚══════════════════════════════════════════════════════════════╝"); + Console.WriteLine(); + Console.WriteLine($"Runtime: {System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription}"); + Console.WriteLine(); + + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + var sp = services.BuildServiceProvider(); + var factory = sp.GetRequiredService(); + + Console.WriteLine("── Workload A: 10,000 growing variable-column updates (storage growth) ──"); + RunGrowingUpdates(factory); + Console.WriteLine(); + + Console.WriteLine("── Workload B: 1,000 variable-length updates + arena compaction (storage) ──"); + RunVariableUpdates(factory); + Console.WriteLine(); + + Console.WriteLine($"── Workload C: {SelectRounds} full scans, WHERE on a non-indexed INTEGER column over {SelectRows:N0} rows ──"); + RunSelectWhere(factory); + Console.WriteLine(); + + Console.WriteLine("── Workload D: batch INSERT throughput ──"); + RunInsertThroughput(factory); + } + + private static DatabaseConfig BuildConfig(bool fixedWidth) => new() + { + NoEncryptMode = true, + UseGroupCommitWal = false, + EnableAdaptiveWalBatching = false, + HighSpeedInsertMode = true, + GroupCommitSize = 1000, + WalDurabilityMode = Services.DurabilityMode.Async, + EnablePageCache = true, + PageCacheCapacity = 10_000, + UseMemoryMapping = true, + UseBufferedIO = true, + EnableHashIndexes = true, + EnableQueryCache = false, + EnableBTreeSelection = true, + EnableSimdAndProjectionPushdown = true, + SqlValidationMode = Services.SqlQueryValidator.ValidationMode.Disabled, + StrictParameterValidation = false, + FixedWidthRecordLayout = fixedWidth, + }; + + private static (Database db, string dir) CreateDatabase(DatabaseFactory factory, bool fixedWidth) + { + var dir = Path.Combine(Path.GetTempPath(), $"scdb_fwbench_{fixedWidth}_{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + var db = (Database)factory.Create(dir, "bench123", isReadOnly: false, config: BuildConfig(fixedWidth)); + return (db, dir); + } + + private static long DataBytes(string dir) + { + long total = 0; + foreach (var file in Directory.EnumerateFiles(dir)) + { + var name = Path.GetFileName(file); + if (name.EndsWith(".dat", StringComparison.OrdinalIgnoreCase) || + name.EndsWith(".ovf", StringComparison.OrdinalIgnoreCase)) + { + total += new FileInfo(file).Length; + } + } + + return total; + } + + private static void RunGrowingUpdates(DatabaseFactory factory) + { + // Growing values force the legacy path to append a new record per update (its .dat grows + // with every new record length). Fixed-width keeps the .dat constant and grows only the + // overflow arena, which the auto-compaction (1000-update threshold) keeps bounded. + foreach (var fixedWidth in new[] { true, false }) + { + var label = fixedWidth ? "fixed-width" : "legacy "; + var (db, dir) = CreateDatabase(factory, fixedWidth); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + + // Warm-up (JIT + index structures). + for (int i = 0; i < 100; i++) + { + db.ExecuteSQL("UPDATE t SET name = 'warmup-value' WHERE id = 1"); + } + + long startBytes = DataBytes(dir); + var sw = Stopwatch.StartNew(); + for (int i = 0; i < SameLengthUpdates; i++) + { + var value = new string((char)('A' + (i % 26)), 1 + (i % 200)); + db.ExecuteSQL($"UPDATE t SET name = '{value}' WHERE id = 1"); + } + + sw.Stop(); + long growth = DataBytes(dir) - startBytes; + Console.WriteLine($" {label}: {SameLengthUpdates:N0} updates in {sw.Elapsed.TotalSeconds:F2}s, storage growth {growth / 1024.0:F1} KB"); + } + finally + { + (db as IDisposable)?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + } + + private static void RunVariableUpdates(DatabaseFactory factory) + { + foreach (var fixedWidth in new[] { true, false }) + { + var label = fixedWidth ? "fixed-width" : "legacy "; + var (db, dir) = CreateDatabase(factory, fixedWidth); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a')"); + for (int i = 0; i < 100; i++) + { + db.ExecuteSQL("UPDATE t SET name = 'warmup-value' WHERE id = 1"); + } + + long startBytes = DataBytes(dir); + + var sw = Stopwatch.StartNew(); + for (int i = 0; i < VariableUpdates; i++) + { + var value = new string((char)('a' + (i % 26)), (i % 50) + 1); + db.ExecuteSQL($"UPDATE t SET name = '{value}' WHERE id = 1"); + } + + // Force the auto-compaction (B3) so the arena GC is measured too. + Assert(db.TryGetTable("t", out var table)); + var concrete = (DataStructures.Table)table; + concrete.CompactStorage(); + + sw.Stop(); + long growth = DataBytes(dir) - startBytes; + Console.WriteLine($" {label}: {VariableUpdates:N0} updates + compact in {sw.Elapsed.TotalSeconds:F2}s, storage growth {growth:N0} bytes"); + } + finally + { + (db as IDisposable)?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + } + + private static void RunSelectWhere(DatabaseFactory factory) + { + foreach (var fixedWidth in new[] { true, false }) + { + var label = fixedWidth ? "fixed-width" : "legacy "; + var (db, dir) = CreateDatabase(factory, fixedWidth); + try + { + db.ExecuteSQL("CREATE TABLE s (id INTEGER PRIMARY KEY, category INTEGER, payload TEXT)"); + for (int i = 0; i < SelectRows; i += 1000) + { + var rows = new List>(1000); + for (int j = 0; j < 1000; j++) + { + rows.Add(new Dictionary + { + ["id"] = i + j, + ["category"] = i + j, + ["payload"] = $"payload-{i + j}-with-some-length", + }); + } + + db.InsertBatch("s", rows); + } + + // Warm-up + db.ExecuteQuery("SELECT * FROM s WHERE category = -1"); + + var sw = Stopwatch.StartNew(); + long rowsReturned = 0; + for (int r = 0; r < SelectRounds; r++) + { + rowsReturned += db.ExecuteQuery("SELECT * FROM s WHERE category = -1").Count; + } + + sw.Stop(); + double perQueryMs = sw.Elapsed.TotalMilliseconds / SelectRounds; + Console.WriteLine($" {label}: {SelectRounds} scans ({SelectRows:N0} rows each, non-indexed WHERE) in {sw.Elapsed.TotalSeconds:F2}s → {perQueryMs:F2} ms/query (rows returned: {rowsReturned})"); + } + finally + { + (db as IDisposable)?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + } + + private static void RunInsertThroughput(DatabaseFactory factory) + { + const int InsertCount = 100_000; + const int BatchSize = 5_000; + + foreach (var fixedWidth in new[] { true, false }) + { + var label = fixedWidth ? "fixed-width" : "legacy "; + var (db, dir) = CreateDatabase(factory, fixedWidth); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL, flag BOOLEAN)"); + + // Warm-up batch. + db.InsertBatch("t", Enumerable.Range(0, 1000).Select(i => new Dictionary + { + ["id"] = i, + ["name"] = $"warm-{i}", + ["score"] = i * 0.5, + ["flag"] = (i & 1) == 0, + }).ToList()); + + var sw = Stopwatch.StartNew(); + for (int batch = 0; batch < InsertCount; batch += BatchSize) + { + var rows = new List>(BatchSize); + for (int i = batch; i < batch + BatchSize && i < InsertCount; i++) + { + rows.Add(new Dictionary + { + ["id"] = 1000 + i, + ["name"] = $"user-{i}", + ["score"] = i * 0.5, + ["flag"] = (i & 1) == 0, + }); + } + + db.InsertBatch("t", rows); + } + + sw.Stop(); + double perSec = InsertCount / sw.Elapsed.TotalSeconds; + Console.WriteLine($" {label}: {InsertCount:N0} inserts in {sw.Elapsed.TotalSeconds:F2}s → {perSec:N0} rows/s"); + } + finally + { + (db as IDisposable)?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + } + + private static void Assert(bool condition) + { + if (!condition) + { + throw new InvalidOperationException("Assertion failed"); + } + } +} diff --git a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs index 48381ce8..1a6ac5f4 100644 --- a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs +++ b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs @@ -30,6 +30,41 @@ private Program() { } // Static utility class - prevent instantiation. static async Task Main(string[] args) { + // Optional: --readtest → focused SQL-vs-Direct read micro-benchmark (median of N runs). + if (args.Any(a => a.Equals("--readtest", StringComparison.OrdinalIgnoreCase))) + { + RunReadMicroBenchmark(); + return; + } + + // Optional: --inserttest → focused SQL-vs-Direct insert micro-benchmark (median of N runs). + if (args.Any(a => a.Equals("--inserttest", StringComparison.OrdinalIgnoreCase))) + { + RunInsertMicroBenchmark(); + return; + } + + // Optional: --fixedwidth → run the fixed-width vs legacy before/after benchmark only. + if (args.Any(a => a.Equals("--fixedwidth", StringComparison.OrdinalIgnoreCase))) + { + FixedWidthBenchmark.Run(); + return; + } + + // Optional: --pk → fair PK-based comparison: SharpCoreDB on a table with an + // `id INTEGER PRIMARY KEY` (mirroring the SQLite harness schema) with UPDATE/DELETE by PK, + // so the PK B-tree fast paths and the recommended usage are measured vs SQLite. + if (args.Any(a => a.Equals("--pk", StringComparison.OrdinalIgnoreCase))) + { + var engineArgPk = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase)); + var engineTypePk = engineArgPk is not null + && engineArgPk.Substring("--engine=".Length).Equals("pagebased", StringComparison.OrdinalIgnoreCase) + ? SharpCoreDB.Interfaces.StorageEngineType.PageBased + : SharpCoreDB.Interfaces.StorageEngineType.AppendOnly; + RunPkComparison(engineTypePk); + return; + } + // Optional: --engine=appendonly (default) | --engine=pagebased // PageBased is the v2.0 in-place-update engine (WP10-WP13 storage engine roadmap). var engineArg = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase)); @@ -105,6 +140,213 @@ static async Task Main(string[] args) // ══════════════════════════════════════ // SharpCoreDB // ══════════════════════════════════════ + + /// + /// Focused read micro-benchmark: SQL (parameterized point-lookup) vs Direct API + /// (FindByIndex) on the same database. Reports the median of several runs so + /// machine load does not dominate the result. + /// + static void RunReadMicroBenchmark() + { + const int rows = 100_000; + const int queries = 10_000; + const int reps = 7; + + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + var sp = services.BuildServiceProvider(); + var factory = sp.GetRequiredService(); + var config = BuildConfig(SharpCoreDB.Interfaces.StorageEngineType.AppendOnly); + var dbPath = Path.Combine(Path.GetTempPath(), $"scdb-readtest-{Guid.NewGuid()}"); + + using var db = (SharpCoreDB.Database)factory.Create( + dbPath: dbPath, + masterPassword: "pw", + isReadOnly: false, + config: config); + + try + { + db.ExecuteSQL("CREATE TABLE docs (name TEXT NOT NULL, email TEXT, age INTEGER, score REAL, data TEXT)"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + for (int batch = 0; batch < rows; batch += 10_000) + { + var list = new List>(10_000); + for (int i = batch; i < batch + 10_000; i++) + { + list.Add(new Dictionary + { + ["name"] = $"User{i}", + ["email"] = $"user{i}@test.com", + ["age"] = 20 + i % 60, + ["score"] = i * 0.1, + ["data"] = $"payload-{i}", + }); + } + + db.InsertBatch("docs", list); + } + + db.Flush(); + + // Warmup (JIT + index load). + for (int i = 0; i < 1000; i++) + { + db.ExecuteQuery("SELECT * FROM docs WHERE name = @name", + new Dictionary { ["@name"] = $"User{i}" }); + db.FindByIndex("docs", "name", $"User{i}"); + } + + double[] sqlTimes = new double[reps]; + double[] directTimes = new double[reps]; + + for (int r = 0; r < reps; r++) + { + var sw = Stopwatch.StartNew(); + for (int i = 0; i < queries; i++) + { + db.ExecuteQuery("SELECT * FROM docs WHERE name = @name", + new Dictionary { ["@name"] = $"User{i}" }); + } + + sw.Stop(); + sqlTimes[r] = sw.Elapsed.TotalSeconds; + + sw.Restart(); + for (int i = 0; i < queries; i++) + { + db.FindByIndex("docs", "name", $"User{i}"); + } + + sw.Stop(); + directTimes[r] = sw.Elapsed.TotalSeconds; + } + + Array.Sort(sqlTimes); + Array.Sort(directTimes); + double sqlMedian = sqlTimes[reps / 2]; + double directMedian = directTimes[reps / 2]; + + Console.WriteLine(); + Console.WriteLine("═══ READ micro-benchmark (10,000 point reads via name hash index, median of 7) ═══"); + Console.WriteLine($" SQL : {sqlMedian:F3}s ({queries / sqlMedian:N0} ops/s)"); + Console.WriteLine($" Direct : {directMedian:F3}s ({queries / directMedian:N0} ops/s)"); + Console.WriteLine($" SQL/Direct overhead: {(sqlMedian / directMedian):F2}x"); + } + finally + { + try { Directory.Delete(dbPath, true); } + catch { } + } + } + + /// + /// Focused insert micro-benchmark: SQL (ExecuteBatchSQL with INSERT statements) vs + /// Direct API (InsertBatch). Each repetition runs on a fresh database so append-only + /// growth and unique keys do not skew the result; median of several runs is reported. + /// + static void RunInsertMicroBenchmark() + { + const int inserts = 50_000; + const int batch = 10_000; + const int reps = 5; + + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + var sp = services.BuildServiceProvider(); + var factory = sp.GetRequiredService(); + var config = BuildConfig(SharpCoreDB.Interfaces.StorageEngineType.AppendOnly); + + double[] sqlTimes = new double[reps]; + double[] directTimes = new double[reps]; + + for (int r = 0; r < reps; r++) + { + var sqlPath = Path.Combine(Path.GetTempPath(), $"scdb-insert-sql-{Guid.NewGuid()}"); + using (var db = (SharpCoreDB.Database)factory.Create(sqlPath, "pw", isReadOnly: false, config: config)) + { + db.ExecuteSQL("CREATE TABLE docs (name TEXT NOT NULL, email TEXT, age INTEGER, score REAL, data TEXT)"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + // Build the statements once (outside the timed region — this is caller work, + // identical for SQLite in the comparative benchmark). + var stmtBatches = new List>(); + for (int b = 0; b < inserts; b += batch) + { + var stmts = new List(batch); + for (int i = b; i < b + batch; i++) + { + stmts.Add(string.Format(CultureInfo.InvariantCulture, + "INSERT INTO docs VALUES ('User{0}', 'user{0}@test.com', {1}, {2}, 'payload-{0}')", + i, 20 + i % 60, i * 0.1)); + } + + stmtBatches.Add(stmts); + } + + var sw = Stopwatch.StartNew(); + foreach (var stmts in stmtBatches) + { + db.ExecuteBatchSQL(stmts); + } + + sw.Stop(); + sqlTimes[r] = sw.Elapsed.TotalSeconds; + } + + try { Directory.Delete(sqlPath, true); } catch { } + + var directPath = Path.Combine(Path.GetTempPath(), $"scdb-insert-direct-{Guid.NewGuid()}"); + using (var db = (SharpCoreDB.Database)factory.Create(directPath, "pw", isReadOnly: false, config: config)) + { + db.ExecuteSQL("CREATE TABLE docs (name TEXT NOT NULL, email TEXT, age INTEGER, score REAL, data TEXT)"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + var rowBatches = new List>>(); + for (int b = 0; b < inserts; b += batch) + { + var rows = new List>(batch); + for (int i = b; i < b + batch; i++) + { + rows.Add(new Dictionary + { + ["name"] = $"User{i}", + ["email"] = $"user{i}@test.com", + ["age"] = 20 + i % 60, + ["score"] = i * 0.1, + ["data"] = $"payload-{i}", + }); + } + + rowBatches.Add(rows); + } + + var sw = Stopwatch.StartNew(); + foreach (var rows in rowBatches) + { + db.InsertBatch("docs", rows); + } + + sw.Stop(); + directTimes[r] = sw.Elapsed.TotalSeconds; + } + + try { Directory.Delete(directPath, true); } catch { } + } + + Array.Sort(sqlTimes); + Array.Sort(directTimes); + double sqlMedian = sqlTimes[reps / 2]; + double directMedian = directTimes[reps / 2]; + + Console.WriteLine(); + Console.WriteLine($"═══ INSERT micro-benchmark ({inserts:N0} batched inserts, median of {reps}) ═══"); + Console.WriteLine($" SQL : {sqlMedian:F3}s ({inserts / sqlMedian:N0} ops/s)"); + Console.WriteLine($" Direct : {directMedian:F3}s ({inserts / directMedian:N0} ops/s)"); + Console.WriteLine($" SQL/Direct overhead: {(sqlMedian / directMedian):F2}x"); + } + static DatabaseConfig BuildConfig(SharpCoreDB.Interfaces.StorageEngineType engineType) { return new DatabaseConfig @@ -588,6 +830,164 @@ data TEXT return result; } + // ══════════════════════════════════════ + // PK-based "fair usage" SharpCoreDB scenario + // ══════════════════════════════════════ + + /// + /// SharpCoreDB on the same schema/API shape SQLite gets in the harness: an + /// id INTEGER PRIMARY KEY table, batched inserts, and UPDATE/DELETE by primary key via + /// ExecuteBatchSQL (single transaction). This exercises the PK B-tree fast paths and the + /// recommended usage; the no-PK harness scenario above under-measures the engine on DML. + /// + static BenchmarkResult RunSharpCoreDBPk(SharpCoreDB.Interfaces.StorageEngineType engineType) + { + var dbPath = Path.Combine(Path.GetTempPath(), $"bench-sharpcoredb-pk-{Guid.NewGuid()}"); + var result = new BenchmarkResult(); + + try + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + var sp = services.BuildServiceProvider(); + + var factory = sp.GetRequiredService(); + var config = BuildConfig(engineType); + + using var db = (SharpCoreDB.Database)factory.Create( + dbPath: dbPath, + masterPassword: "bench123", + isReadOnly: false, + config: config); + + db.ExecuteSQL(@"CREATE TABLE docs ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT, + age INTEGER, + score REAL, + data TEXT + )"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + // INSERT (batched via InsertBatch with explicit ids, mirroring SQLite's rowid 1..N) + var sw = Stopwatch.StartNew(); + for (int batch = 0; batch < InsertCount; batch += BatchSize) + { + int end = Math.Min(batch + BatchSize, InsertCount); + var rows = new List>(end - batch); + for (int i = batch; i < end; i++) + { + rows.Add(new Dictionary + { + ["id"] = i + 1, + ["name"] = $"User{i}", + ["email"] = $"user{i}@test.com", + ["age"] = 20 + i % 60, + ["score"] = i * 0.1, + ["data"] = $"payload-{i}", + }); + } + + db.InsertBatch("docs", rows); + } + + db.Flush(); + sw.Stop(); + result.InsertTime = sw.Elapsed.TotalSeconds; + result.InsertOpsPerSec = (int)(InsertCount / result.InsertTime); + Console.WriteLine($" INSERT {InsertCount:N0}: {result.InsertTime:F2}s ({result.InsertOpsPerSec:N0} ops/sec)"); + + // READ by PK + sw.Restart(); + for (int i = 1; i <= ReadCount; i++) + { + db.ExecuteQuery("SELECT * FROM docs WHERE id = @id", new Dictionary { ["@id"] = i }); + } + + sw.Stop(); + result.ReadTime = sw.Elapsed.TotalSeconds; + result.ReadOpsPerSec = (int)(ReadCount / result.ReadTime); + Console.WriteLine($" READ {ReadCount:N0}: {result.ReadTime:F2}s ({result.ReadOpsPerSec:N0} ops/sec)"); + + // UPDATE by PK (single ExecuteBatchSQL transaction, like SQLite's single tx) + sw.Restart(); + var updateStmts = new List(UpdateCount); + for (int i = 1; i <= UpdateCount; i++) + { + updateStmts.Add(string.Format(CultureInfo.InvariantCulture, + "UPDATE docs SET score = {0:F1} WHERE id = {1}", i * 99.9, i)); + } + + db.ExecuteBatchSQL(updateStmts); + db.Flush(); + sw.Stop(); + result.UpdateTime = sw.Elapsed.TotalSeconds; + result.UpdateOpsPerSec = (int)(UpdateCount / result.UpdateTime); + Console.WriteLine($" UPDATE {UpdateCount:N0}: {result.UpdateTime:F2}s ({result.UpdateOpsPerSec:N0} ops/sec)"); + + // DELETE by PK + sw.Restart(); + var deleteStmts = new List(DeleteCount); + for (int i = 1; i <= DeleteCount; i++) + { + deleteStmts.Add($"DELETE FROM docs WHERE id = {i}"); + } + + db.ExecuteBatchSQL(deleteStmts); + db.Flush(); + sw.Stop(); + result.DeleteTime = sw.Elapsed.TotalSeconds; + result.DeleteOpsPerSec = (int)(DeleteCount / result.DeleteTime); + Console.WriteLine($" DELETE {DeleteCount:N0}: {result.DeleteTime:F2}s ({result.DeleteOpsPerSec:N0} ops/sec)"); + } + finally + { + try { if (Directory.Exists(dbPath)) Directory.Delete(dbPath, true); } catch { /* temp */ } + } + + return result; + } + + /// + /// Runs the fair PK scenario (SharpCoreDB vs SQLite) and prints the comparison. + /// + static void RunPkComparison(SharpCoreDB.Interfaces.StorageEngineType engineType) + { + var engineLabel = engineType == SharpCoreDB.Interfaces.StorageEngineType.PageBased ? "PageBased" : "AppendOnly"; + Console.WriteLine("╔══════════════════════════════════════════════════════════╗"); + Console.WriteLine("║ Fair PK comparison: SharpCoreDB vs SQLite ║"); + Console.WriteLine("║ (id INTEGER PRIMARY KEY, UPDATE/DELETE by PK) ║"); + Console.WriteLine("╚══════════════════════════════════════════════════════════╝"); + Console.WriteLine(); + Console.WriteLine($"Engine: {engineLabel}"); + + Console.WriteLine("━━━ SharpCoreDB (SQL, PK) ━━━"); + var scdb = RunSharpCoreDBPk(engineType); + Console.WriteLine(); + + Console.WriteLine("━━━ SQLite (reference) ━━━"); + var sqlite = RunSQLite(); + Console.WriteLine(); + + Console.WriteLine("║ Database │ INSERT │ READ │ UPDATE │ DELETE ║"); + Console.WriteLine($"║ SharpCoreDB │ {scdb.InsertOpsPerSec,10:N0} │ {scdb.ReadOpsPerSec,8:N0} │ {scdb.UpdateOpsPerSec,8:N0} │ {scdb.DeleteOpsPerSec,8:N0} ║"); + Console.WriteLine($"║ SQLite │ {sqlite.InsertOpsPerSec,10:N0} │ {sqlite.ReadOpsPerSec,8:N0} │ {sqlite.UpdateOpsPerSec,8:N0} │ {sqlite.DeleteOpsPerSec,8:N0} ║"); + Console.WriteLine($"\n UPDATE gap: {sqlite.UpdateOpsPerSec / (double)scdb.UpdateOpsPerSec:F1}x DELETE gap: {sqlite.DeleteOpsPerSec / (double)scdb.DeleteOpsPerSec:F1}x"); + + var results = new Dictionary + { + ["SharpCoreDB (SQL, PK)"] = scdb, + ["SQLite"] = sqlite, + }; + + var dir = "results"; + Directory.CreateDirectory(dir); + var path = Path.Combine(dir, $"pk_comparative_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + File.WriteAllText(path, JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true })); + Console.WriteLine($"\nResults saved to: {path}"); + } + // ══════════════════════════════════════ // LiteDB // ══════════════════════════════════════