From ce5e2a030060c91fb3fb106eafca5196765d8447 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:20:53 +0000 Subject: [PATCH 01/10] feat: keep Date/Map/Set Encoded native and JSON-convert in stores Query filters now take Date/Set/Map values. Cosmos/SQL/memory adapters lower those to JSON (ISO strings, arrays, entries), and repository persistence round-trips through Schema.toCodecJson. Use DateFromString / ReadonlySetFromArray / ReadonlyMapFromArray when Encoded must be JSON. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- .changeset/native-encoded-query-json.md | 9 ++ .../src/Model/Repository/internal/internal.ts | 34 +++--- .../effect-app/src/Model/filter/filterApi.ts | 2 +- packages/effect-app/src/Schema/ext.ts | 57 ++++------ packages/effect-app/test/schema.test.ts | 57 +++++----- packages/infra/examples/query.ts | 8 +- packages/infra/src/Store/Cosmos/query.ts | 15 ++- packages/infra/src/Store/SQL/query.ts | 15 +-- packages/infra/src/Store/codeFilter.ts | 47 ++++---- packages/infra/src/Store/utils.ts | 57 +++++++++- packages/infra/test/cosmos-query.test.ts | 36 ++++++- packages/infra/test/query.test.ts | 29 ++--- packages/infra/test/sql-store.test.ts | 100 +++++++++++------- .../OmegaForm/DateValidation.test.ts | 2 +- .../src/components/OmegaForm/meta/checks.ts | 5 +- .../stories/OmegaForm/AutoGeneration.vue | 2 +- .../vue-components/stories/OmegaForm/Date.vue | 2 +- 17 files changed, 295 insertions(+), 182 deletions(-) create mode 100644 .changeset/native-encoded-query-json.md diff --git a/.changeset/native-encoded-query-json.md b/.changeset/native-encoded-query-json.md new file mode 100644 index 000000000..50002599f --- /dev/null +++ b/.changeset/native-encoded-query-json.md @@ -0,0 +1,9 @@ +--- +"effect-app": minor +"@effect-app/infra": minor +"@effect-app/vue-components": minor +--- + +Stop forcing Date/Map/Set Encoded shapes to JSON. + +`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values; document-store adapters convert them with `Schema.toCodecJson` (Date → ISO string, Set → array, Map → entries). Repository persistence also round-trips through `toCodecJson` so JSON stores stay compatible. diff --git a/packages/effect-app/src/Model/Repository/internal/internal.ts b/packages/effect-app/src/Model/Repository/internal/internal.ts index 33db662b3..0ffc556e1 100644 --- a/packages/effect-app/src/Model/Repository/internal/internal.ts +++ b/packages/effect-app/src/Model/Repository/internal/internal.ts @@ -34,6 +34,9 @@ import { ValidationError, ValidationResult } from "../validation.ts" const dedupe = Array.dedupeWith(Equivalence.String) +/** JSON persistence codec: Encoded Date/Map/Set become JSON while the store stays typed as Encoded. */ +const persistJson = (schema: S.Codec) => S.toCodecJson(schema) as unknown as S.Codec + // ms buckets: dense under 100ms (common path), then mid-tail and multi-second stalls. // Rare 0.5–1s+ encodes (fat aggregates) must not collapse into a single overflow bin. const schemaDurationBoundaries = [ @@ -179,14 +182,17 @@ export function makeRepoInternal< .gen(function*() { const rctx: Context.Context = args.schemaContext ?? Context.empty() as any const provideRctx = Effect.provide(rctx) + // Persist via JSON codec so Date/Map/Set Encoded values round-trip through + // document stores. Query filters keep native Encoded; adapters lower to JSON. + const persistCodec = persistJson(schema) const encodeMany = (items: readonly T[]) => - S.encodeEffect(S.Array(schema))(items).pipe( + S.encodeEffect(S.Array(persistCodec))(items).pipe( provideRctx, timeSchema("encode", name, undefined, items.length, entityStateFromItems(items)) ) - const decode = flow(S.decodeEffectConcurrently(schema), provideRctx) + const decode = flow(S.decodeEffectConcurrently(persistCodec), provideRctx) const decodeMany = flow( - S.decodeEffectConcurrently(S.Array(schema)), + S.decodeEffectConcurrently(S.Array(persistCodec)), provideRctx ) @@ -522,7 +528,7 @@ export function makeRepoInternal< const getDecodeMany = (s: S.Codec) => { let dec = decodeManyCache.get(s) if (!dec) { - dec = S.decodeEffectConcurrently(S.Array(s)) + dec = S.decodeEffectConcurrently(S.Array(persistJson(s))) decodeManyCache.set(s, dec) } return dec @@ -581,7 +587,7 @@ export function makeRepoInternal< .pipe( Effect.andThen( (items) => - S.decodeEffectConcurrently(S.Array(a.schema ?? schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(persistJson(a.schema ?? schema)))(items).pipe( provideRctx, timeSchema("decode", name, "aggregate", items.length) ) @@ -593,7 +599,7 @@ export function makeRepoInternal< .pipe( Effect.andThen( (items) => - S.decodeEffectConcurrently(S.Array(a.schema ?? schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(persistJson(a.schema ?? schema)))(items).pipe( provideRctx, timeSchema("decode", name, "project", items.length) ) @@ -604,7 +610,7 @@ export function makeRepoInternal< // TODO: mapFrom but need to support per field and dependencies .pipe( Effect.flatMap((items) => - S.decodeEffectConcurrently(S.Array(a.schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(persistJson(a.schema)))(items).pipe( Effect.map(Array.getSomes), provideRctx, timeSchema("decode", name, "collect", items.length) @@ -697,7 +703,7 @@ export function makeRepoInternal< const rawData = rawResult.value as Encoded const jitMResult = mapFrom(rawData) // apply jitM - const decodeResult = yield* S.decodeEffectConcurrently(schema)(jitMResult).pipe( + const decodeResult = yield* S.decodeEffectConcurrently(persistCodec)(jitMResult).pipe( Effect.result, provideRctx ) @@ -737,7 +743,7 @@ export function makeRepoInternal< queryRaw(schema: S.Codec, q: Q.RawQuery) { return store.queryRaw(q).pipe( Effect.flatMap((items) => - S.decodeEffectConcurrently(S.Array(schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(persistJson(schema)))(items).pipe( timeSchema("decode", name, undefined, items.length) ) ), @@ -756,9 +762,10 @@ export function makeRepoInternal< * @internal */ mapped: (schema: S.Codec) => { - const dec = S.decodeEffectConcurrently(schema) - const encMany = S.encodeEffect(S.Array(schema)) - const decMany = S.decodeEffectConcurrently(S.Array(schema)) + const persistMapped = persistJson(schema) + const dec = S.decodeEffectConcurrently(persistMapped) + const encMany = S.encodeEffect(S.Array(persistMapped)) + const decMany = S.decodeEffectConcurrently(S.Array(persistMapped)) const spanAttrs = { kind: "client" as const, attributes: { "app.entity": name } } return { all: allE.pipe( @@ -856,8 +863,9 @@ export function makeStore() { ) { function encodeToEncoded() { const getEtag = () => undefined + const persistCodec = persistJson(schema) return (t: T) => - S.encodeEffect(schema)(t).pipe( + S.encodeEffect(persistCodec)(t).pipe( Effect.orDie, Effect.map((_) => mapToPersistenceModel(_, getEtag)) ) diff --git a/packages/effect-app/src/Model/filter/filterApi.ts b/packages/effect-app/src/Model/filter/filterApi.ts index aba49c20b..83e014bc2 100644 --- a/packages/effect-app/src/Model/filter/filterApi.ts +++ b/packages/effect-app/src/Model/filter/filterApi.ts @@ -44,7 +44,7 @@ export type FilterR = { op: Ops path: string - value: string // ToDO: Value[] + value: unknown } export type FilterResult = diff --git a/packages/effect-app/src/Schema/ext.ts b/packages/effect-app/src/Schema/ext.ts index 0311be652..df0fafeb3 100644 --- a/packages/effect-app/src/Schema/ext.ts +++ b/packages/effect-app/src/Schema/ext.ts @@ -82,8 +82,6 @@ export const withDefaultParseOptions = ( return (input: any, options?: SchemaAST.ParseOptions) => run(input, { ...defaultParseOptions, ...options }) }) as Decode -// TODO: v4 migration - Date is no longer by default encoded to string. - const DateString = S.String.annotate({ identifier: "DateOrInvalid", description: "an ISO 8601 date string that will be decoded as a Date (may be invalid)", @@ -107,12 +105,15 @@ export interface DateFromString extends S.decodeTo {} * Encoding: * - A `Date` is encoded as a `string`. * + * Use this when the Encoded form must be JSON (`string`). Domain models should + * prefer {@link Date}, whose Encoded form is `Date`; JSON stores convert via + * `Schema.toCodecJson`. + * * @since 4.0.0 */ export const DateFromString: DateFromString = DateString.pipe(S.decodeTo(S.Date, SchemaTransformation.dateFromString)) -/** Like the default Schema `Date` but from String, with default helpers. */ -export const Date = extendM(DateFromString, (s) => ({ +const dateHelpers = (s: S.Date) => ({ /** * Construction-only default `new Date()`. Applied only when the field is * omitted from `.make(...)` input. NOT applied during decode — cannot be @@ -127,37 +128,17 @@ export const Date = extendM(DateFromString, (s) => ({ * file-level note. */ withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new global.Date()))) -})) - -const DateValidString = S.String.annotate({ - identifier: "Date", - description: "a valid ISO 8601 date string that will be decoded as a Date", - format: "date-time" }) -// Schema.Date rejects invalid Dates since beta.91+; no separate isDateValid check needed. -const DateValidFromString = DateValidString - .pipe( - S.decodeTo(S.Date, SchemaTransformation.dateFromString) - ) +/** Like the default Schema `Date` (Encoded is `Date`) with default helpers. */ +export const Date = extendM(S.Date, dateHelpers) -/** Like the default Schema `Date` (valid only) but from String, with default helpers. */ -export const DateValid = extendM(DateValidFromString, (s) => ({ - /** - * Construction-only default `new Date()`. Applied only when the field is - * omitted from `.make(...)` input. NOT applied during decode — cannot be - * used to JIT-migrate database fields. See file-level note. - */ - withConstructorDefault: s.pipe(S.withConstructorDefault(Effect.sync(() => new global.Date()))), - /** - * Decode-time default `new Date()`. **Discouraged for persisted data:** a - * missing field may be data corruption, not an old-shape document; silently - * substituting `new Date()` hides the problem. Prefer an explicit, - * preferably versioned migration over a decode-time fallback. See - * file-level note. - */ - withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new global.Date()))) -})) +/** + * Alias of {@link Date}. Core `Schema.Date` already rejects invalid Dates. + * + * @deprecated Use {@link Date}. + */ +export const DateValid = Date /** Like the default Schema `Boolean` but with default helpers. */ export const Boolean = Object.assign(S.Boolean, { @@ -337,10 +318,10 @@ export const ReadonlyMapFromArray = (value: ValueSchema) => pipe( - ReadonlySetFromArray(value), + S.ReadonlySet(value), (s) => Object.assign(s, { /** @@ -365,13 +346,13 @@ export const ReadonlySet = (value: ValueSchema) => }) ) -/** Like the default Schema `ReadonlyMap` but from Array, with default helpers. */ +/** Like the default Schema `ReadonlyMap` (Encoded is `Map`) with default helpers. */ export const ReadonlyMap = (pair: { readonly key: KeySchema readonly value: ValueSchema }) => pipe( - ReadonlyMapFromArray(pair), + S.ReadonlyMap(pair.key, pair.value), (s) => Object.assign(s, { /** @@ -503,9 +484,9 @@ export type WithDefaults = ( // export type UnionToIntersection3 = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I // : never -/** Union of core `Schema.Date` (Date objects) and string-encoded `Date`, with default helpers. */ +/** Union of core `Schema.Date` (Date objects) and string-encoded `DateFromString`, with default helpers. */ export const inputDate = extendM( - S.Union([S.Date, Date]), + S.Union([S.Date, DateFromString]), (s) => ({ /** * Construction-only default `new Date()`. Applied only when the field is diff --git a/packages/effect-app/test/schema.test.ts b/packages/effect-app/test/schema.test.ts index 976220a7a..871743b0f 100644 --- a/packages/effect-app/test/schema.test.ts +++ b/packages/effect-app/test/schema.test.ts @@ -271,8 +271,8 @@ test("TaggedUnion match dispatches on _tag", () => { A: (v) => `got A: ${v.a}`, B: (v) => `got B: ${v.b}` }) - expect(matcher({ _tag: "A", a: "hello" } as T)).toBe("got A: hello") - expect(matcher({ _tag: "B", b: 42 } as T)).toBe("got B: 42") + expect(matcher({ _tag: "A", a: "hello" })).toBe("got A: hello") + expect(matcher({ _tag: "B", b: 42 })).toBe("got B: 42") }) test("TaggedUnion with single member", () => { @@ -320,7 +320,7 @@ test("TaggedUnion with encodeKeys renaming a non-tag key", () => { // encode back to snake_case type T = S.Schema.Type - const encoded = S.encodeSync(schema)({ _tag: "A", firstName: "Alice" } as T) + const encoded = S.encodeSync(schema)({ _tag: "A", firstName: "Alice" }) expect(encoded).toEqual({ _tag: "A", first_name: "Alice" }) // guards work on decoded values @@ -387,7 +387,7 @@ describe("ReadonlySetFromArray", () => { describe("ReadonlyMapFromArray", () => { test("decodes an array of tuples to a Map", () => { - const schema = S.ReadonlyMap({ key: S.String, value: S.Finite }) + const schema = S.ReadonlyMapFromArray({ key: S.String, value: S.Finite }) const decoded = S.decodeUnknownSync(schema)([["a", 1], ["b", 2]]) expect(decoded).toEqual(new Map([["a", 1], ["b", 2]])) }) @@ -445,11 +445,18 @@ describe("ReadonlySet (with withConstructorDefault)", () => { expect(made.items).toEqual(new Set()) }) - test("decodes array with NumberFromString values", () => { + test("decodes a Set with NumberFromString values", () => { const schema = S.ReadonlySet(S.NumberFromString) - const decoded = S.decodeUnknownSync(schema)(["1", "2"]) + const decoded = S.decodeUnknownSync(schema)(new Set(["1", "2"])) expect(decoded).toEqual(new Set([1, 2])) }) + + test("Encoded is a Set, not an array", () => { + const schema = S.ReadonlySet(S.String) + const encoded = S.encodeSync(schema)(new Set(["a"])) + expect(encoded).toEqual(new Set(["a"])) + expectTypeOf(encoded).toEqualTypeOf>() + }) }) describe("ReadonlyMap (with withConstructorDefault)", () => { @@ -460,11 +467,18 @@ describe("ReadonlyMap (with withConstructorDefault)", () => { expect(made.items).toEqual(new Map()) }) - test("decodes array of tuples with NumberFromString keys", () => { + test("decodes a Map with NumberFromString keys", () => { const schema = S.ReadonlyMap({ key: S.NumberFromString, value: S.String }) - const decoded = S.decodeUnknownSync(schema)([["1", "one"]]) + const decoded = S.decodeUnknownSync(schema)(new Map([["1", "one"]])) expect(decoded).toEqual(new Map([[1, "one"]])) }) + + test("Encoded is a Map, not an array of tuples", () => { + const schema = S.ReadonlyMap({ key: S.String, value: S.Finite }) + const encoded = S.encodeSync(schema)(new Map([["a", 1]])) + expect(encoded).toEqual(new Map([["a", 1]])) + expectTypeOf(encoded).toEqualTypeOf>() + }) }) describe("JSON Schema", () => { @@ -506,8 +520,18 @@ describe("JSON Schema", () => { }) }) - test("Date has identifier DateOrInvalid and ISO 8601 description", () => { + test("Date Encoded is Date; JSON codec encodes ISO strings", () => { + const d = new Date("2024-01-01T00:00:00.000Z") + expect(S.decodeUnknownSync(S.Date)(d)).toBe(d) + expect(S.encodeSync(S.Date)(d)).toBe(d) + expect(S.encodeSync(S.toCodecJson(S.Date))(d)).toBe("2024-01-01T00:00:00.000Z") const doc = S.toJsonSchemaDocument(S.Date) + expect(doc.dialect).toBe("draft-2020-12") + expect(doc.schema).toEqual({ type: "string" }) + }) + + test("DateFromString keeps string Encoded", () => { + const doc = S.toJsonSchemaDocument(S.DateFromString) expect(doc).toStrictEqual({ dialect: "draft-2020-12", schema: { "$ref": "#/$defs/DateOrInvalid" }, @@ -521,21 +545,6 @@ describe("JSON Schema", () => { }) }) - test("DateValid has identifier Date and ISO 8601 description", () => { - const doc = S.toJsonSchemaDocument(S.DateValid) - expect(doc).toStrictEqual({ - dialect: "draft-2020-12", - schema: { "$ref": "#/$defs/Date" }, - definitions: { - Date: { - type: "string", - description: "a valid ISO 8601 date string that will be decoded as a Date", - format: "date-time" - } - } - }) - }) - test("PhoneNumber has format phone", () => { const doc = specialJsonSchemaDocument(S.PhoneNumber) expect(doc).toStrictEqual({ diff --git a/packages/infra/examples/query.ts b/packages/infra/examples/query.ts index 37d0d2652..71ef736f3 100644 --- a/packages/infra/examples/query.ts +++ b/packages/infra/examples/query.ts @@ -76,7 +76,7 @@ const program = Effect.gen(function*() { and("_tag", "Something"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z"), // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")), and("_tag", "Something") ), order("displayName"), @@ -90,7 +90,7 @@ const program = Effect.gen(function*() { and("_tag", "Something"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z"), // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")), and("_tag", "Something") ), order("displayName"), @@ -112,7 +112,7 @@ expectTypeOf(test1).toEqualTypeOf< readonly _tag: "Something" readonly id: string readonly displayName: string - readonly n: string + readonly n: Date readonly union: { readonly _tag: "string" readonly value: string @@ -129,7 +129,7 @@ expectTypeOf(testneq1).toEqualTypeOf< readonly _tag: "Something" readonly id: string readonly displayName: string - readonly n: string + readonly n: Date readonly union: { readonly _tag: "number" readonly value: number diff --git a/packages/infra/src/Store/Cosmos/query.ts b/packages/infra/src/Store/Cosmos/query.ts index ebcc37983..e7589f059 100644 --- a/packages/infra/src/Store/Cosmos/query.ts +++ b/packages/infra/src/Store/Cosmos/query.ts @@ -9,6 +9,7 @@ import type { SupportedValues } from "effect-app/Store" import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.ts" import { isRelationCheck } from "../codeFilter.ts" +import { jsonifyFilter, toJsonQueryValue } from "../utils.ts" export function logQuery(q: { query: string @@ -62,6 +63,8 @@ export function buildWhereCosmosQuery3( skip?: number, limit?: number ) { + filter = jsonifyFilter(filter) + defaultValues = toJsonQueryValue(defaultValues) as Record const statement = (x: FilterR, i: number) => { if (x.path === idKey) { x = { ...x, path: "id" } @@ -89,21 +92,17 @@ export function buildWhereCosmosQuery3( return `(NOT ARRAY_CONTAINS(${k}, ${v}))` case "includes-any": - return `ARRAY_CONTAINS_ANY(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") - })` + return `ARRAY_CONTAINS_ANY(${k}, ${(x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ")})` case "notIncludes-any": return `(NOT ARRAY_CONTAINS_ANY(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") + (x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") }))` case "includes-all": - return `ARRAY_CONTAINS_ALL(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") - })` + return `ARRAY_CONTAINS_ALL(${k}, ${(x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ")})` case "notIncludes-all": return `(NOT ARRAY_CONTAINS_ALL(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") + (x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") }))` case "contains": diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index f6790152f..5bb044bf6 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -6,6 +6,7 @@ import type { AggregateIrExpression, ComputedProjectionIrExpression, ComputedPro import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.ts" import { isRelationCheck } from "../codeFilter.ts" +import { jsonifyFilter, toJsonQueryValue } from "../utils.ts" export interface SQLDialect { readonly jsonExtract: (path: string) => string @@ -177,6 +178,8 @@ export function buildWhereSQLQuery( limit?: number, namespace?: string ) { + filter = jsonifyFilter(filter) + defaultValues = toJsonQueryValue(defaultValues) as Record const params: unknown[] = [] let paramIndex = 1 @@ -214,7 +217,7 @@ export function buildWhereSQLQuery( switch (x.op) { case "in": { - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const hasNull = vals.some((v) => v == null) const nonNullVals = vals.filter((v) => v != null) const parts: string[] = [] @@ -226,7 +229,7 @@ export function buildWhereSQLQuery( return parts.length > 1 ? `(${parts.join(" OR ")})` : parts[0] ?? "1=0" } case "notIn": { - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const hasNull = vals.some((v) => v == null) const nonNullVals = vals.filter((v) => v != null) const parts: string[] = [] @@ -251,26 +254,26 @@ export function buildWhereSQLQuery( case "includes-any": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayContainsAny(arrPath, placeholders) } case "notIncludes-any": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayNotContainsAny(arrPath, placeholders) } case "includes-all": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayContainsAll(arrPath, placeholders) } case "notIncludes-all": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayNotContainsAll(arrPath, placeholders) } diff --git a/packages/infra/src/Store/codeFilter.ts b/packages/infra/src/Store/codeFilter.ts index 0b270f544..8a04fb5fe 100644 --- a/packages/infra/src/Store/codeFilter.ts +++ b/packages/infra/src/Store/codeFilter.ts @@ -6,54 +6,55 @@ import type { FieldValues } from "effect-app/Model/filter/types" import * as Option from "effect-app/Option" import type { Filter } from "effect-app/Store" import { assertUnreachable } from "effect-app/utils" -import { compare, get, greaterThan, greaterThanExclusive, lowerThan, lowerThanExclusive } from "./utils.ts" +import { compare, get, greaterThan, greaterThanExclusive, lowerThan, lowerThanExclusive, toJsonQueryValue } from "./utils.ts" -const vAsArr = (v: string) => v as unknown as any[] +const vAsArr = (v: unknown) => toJsonQueryValue(v) as any[] const filterStatement = (x: any, p: FilterR) => { - const k = get(x, p.path) + const k = toJsonQueryValue(get(x, p.path)) + const v = toJsonQueryValue(p.value) switch (p.op) { case "in": - return p.value.includes(k) + return (v as unknown[]).includes(k) case "notIn": - return !p.value.includes(k) + return !(v as unknown[]).includes(k) case "lt": - return lowerThan(k, p.value) + return lowerThan(k as any, v as any) case "lte": - return lowerThanExclusive(k, p.value) + return lowerThanExclusive(k as any, v as any) case "gt": - return greaterThan(k, p.value) + return greaterThan(k as any, v as any) case "gte": - return greaterThanExclusive(k, p.value) + return greaterThanExclusive(k as any, v as any) case "includes": - return (k as Array).includes(p.value) + return (k as Array).includes(v) case "notIncludes": - return !(k as Array).includes(p.value) + return !(k as Array).includes(v) case "includes-any": - return (vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) + return (vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) case "notIncludes-any": - return !(vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) + return !(vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) case "includes-all": - return (vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) + return (vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) case "notIncludes-all": - return !(vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) + return !(vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) case "contains": - return (k as string).toLowerCase().includes(p.value.toLowerCase()) + return (k as string).toLowerCase().includes((v as string).toLowerCase()) case "endsWith": - return (k as string).toLowerCase().endsWith(p.value.toLowerCase()) + return (k as string).toLowerCase().endsWith((v as string).toLowerCase()) case "startsWith": - return (k as string).toLowerCase().startsWith(p.value.toLowerCase()) + return (k as string).toLowerCase().startsWith((v as string).toLowerCase()) case "notContains": - return !(k as string).toLowerCase().includes(p.value.toLowerCase()) + return !(k as string).toLowerCase().includes((v as string).toLowerCase()) case "notEndsWith": - return !(k as string).toLowerCase().endsWith(p.value.toLowerCase()) + return !(k as string).toLowerCase().endsWith((v as string).toLowerCase()) case "notStartsWith": - return !(k as string).toLowerCase().startsWith(p.value.toLowerCase()) + return !(k as string).toLowerCase().startsWith((v as string).toLowerCase()) case "neq": - return !compare(k, p.value) + return !compare(k, v) case "eq": case undefined: - return compare(k, p.value) + return compare(k, v) default: { return assertUnreachable(p.op) } diff --git a/packages/infra/src/Store/utils.ts b/packages/infra/src/Store/utils.ts index f1505adef..0204ccb39 100644 --- a/packages/infra/src/Store/utils.ts +++ b/packages/infra/src/Store/utils.ts @@ -1,9 +1,56 @@ import crypto from "crypto" import * as Effect from "effect-app/Effect" +import type { FilterResult } from "effect-app/Model/filter/filterApi" import * as Option from "effect-app/Option" +import * as S from "effect-app/Schema" import type { PersistenceModelType, SupportedValues2 } from "effect-app/Store" import { OptimisticConcurrencyException } from "../errors.ts" +const dateJson = S.toCodecJson(S.Date) + +/** + * Lower Date / Map / Set query and document values to JSON, matching + * `Schema.toCodecJson` of those declarations so document-DB adapters can bind + * native Encoded values as JSON parameters. + */ +export function toJsonQueryValue(value: unknown): unknown { + if (value instanceof Date) { + return S.encodeSync(dateJson)(value) + } + if (value instanceof Map) { + return [...value.entries()].map(([k, v]) => [toJsonQueryValue(k), toJsonQueryValue(v)]) + } + if (value instanceof Set) { + return [...value].map(toJsonQueryValue) + } + if (Array.isArray(value)) { + return value.map(toJsonQueryValue) + } + if (value !== null && typeof value === "object") { + const proto = Object.getPrototypeOf(value) + if (proto === Object.prototype || proto === null) { + const out: Record = {} + for (const [k, v] of Object.entries(value)) { + out[k] = toJsonQueryValue(v) + } + return out + } + const toJSON = (value as { toJSON?: () => unknown }).toJSON + if (typeof toJSON === "function") { + return toJsonQueryValue(toJSON.call(value)) + } + } + return value +} + +export function jsonifyFilter(filter: readonly FilterResult[]): FilterResult[] { + return filter.map((r) => + r.t === "and-scope" || r.t === "or-scope" || r.t === "where-scope" + ? { ...r, result: jsonifyFilter(r.result) } + : { ...r, value: toJsonQueryValue(r.value) } + ) +} + /** Traverse an object by a dot-separated path string, e.g. `"a.b.c"`. */ export function get(obj: any, path: string): any { return path.split(".").reduce((res: any, key: string) => (res != null ? res[key] : res), obj) @@ -55,21 +102,21 @@ export function lowercaseIfString(val: T) { } export function compare(valA: unknown, valB: unknown) { - return valA === valB + return toJsonQueryValue(valA) === toJsonQueryValue(valB) } export function lowerThan(valA: SupportedValues2, valB: SupportedValues2) { - return valA < valB + return (toJsonQueryValue(valA) as SupportedValues2) < (toJsonQueryValue(valB) as SupportedValues2) } export function lowerThanExclusive(valA: SupportedValues2, valB: SupportedValues2) { - return valA <= valB + return (toJsonQueryValue(valA) as SupportedValues2) <= (toJsonQueryValue(valB) as SupportedValues2) } export function greaterThan(valA: SupportedValues2, valB: SupportedValues2) { - return valA > valB + return (toJsonQueryValue(valA) as SupportedValues2) > (toJsonQueryValue(valB) as SupportedValues2) } export function greaterThanExclusive(valA: SupportedValues2, valB: SupportedValues2) { - return valA >= valB + return (toJsonQueryValue(valA) as SupportedValues2) >= (toJsonQueryValue(valB) as SupportedValues2) } diff --git a/packages/infra/test/cosmos-query.test.ts b/packages/infra/test/cosmos-query.test.ts index c272d382d..adbc8c08e 100644 --- a/packages/infra/test/cosmos-query.test.ts +++ b/packages/infra/test/cosmos-query.test.ts @@ -13,6 +13,32 @@ type OrderEnc = S.Codec.Encoded // Length projection via `relation(...).length()` should emit a scalar // ARRAY_LENGTH expression rather than pulling (or reshaping) the array. +describe("cosmos query filter: native Encoded values", () => { + it("binds Date as ISO string parameters", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "n", op: "eq", value: new Date("2024-01-01T00:00:00.000Z") }], + "Orders", + {} + ) + expect(result.parameters).toEqual( + expect.arrayContaining([{ name: "@v0", value: "2024-01-01T00:00:00.000Z" }]) + ) + }) + + it("binds Map as array of tuples", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "eq", value: new Map([["k", "v"]]) }], + "Orders", + {} + ) + expect(result.parameters).toEqual( + expect.arrayContaining([{ name: "@v0", value: [["k", "v"]] }]) + ) + }) +}) + describe("cosmos query projection: array length", () => { it("projects packages length via ARRAY_LENGTH", () => { const q = make().pipe( @@ -29,13 +55,13 @@ describe("cosmos query projection: array length", () => { ir.filter ?? [], "Orders", {}, - ir.select as any + ir.select ) expect(result.query).toMatch(/ARRAY_LENGTH\(f(?:\.packages|\["packages"\])\)/) expect(result.query).toContain("AS packageCount") // Must not pull the full array nor reshape via subquery - expect(result.query).not.toMatch(/ARRAY\s*\(\s*SELECT[^)]*FROM\s+t\s+in\s+f[\.\["]/i) + expect(result.query).not.toMatch(/ARRAY\s*\(\s*SELECT[^)]*FROM\s+t\s+in\s+f[.["]/i) expect(result.query).not.toMatch(/SELECT VALUE COUNT/) }) }) @@ -83,7 +109,7 @@ describe("cosmos query projection: union array fields", () => { const packageSelects = select.filter((item) => typeof item === "object" && item !== null && "key" in item && item.key === "packages" ) - const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select as any) + const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select) expect(packageSelects).toHaveLength(1) expect(result.query.match(/\bAS\s+packages\b/g) ?? []).toHaveLength(1) @@ -102,7 +128,7 @@ describe("cosmos query projection: union array fields", () => { const packageSelects = select.filter((item) => typeof item === "object" && item !== null && "key" in item && item.key === "packages" ) - const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select as any) + const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select) expect(packageSelects).toHaveLength(1) expect(result.query.match(/\bAS\s+packages\b/g) ?? []).toHaveLength(1) @@ -146,7 +172,7 @@ describe("cosmos query projection: relation-every parameter binding", () => { ) const ir = toFilter(q as any, DN as any) - const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "DN", {}, ir.select as any) + const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "DN", {}, ir.select) // Each filter element binds exactly one parameter: 2 every filters + 2 main filter = 4. expect(result.parameters).toHaveLength(4) diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index 680c7e5f1..2e166ee16 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -41,7 +41,7 @@ const q = make() where("displayName", "Verona"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z") // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")) ), order("displayName"), page({ take: 10 }), @@ -141,7 +141,7 @@ it("works with repo", () => where("displayName", "Verona"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z") // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")) ), order("displayName"), page({ take: 10 }), @@ -168,6 +168,11 @@ it("works with repo", () => expect(q1).toEqual(items.slice(0, 2).toReversed().map(Struct.pick(["id", "displayName"]))) expect(q2).toEqual(items.slice(0, 2).toReversed().map(Struct.pick(["displayName"]))) + + const byDate = yield* somethingRepo.query( + where("n", new Date("2020-01-01T00:00:00.000Z")) + ) + expect(byDate.map((_) => _.displayName)).toEqual(["Verona", "Riley"]) }) .pipe( Effect.provide(Layer.mergeAll(SomethingRepo.Test, SomeService.Default)), @@ -196,8 +201,8 @@ it("collect", () => })), S.toType(S.Option(S.String)), (_) => - _.displayName === "Riley" && _.n === "2020-01-01T00:00:00.000Z" - ? Option.some(`${_.displayName}-${_.n}`) + _.displayName === "Riley" && _.n.toISOString() === "2020-01-01T00:00:00.000Z" + ? Option.some(`${_.displayName}-${_.n.toISOString()}`) : Option.none() ), "collect" @@ -215,7 +220,7 @@ it("collect", () => QueryEnd<{ readonly id: string readonly displayName: string - readonly n: string + readonly n: Date readonly union: { readonly _tag: "string" readonly value: string @@ -530,7 +535,7 @@ it( const schema = S.Struct({ id: S.String, createdAt: S.Date.pipe( - S.withDecodingDefault(Effect.sync(() => new Date().toISOString())), + S.withDecodingDefault(Effect.sync(() => new Date())), S.withConstructorDefault(Effect.sync(() => new Date())) ) }) @@ -543,7 +548,7 @@ it( const outputSchema = S.Struct({ id: S.Literal("123"), createdAt: S.Date.pipe( - S.withDecodingDefault(Effect.sync(() => new Date().toISOString())), + S.withDecodingDefault(Effect.sync(() => new Date())), S.withConstructorDefault(Effect.sync(() => new Date())) ) }) @@ -824,7 +829,7 @@ it("ProjectableFromDomain distributes over tagged union Encoded", () => { type GoodCheck = ProjectableFromDomain type BadCheck = ProjectableFromDomain - const _good: GoodCheck = undefined as unknown + const _good: GoodCheck = undefined // @ts-expect-error cancelled branch requires activeRequest not present on domain cancelled const _bad: BadCheck = undefined as unknown void _good @@ -852,7 +857,7 @@ it("ProjectableFromDomain allows dual same-tag domain variants", () => { type GoodCheck = ProjectableFromDomain type BadFlatCheck = ProjectableFromDomain - const _good: GoodCheck = undefined as unknown + const _good: GoodCheck = undefined // @ts-expect-error packages is not on domain initial; multi-tag flat intersection rejects it const _badFlat: BadFlatCheck = undefined as unknown void _good @@ -2129,7 +2134,7 @@ it("memFilter: agg-count-when groups rows and counts conditionally", () => { }, { key: "total", aggregate: { _tag: "agg-count" } } ] as any - })(rows as any) as any[] + })(rows) as any[] expect(result.length).toBe(2) const nyc = result.find((r: any) => r.city === "NYC")! @@ -2155,7 +2160,7 @@ it("memFilter: agg-sum / agg-min / agg-max aggregate numerics", () => { { key: "min", aggregate: { _tag: "agg-min", field: "salary" } }, { key: "max", aggregate: { _tag: "agg-max", field: "salary" } } ] as any - })(rows as any) as any[] + })(rows) as any[] expect(result.length).toBe(2) const eng = result.find((r: any) => r.dept === "eng")! @@ -2179,7 +2184,7 @@ it("memFilter: aggregate with nested path grouping", () => { { key: "city", path: "address.city" }, { key: "count", aggregate: { _tag: "agg-count" } } ] as any - })(rows as any) as any[] + })(rows) as any[] expect(result.length).toBe(2) expect(result.find((r: any) => r.city === "NYC")!.count).toBe(2) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 433f7373e..48f11b9b3 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -24,11 +24,33 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("John") }) + it("where eq Date binds ISO string", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "n", op: "eq", value: new Date("2024-01-01T00:00:00.000Z") }], + "users", + {} + ) + expect(result.params).toContain("2024-01-01T00:00:00.000Z") + }) + + it("where includes Set binds array values", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "tags", op: "in", value: new Set(["a", "b"]) }], + "users", + {} + ) + expect(result.params).toEqual(expect.arrayContaining(["a", "b"])) + }) + it("where eq number", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "eq", value: 25 as any }], + [{ t: "where", path: "age", op: "eq", value: 25 }], "users", {} ) @@ -40,7 +62,7 @@ describe("SQL query builder (SQLite dialect)", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 18 as any }], + [{ t: "where", path: "age", op: "gt", value: 18 }], "users", {} ) @@ -70,7 +92,7 @@ describe("SQL query builder (SQLite dialect)", () => { "id", [ { t: "where", path: "name", op: "eq", value: "Alice" }, - { t: "and", path: "age", op: "gt", value: 18 as any } + { t: "and", path: "age", op: "gt", value: 18 } ], "users", {} @@ -83,7 +105,7 @@ describe("SQL query builder (SQLite dialect)", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "id", op: "in", value: ["a", "b", "c"] as any }], + [{ t: "where", path: "id", op: "in", value: ["a", "b", "c"] }], "users", {} ) @@ -167,7 +189,7 @@ describe("SQL query builder (SQLite dialect)", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "tags", op: "includes-any", value: ["admin", "user"] as any }], + [{ t: "where", path: "tags", op: "includes-any", value: ["admin", "user"] }], "users", {} ) @@ -495,7 +517,7 @@ describe("SQL query builder (PostgreSQL dialect)", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "status", op: "in", value: ["active", "pending"] as any }], + [{ t: "where", path: "status", op: "in", value: ["active", "pending"] }], "users", {} ) @@ -787,7 +809,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r1 = query(db, q1.sql, q1.params) expect(r1.length).toBe(1) - expect((r1[0] as any).id).toBe("1") + expect(r1[0].id).toBe("1") const q2 = buildWhereSQLQuery( sqliteDialect, @@ -798,13 +820,13 @@ describe("SQL Store (SQLite integration)", () => { ) const r2 = query(db, q2.sql, q2.params) expect(r2.length).toBe(1) - expect((r2[0] as any).id).toBe("2") + expect(r2[0].id).toBe("2") // Both queryable by id column const q3 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "id", op: "in", value: ["1", "2"] as any }], + [{ t: "where", path: "id", op: "in", value: ["1", "2"] }], "test_compat", {} ) @@ -830,7 +852,7 @@ describe("SQL Store (SQLite integration)", () => { const q1 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 28 as any }], + [{ t: "where", path: "age", op: "gt", value: 28 }], "test_noid", {} ) @@ -846,8 +868,8 @@ describe("SQL Store (SQLite integration)", () => { ) const r2 = query(db, q2.sql, q2.params) expect(r2.length).toBe(1) - expect((r2[0] as any).id).toBe("2") - expect((JSON.parse((r2[0] as any).data) as any).name).toBe("Bob") + expect(r2[0].id).toBe("2") + expect((JSON.parse(r2[0].data) as any).name).toBe("Bob") // Order + limit still works const q3 = buildWhereSQLQuery( @@ -863,7 +885,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r3 = query(db, q3.sql, q3.params) expect(r3.length).toBe(2) - expect((JSON.parse((r3[0] as any).data) as any).name).toBe("Bob") // youngest first + expect((JSON.parse(r3[0].data) as any).name).toBe("Bob") // youngest first })) it("query builder generates valid SQL for SQLite", () => @@ -896,13 +918,13 @@ describe("SQL Store (SQLite integration)", () => { {} ) expect(query(db, q1.sql, q1.params).length).toBe(1) - expect((JSON.parse((query(db, q1.sql, q1.params)[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(query(db, q1.sql, q1.params)[0].data) as any).name).toBe("Alice") // Test gt const q2 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 28 as any }], + [{ t: "where", path: "age", op: "gt", value: 28 }], "test_people", {} ) @@ -927,20 +949,20 @@ describe("SQL Store (SQLite integration)", () => { "id", [ { t: "where", path: "name", op: "eq", value: "Alice" }, - { t: "and", path: "age", op: "gt", value: 25 as any } + { t: "and", path: "age", op: "gt", value: 25 } ], "test_people", {} ) const r4 = query(db, q4.sql, q4.params) expect(r4.length).toBe(1) - expect((JSON.parse((r4[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(r4[0].data) as any).name).toBe("Alice") // Test IN const q5 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "id", op: "in", value: ["1", "3"] as any }], + [{ t: "where", path: "id", op: "in", value: ["1", "3"] }], "test_people", {} ) @@ -966,7 +988,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r7 = query(db, q7.sql, q7.params) expect(r7.length).toBe(1) - expect((JSON.parse((r7[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(r7[0].data) as any).name).toBe("Alice") // Test includes (array) const q8 = buildWhereSQLQuery( @@ -987,7 +1009,7 @@ describe("SQL Store (SQLite integration)", () => { { t: "or-scope", result: [ - { t: "where", path: "age", op: "gt", value: 30 as any }, + { t: "where", path: "age", op: "gt", value: 30 }, { t: "and", path: "name", op: "contains", value: "ar" } ], relation: "some" @@ -1012,7 +1034,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r10 = query(db, q10.sql, q10.params) expect(r10.length).toBe(2) - expect((JSON.parse((r10[0] as any).data) as any).name).toBe("Charlie") // oldest first + expect((JSON.parse(r10[0].data) as any).name).toBe("Charlie") // oldest first })) it("computed relation-every / distinct-count / sum / collect run on SQLite", () => @@ -1154,7 +1176,7 @@ describe("SQL Store (SQLite integration)", () => { const results = query(db, nsSql, params) // Should only get Alice and Bob (primary namespace), not Charlie (other namespace) expect(results.length).toBe(2) - const names = results.map((r) => (JSON.parse((r as any).data) as any).name).sort() + const names = results.map((r) => (JSON.parse(r.data) as any).name).sort() expect(names).toEqual(["Alice", "Bob"]) })) @@ -1285,7 +1307,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: true as any }], + [{ t: "where", path: "flag", op: "eq", value: true }], "t", {} ) @@ -1297,7 +1319,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: false as any }], + [{ t: "where", path: "flag", op: "eq", value: false }], "t", {} ) @@ -1308,7 +1330,7 @@ describe("boolean WHERE clauses — query builder", () => { const r1 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "neq", value: true as any }], + [{ t: "where", path: "flag", op: "neq", value: true }], "t", {} ) @@ -1317,7 +1339,7 @@ describe("boolean WHERE clauses — query builder", () => { const r2 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "in", value: [true, false] as any }], + [{ t: "where", path: "flag", op: "in", value: [true, false] }], "t", {} ) @@ -1328,7 +1350,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: true as any }], + [{ t: "where", path: "flag", op: "eq", value: true }], "t", {} ) @@ -1340,7 +1362,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: false as any }], + [{ t: "where", path: "flag", op: "eq", value: false }], "t", {} ) @@ -1351,7 +1373,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "flag", op: "in", value: [true, false] as any }], + [{ t: "where", path: "flag", op: "in", value: [true, false] }], "t", {} ) @@ -1373,7 +1395,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 18 as any }], + [{ t: "where", path: "age", op: "gt", value: 18 }], "t", {} ) @@ -1404,13 +1426,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: true as any }], + [{ t: "where", path: "flag", op: "eq", value: true }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(rows[0].data) as any).name).toBe("Alice") })) it("where flag = false matches only false rows", () => @@ -1426,13 +1448,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: false as any }], + [{ t: "where", path: "flag", op: "eq", value: false }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Bob") + expect((JSON.parse(rows[0].data) as any).name).toBe("Bob") })) it("where nested boolean path works", () => @@ -1448,13 +1470,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "meta.active", op: "eq", value: true as any }], + [{ t: "where", path: "meta.active", op: "eq", value: true }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(rows[0].data) as any).name).toBe("Alice") })) it("where neq boolean works", () => @@ -1470,13 +1492,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "neq", value: true as any }], + [{ t: "where", path: "flag", op: "neq", value: true }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Bob") + expect((JSON.parse(rows[0].data) as any).name).toBe("Bob") })) }) @@ -1573,7 +1595,7 @@ describe("toRow strips _etag and id from data", () => { const toRow = (e: any, idKey: IdKey) => { const newE = makeETag(e) const id = newE[idKey] as string - const { _etag, [idKey]: _id, ...rest } = newE as any + const { _etag, [idKey]: _id, ...rest } = newE const data = JSON.stringify(rest) return { id, _etag: newE._etag!, data, item: newE } } diff --git a/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts b/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts index 6960e2df7..9d794a84d 100644 --- a/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts +++ b/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts @@ -41,7 +41,7 @@ describe("Date field validation", () => { setup() { const form = useOmegaForm( S.Struct({ - date: S.Date + date: S.DateFromString }), { onSubmit: async ({ value }) => { diff --git a/packages/vue-components/src/components/OmegaForm/meta/checks.ts b/packages/vue-components/src/components/OmegaForm/meta/checks.ts index eef6a693f..78aca09f4 100644 --- a/packages/vue-components/src/components/OmegaForm/meta/checks.ts +++ b/packages/vue-components/src/components/OmegaForm/meta/checks.ts @@ -93,7 +93,10 @@ export const getFieldMetadataFromAst = (property: S.AST.AST) => { base.type = "boolean" } else if ( S.AST.isDeclaration(property) - && (property.annotations as any)?.typeConstructor?._tag === "Date" + && ( + (property.annotations as any)?.typeConstructor?._tag === "Date" + || (property.annotations as any)?.representation?.id === "effect/schema/Date" + ) ) { base.type = "date" } else { diff --git a/packages/vue-components/stories/OmegaForm/AutoGeneration.vue b/packages/vue-components/stories/OmegaForm/AutoGeneration.vue index 7071b30fb..ea511b1e2 100644 --- a/packages/vue-components/stories/OmegaForm/AutoGeneration.vue +++ b/packages/vue-components/stories/OmegaForm/AutoGeneration.vue @@ -63,7 +63,7 @@ const schema = S.Struct({ boolean: S.Boolean, email: S.Email, url: S.Url, - date: S.Date + date: S.DateFromString }) type Meta = OmegaAutoGenMeta< typeof schema.Encoded, diff --git a/packages/vue-components/stories/OmegaForm/Date.vue b/packages/vue-components/stories/OmegaForm/Date.vue index 3466bf98e..53e7420a5 100644 --- a/packages/vue-components/stories/OmegaForm/Date.vue +++ b/packages/vue-components/stories/OmegaForm/Date.vue @@ -26,7 +26,7 @@ import * as S from "effect-app/Schema" import { useOmegaForm } from "../../src/components/OmegaForm" const schema = S.Struct({ - date: S.NullOr(S.Date) + date: S.NullOr(S.DateFromString) }) const form = useOmegaForm(schema, { From 5546b8c82208eb5f8f8efbdd8d17c93658e3c5f1 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:21:19 +0000 Subject: [PATCH 02/10] fix: drop unused schema test type aliases Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- packages/effect-app/test/schema.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/effect-app/test/schema.test.ts b/packages/effect-app/test/schema.test.ts index 871743b0f..4d5083f77 100644 --- a/packages/effect-app/test/schema.test.ts +++ b/packages/effect-app/test/schema.test.ts @@ -265,7 +265,6 @@ test("TaggedUnion match dispatches on _tag", () => { S.TaggedStruct("A", { a: S.String }), S.TaggedStruct("B", { b: S.Finite }) ]) - type T = S.Schema.Type const matcher = schema.match({ A: (v) => `got A: ${v.a}`, @@ -319,7 +318,6 @@ test("TaggedUnion with encodeKeys renaming a non-tag key", () => { expect(decoded2).toEqual({ _tag: "B", lastName: 42 }) // encode back to snake_case - type T = S.Schema.Type const encoded = S.encodeSync(schema)({ _tag: "A", firstName: "Alice" }) expect(encoded).toEqual({ _tag: "A", first_name: "Alice" }) From 621c4f5cff6694ecd76d52f8d6c4e3ab0441db75 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 09:25:23 +0000 Subject: [PATCH 03/10] feat(query): native Date/Set values on includes/in ops Unwrap ReadonlySet in includes/includes-any and accept Set or array needles for in. Adapters already JSON-convert those values. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- .changeset/native-encoded-query-json.md | 2 +- packages/effect-app/src/Model/query/dsl.ts | 17 ++++---- packages/infra/test/cosmos-query.test.ts | 33 ++++++++++++++++ packages/infra/test/query.test.ts | 46 ++++++++++++++++++++++ packages/infra/test/sql-store.test.ts | 31 ++++++++++++++- 5 files changed, 120 insertions(+), 9 deletions(-) diff --git a/.changeset/native-encoded-query-json.md b/.changeset/native-encoded-query-json.md index 50002599f..fc7d54cb1 100644 --- a/.changeset/native-encoded-query-json.md +++ b/.changeset/native-encoded-query-json.md @@ -6,4 +6,4 @@ Stop forcing Date/Map/Set Encoded shapes to JSON. -`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values; document-store adapters convert them with `Schema.toCodecJson` (Date → ISO string, Set → array, Map → entries). Repository persistence also round-trips through `toCodecJson` so JSON stores stay compatible. +`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values, including array ops (`includes` / `in` / `includes-any`) on `Date[]` and `ReadonlySet` fields. Document-store adapters convert them with `Schema.toCodecJson` (Date → ISO string, Set → array, Map → entries). Repository persistence also round-trips through `toCodecJson` so JSON stores stay compatible. diff --git a/packages/effect-app/src/Model/query/dsl.ts b/packages/effect-app/src/Model/query/dsl.ts index d9efe2d04..3d114eb73 100644 --- a/packages/effect-app/src/Model/query/dsl.ts +++ b/packages/effect-app/src/Model/query/dsl.ts @@ -1157,7 +1157,11 @@ export const aggregate: { return new Project({ current, schema, mode: "aggregate", aggregateMap } as any) } -type GetArV = T extends readonly (infer R)[] ? R : never +type GetArV = T extends ReadonlySet ? R + : T extends readonly (infer R)[] ? R + : never + +type InValues = readonly T[] | ReadonlySet export type FilterContinuations = { < @@ -1207,13 +1211,12 @@ export type FilterContinuations = { < TFieldValues extends FieldValues, TFieldName extends FieldPath, - const V extends readonly FieldPathValue[], TFieldValuesRefined extends TFieldValues = TFieldValues, E extends boolean = false >( path: TFieldName, op: "in" | "notIn", - value: V + value: InValues> ): ( current: IsCurrentInitial extends true ? Query : QueryWhere @@ -1249,7 +1252,7 @@ export type FilterContinuations = { | "notIncludes-any" | "includes-all" | "notIncludes-all", - value: readonly GetArV[] + value: InValues> ): ( current: IsCurrentInitial extends true ? Query : QueryWhere @@ -1318,12 +1321,12 @@ export type FilterContinuationsWithSubpath = { TFieldName extends FieldPath, TFieldValuesSub extends TFieldValues[TFieldName][number], TFieldNameSub extends FieldPath, - const V extends readonly FieldPathValue[] + V extends FieldPathValue >( subPath: TFieldName, restPath: TFieldNameSub, op: "in" | "notIn", - value: V + value: InValues ): ( current: Query ) => QueryWhere @@ -1357,7 +1360,7 @@ export type FilterContinuationsWithSubpath = { | "notIncludes-any" | "includes-all" | "notIncludes-all", - value: readonly GetArV[] + value: InValues> ): ( current: Query ) => QueryWhere diff --git a/packages/infra/test/cosmos-query.test.ts b/packages/infra/test/cosmos-query.test.ts index adbc8c08e..7c4535322 100644 --- a/packages/infra/test/cosmos-query.test.ts +++ b/packages/infra/test/cosmos-query.test.ts @@ -37,6 +37,39 @@ describe("cosmos query filter: native Encoded values", () => { expect.arrayContaining([{ name: "@v0", value: [["k", "v"]] }]) ) }) + + it("binds includes Date as ISO string", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "dates", op: "includes", value: new Date("2024-01-01T00:00:00.000Z") }], + "Orders", + {} + ) + expect(result.query).toContain("ARRAY_CONTAINS") + expect(result.parameters).toEqual( + expect.arrayContaining([{ name: "@v0", value: "2024-01-01T00:00:00.000Z" }]) + ) + }) + + it("binds includes-any Date Set as ISO parameters", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ + t: "where", + path: "dates", + op: "includes-any", + value: new Set([new Date("2024-01-01T00:00:00.000Z")]) + }], + "Orders", + {} + ) + expect(result.parameters).toEqual( + expect.arrayContaining([ + { name: "@v0", value: ["2024-01-01T00:00:00.000Z"] }, + { name: "@v0__0", value: "2024-01-01T00:00:00.000Z" } + ]) + ) + }) }) describe("cosmos query projection: array length", () => { diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index 2e166ee16..b37fec667 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -1350,6 +1350,27 @@ it("does not allow string queries on arrays", () => expectTypeOf(good2).toEqualTypeOf>() expectTypeOf(good3).toEqualTypeOf>() expectTypeOf(good4).toEqualTypeOf>() + + type Native = { + readonly id: string + readonly dates: Date[] + readonly dateSet: ReadonlySet + readonly tags: ReadonlySet + } + const native = make() + const d = new Date("2020-01-01T00:00:00.000Z") + const n1 = native.pipe(where("dates", "includes", d)) + const n2 = native.pipe(where("dateSet", "includes", d)) + const n3 = native.pipe(where("tags", "includes", "a")) + const n4 = native.pipe(where("dates", "includes-any", [d])) + const n5 = native.pipe(where("dateSet", "includes-any", new Set([d]))) + const n6 = native.pipe(where("id", "in", new Set(["x"]))) + expectTypeOf(n1).toEqualTypeOf>() + expectTypeOf(n2).toEqualTypeOf>() + expectTypeOf(n3).toEqualTypeOf>() + expectTypeOf(n4).toEqualTypeOf>() + expectTypeOf(n5).toEqualTypeOf>() + expectTypeOf(n6).toEqualTypeOf>() }) .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) @@ -2069,6 +2090,31 @@ it("codeFilter: array includes / includes-any / includes-all", () => { expect(runCF(make().pipe(where("tags", "includes-all", ["red", "blue"])))).toEqual(["3"]) }) +it("codeFilter: Date array / Set includes and in", () => { + const d0 = new Date("2020-01-01T00:00:00.000Z") + const d1 = new Date("2021-01-01T00:00:00.000Z") + type DateRow = { + readonly id: string + readonly dates: Date[] + readonly dateSet: ReadonlySet + readonly tag: string + } + const rows: DateRow[] = [ + { id: "1", dates: [d0], dateSet: new Set([d0]), tag: "a" }, + { id: "2", dates: [d1, d0], dateSet: new Set([d1]), tag: "b" } + ] + const run = (q: any) => (memFilter(toFilter(q))(rows) as DateRow[]).map((_) => _.id) + expect(run(make().pipe(where("dates", "includes", d0))).sort()).toEqual(["1", "2"]) + expect(run(make().pipe(where("dates", "includes", d1)))).toEqual(["2"]) + expect(run(make().pipe(where("dateSet", "includes", d0)))).toEqual(["1"]) + expect(run(make().pipe(where("dates", "includes-any", [d1])))).toEqual(["2"]) + expect(run(make().pipe(where("dateSet", "includes-any", new Set([d0, d1])))).sort()).toEqual([ + "1", + "2" + ]) + expect(run(make().pipe(where("tag", "in", new Set(["a"]))))).toEqual(["1"]) +}) + it("codeFilter: in / notIn", () => { expect(runCF(make().pipe(where("tag", "in", ["x", "z"]))).sort()).toEqual(["1", "3"]) expect(runCF(make().pipe(where("tag", "notIn", ["x", "z"]))).sort()).toEqual(["2", "4"]) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 48f11b9b3..59adc1c25 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -35,7 +35,7 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("2024-01-01T00:00:00.000Z") }) - it("where includes Set binds array values", () => { + it("where in Set binds array values", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", @@ -46,6 +46,35 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toEqual(expect.arrayContaining(["a", "b"])) }) + it("where includes Date binds ISO string", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "dates", op: "includes", value: new Date("2024-01-01T00:00:00.000Z") }], + "users", + {} + ) + expect(result.params).toContain("2024-01-01T00:00:00.000Z") + }) + + it("where includes-any Date[] binds ISO strings", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ + t: "where", + path: "dates", + op: "includes-any", + value: [new Date("2024-01-01T00:00:00.000Z"), new Date("2024-06-01T00:00:00.000Z")] + }], + "users", + {} + ) + expect(result.params).toEqual( + expect.arrayContaining(["2024-01-01T00:00:00.000Z", "2024-06-01T00:00:00.000Z"]) + ) + }) + it("where eq number", () => { const result = buildWhereSQLQuery( sqliteDialect, From 216816ca960283ee7a8dbe007f3471323617c546 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:18:44 +0000 Subject: [PATCH 04/10] feat(store): JSON-codec Encoded Date/Map/Set in memory and disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Memory and Disk now convert documents with Schema.toCodecJson(toEncoded) instead of JSON.parse/stringify (which dropped Map/Set). SQL and Cosmos use the same codec on write/read. Repository encode stays Type→Encoded; adapters own JSON. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- .changeset/native-encoded-query-json.md | 2 +- .../src/Model/Repository/internal/internal.ts | 35 +++----- packages/effect-app/src/Store.ts | 7 +- packages/infra/src/Store/Cosmos.ts | 21 +++-- packages/infra/src/Store/Disk.ts | 29 +++++-- packages/infra/src/Store/Memory.ts | 52 ++++++++---- packages/infra/src/Store/SQL.ts | 38 ++++++--- packages/infra/src/Store/SQL/Pg.ts | 23 +++-- packages/infra/src/Store/jsonDocument.ts | 43 ++++++++++ packages/infra/test/query.test.ts | 85 +++++++++++++++++++ 10 files changed, 260 insertions(+), 75 deletions(-) create mode 100644 packages/infra/src/Store/jsonDocument.ts diff --git a/.changeset/native-encoded-query-json.md b/.changeset/native-encoded-query-json.md index fc7d54cb1..2686c0041 100644 --- a/.changeset/native-encoded-query-json.md +++ b/.changeset/native-encoded-query-json.md @@ -6,4 +6,4 @@ Stop forcing Date/Map/Set Encoded shapes to JSON. -`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values, including array ops (`includes` / `in` / `includes-any`) on `Date[]` and `ReadonlySet` fields. Document-store adapters convert them with `Schema.toCodecJson` (Date → ISO string, Set → array, Map → entries). Repository persistence also round-trips through `toCodecJson` so JSON stores stay compatible. +`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values, including array ops (`includes` / `in` / `includes-any`) on `Date[]` and `ReadonlySet` fields. Memory, Disk, SQL, and Cosmos convert Encoded Date/Map/Set through `Schema.toCodecJson` on write/read; query parameters are lowered the same way. diff --git a/packages/effect-app/src/Model/Repository/internal/internal.ts b/packages/effect-app/src/Model/Repository/internal/internal.ts index 0ffc556e1..0bb5cba8b 100644 --- a/packages/effect-app/src/Model/Repository/internal/internal.ts +++ b/packages/effect-app/src/Model/Repository/internal/internal.ts @@ -34,9 +34,6 @@ import { ValidationError, ValidationResult } from "../validation.ts" const dedupe = Array.dedupeWith(Equivalence.String) -/** JSON persistence codec: Encoded Date/Map/Set become JSON while the store stays typed as Encoded. */ -const persistJson = (schema: S.Codec) => S.toCodecJson(schema) as unknown as S.Codec - // ms buckets: dense under 100ms (common path), then mid-tail and multi-second stalls. // Rare 0.5–1s+ encodes (fat aggregates) must not collapse into a single overflow bin. const schemaDurationBoundaries = [ @@ -182,17 +179,14 @@ export function makeRepoInternal< .gen(function*() { const rctx: Context.Context = args.schemaContext ?? Context.empty() as any const provideRctx = Effect.provide(rctx) - // Persist via JSON codec so Date/Map/Set Encoded values round-trip through - // document stores. Query filters keep native Encoded; adapters lower to JSON. - const persistCodec = persistJson(schema) const encodeMany = (items: readonly T[]) => - S.encodeEffect(S.Array(persistCodec))(items).pipe( + S.encodeEffect(S.Array(schema))(items).pipe( provideRctx, timeSchema("encode", name, undefined, items.length, entityStateFromItems(items)) ) - const decode = flow(S.decodeEffectConcurrently(persistCodec), provideRctx) + const decode = flow(S.decodeEffectConcurrently(schema), provideRctx) const decodeMany = flow( - S.decodeEffectConcurrently(S.Array(persistCodec)), + S.decodeEffectConcurrently(S.Array(schema)), provideRctx ) @@ -528,7 +522,7 @@ export function makeRepoInternal< const getDecodeMany = (s: S.Codec) => { let dec = decodeManyCache.get(s) if (!dec) { - dec = S.decodeEffectConcurrently(S.Array(persistJson(s))) + dec = S.decodeEffectConcurrently(S.Array(s)) decodeManyCache.set(s, dec) } return dec @@ -587,7 +581,7 @@ export function makeRepoInternal< .pipe( Effect.andThen( (items) => - S.decodeEffectConcurrently(S.Array(persistJson(a.schema ?? schema)))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(a.schema ?? schema)))(items).pipe( provideRctx, timeSchema("decode", name, "aggregate", items.length) ) @@ -599,7 +593,7 @@ export function makeRepoInternal< .pipe( Effect.andThen( (items) => - S.decodeEffectConcurrently(S.Array(persistJson(a.schema ?? schema)))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(a.schema ?? schema)))(items).pipe( provideRctx, timeSchema("decode", name, "project", items.length) ) @@ -610,7 +604,7 @@ export function makeRepoInternal< // TODO: mapFrom but need to support per field and dependencies .pipe( Effect.flatMap((items) => - S.decodeEffectConcurrently(S.Array(persistJson(a.schema)))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(a.schema)))(items).pipe( Effect.map(Array.getSomes), provideRctx, timeSchema("decode", name, "collect", items.length) @@ -703,7 +697,7 @@ export function makeRepoInternal< const rawData = rawResult.value as Encoded const jitMResult = mapFrom(rawData) // apply jitM - const decodeResult = yield* S.decodeEffectConcurrently(persistCodec)(jitMResult).pipe( + const decodeResult = yield* S.decodeEffectConcurrently(schema)(jitMResult).pipe( Effect.result, provideRctx ) @@ -743,7 +737,7 @@ export function makeRepoInternal< queryRaw(schema: S.Codec, q: Q.RawQuery) { return store.queryRaw(q).pipe( Effect.flatMap((items) => - S.decodeEffectConcurrently(S.Array(persistJson(schema)))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(schema)))(items as readonly S.Json[]).pipe( timeSchema("decode", name, undefined, items.length) ) ), @@ -762,10 +756,9 @@ export function makeRepoInternal< * @internal */ mapped: (schema: S.Codec) => { - const persistMapped = persistJson(schema) - const dec = S.decodeEffectConcurrently(persistMapped) - const encMany = S.encodeEffect(S.Array(persistMapped)) - const decMany = S.decodeEffectConcurrently(S.Array(persistMapped)) + const dec = S.decodeEffectConcurrently(schema) + const encMany = S.encodeEffect(S.Array(schema)) + const decMany = S.decodeEffectConcurrently(S.Array(schema)) const spanAttrs = { kind: "client" as const, attributes: { "app.entity": name } } return { all: allE.pipe( @@ -863,9 +856,8 @@ export function makeStore() { ) { function encodeToEncoded() { const getEtag = () => undefined - const persistCodec = persistJson(schema) return (t: T) => - S.encodeEffect(persistCodec)(t).pipe( + S.encodeEffect(schema)(t).pipe( Effect.orDie, Effect.map((_) => mapToPersistenceModel(_, getEtag)) ) @@ -895,6 +887,7 @@ export function makeStore() { : undefined, { ...config, + schema, partitionValue: config?.partitionValue ?? ((_) => "primary") /*(isIntegrationEvent(r) ? r.companyId : r.id*/ } diff --git a/packages/effect-app/src/Store.ts b/packages/effect-app/src/Store.ts index eea19b6c4..e554b685b 100644 --- a/packages/effect-app/src/Store.ts +++ b/packages/effect-app/src/Store.ts @@ -11,7 +11,7 @@ import type { FieldPath } from "./Model/filter/types/path/index.ts" import type { AggregateIrExpression, ComputedProjectionIrExpression, RawQuery } from "./Model/query.ts" import type * as Option from "./Option.ts" import * as RequestScopedDependencies from "./RequestScopedDependencies.ts" -import { NonEmptyString255 } from "./Schema.ts" +import { NonEmptyString255, type Top as SchemaTop } from "./Schema.ts" /** * Adapter-neutral unique-key definition for stores that support unique indexes, @@ -46,6 +46,11 @@ export interface StoreConfig { * Unique indexes, mainly for CosmosDB */ uniqueKeys?: UniqueKey[] + /** + * Domain schema whose Encoded shape is stored. JSON adapters use + * `Schema.toCodecJson(Schema.toEncoded(schema))` so Date/Map/Set round-trip. + */ + schema?: SchemaTop } export type SupportedValues = string | boolean | number | null diff --git a/packages/infra/src/Store/Cosmos.ts b/packages/infra/src/Store/Cosmos.ts index fbde39270..d1a336c42 100644 --- a/packages/infra/src/Store/Cosmos.ts +++ b/packages/infra/src/Store/Cosmos.ts @@ -19,6 +19,7 @@ import { DatabaseError, OptimisticConcurrencyException } from "../errors.ts" import { InfraLogger } from "../logger.ts" import { annotateCosmosResponse, annotateDb } from "../otel.ts" import { buildWhereCosmosQuery3, logQuery } from "./Cosmos/query.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" const makeMapId = (idKey: IdKey) => ({ [idKey]: id, ...e }: Encoded) => ({ @@ -96,6 +97,8 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) { const mapId = makeMapId(idKey) const mapReverseId = makeReverseMapId(idKey) + const codec = makeJsonDocumentCodec(config?.schema) + const fromStored = (raw: Encoded) => codec.decode({ ...config?.defaultValues, ...mapReverseId(raw as any) }) type PM = PersistenceModelType type PMCosmos = PersistenceModelType & { id: string }> const containerId = `${prefix}${name}` @@ -205,7 +208,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { dropUndefinedT({ operationType: "Create" as const, resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) } @@ -217,7 +220,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { operationType: "Replace" as const, id: x[idKey], resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) }, @@ -314,7 +317,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { onNone: () => ({ operationType: "Create" as const, resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) } @@ -325,7 +328,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { operationType: "Replace" as const, id: x[idKey], resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) }, @@ -447,7 +450,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { container.items.query(q, { partitionKey: nsBasePartitionKey(ns) }).fetchAll() ) yield* annotateFeed(response) - return response.resources.map((_) => ({ ...defaultValues, ...mapReverseId(_) })) + return response.resources.map((_) => fromStored(_ as unknown as Encoded)) }) .pipe( annotateDb({ @@ -520,7 +523,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { container.items.query<{ f: M }>(q, { partitionKey: nsBasePartitionKey(ns) }).fetchAll() ) yield* annotateFeed(response) - return response.resources.map(({ f }) => ({ ...defaultValues, ...mapReverseId(f as any) }) as any) + return response.resources.map(({ f }) => fromStored(f as Encoded) as any) }) .pipe( annotateDb({ @@ -546,7 +549,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) yield* annotateItem(response) return Option.fromNullishOr(response.resource).pipe( - Option.map((_) => ({ ...defaultValues, ...mapReverseId(_) })) + Option.map((_) => fromStored(_)) ) }) .pipe(annotateDb({ @@ -573,12 +576,12 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { { onNone: () => container.items.create({ - ...mapId(e), + ...mapId(codec.encode(e)), _partitionKey: nsPartitionValue(ns, e) }), onSome: (eTag) => container.item(e[idKey], nsPartitionValue(ns, e)).replace( - { ...mapId(e), _partitionKey: nsPartitionValue(ns, e) }, + { ...mapId(codec.encode(e)), _partitionKey: nsPartitionValue(ns, e) }, { accessCondition: { type: "IfMatch", diff --git a/packages/infra/src/Store/Disk.ts b/packages/infra/src/Store/Disk.ts index 0d5bf3f05..862e33dbb 100644 --- a/packages/infra/src/Store/Disk.ts +++ b/packages/infra/src/Store/Disk.ts @@ -10,6 +10,7 @@ import * as Console from "effect/Console" import { flow } from "effect/Function" import * as Semaphore from "effect/Semaphore" import { annotateDb } from "../otel.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { makeMemoryStoreInt } from "./Memory.ts" function makeDiskStoreInt( @@ -19,9 +20,11 @@ function makeDiskStoreInt, E, R>, - defaultValues?: Partial + defaultValues?: Partial, + schema?: StoreConfig["schema"] ) { type PM = PersistenceModelType + const codec = makeJsonDocumentCodec(schema) return Effect.gen(function*() { if (namespace !== "primary") { dir = dir + "/" + namespace @@ -44,7 +47,7 @@ function makeDiskStoreInt - Effect.sync(() => JSON.parse(x) as PM[]).pipe( + Effect.sync(() => (JSON.parse(x) as PM[]).map((row) => codec.decode(row))).pipe( annotateDb({ operation: "read.parse", system: "disk", @@ -67,7 +70,7 @@ function makeDiskStoreInt) => Effect - .sync(() => JSON.stringify([...v], undefined, 2)) + .sync(() => JSON.stringify([...v].map((row) => codec.encode(row)), undefined, 2)) .pipe( annotateDb({ operation: "stringify", @@ -117,7 +120,8 @@ function makeDiskStoreInt, E, R>, config?: StoreConfig ) { - const primary = yield* makeDiskStoreInt(prefix, idKey, "primary", dir, name, seed, config?.defaultValues).pipe( - Effect.orDie + const primary = yield* makeDiskStoreInt( + prefix, + idKey, + "primary", + dir, + name, + seed, + config?.defaultValues, + config?.schema ) + .pipe( + Effect.orDie + ) const stores = new Map>([["primary", primary]]) const ctx = yield* Effect.context() const semaphores = new Map() @@ -204,7 +218,8 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) { dir, name, seed, - config?.defaultValues + config?.defaultValues, + config?.schema ) .pipe( Effect.orDie, diff --git a/packages/infra/src/Store/Memory.ts b/packages/infra/src/Store/Memory.ts index 059488d41..e6fffe584 100644 --- a/packages/infra/src/Store/Memory.ts +++ b/packages/infra/src/Store/Memory.ts @@ -18,7 +18,8 @@ import * as Struct from "effect/Struct" import { InfraLogger } from "../logger.ts" import { annotateDb } from "../otel.ts" import { codeFilter, codeFilter3_ } from "./codeFilter.ts" -import { get, makeUpdateETag } from "./utils.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" +import { get, jsonifyFilter, makeUpdateETag, toJsonQueryValue } from "./utils.ts" export { get } from "./utils.ts" @@ -332,25 +333,35 @@ export function makeMemoryStoreInt, E, R>, - _defaultValues?: Partial + _defaultValues?: Partial, + schema?: StoreConfig["schema"] ) { type PM = PersistenceModelType return Effect.gen(function*() { const updateETag = makeUpdateETag(modelName) + const codec = makeJsonDocumentCodec(schema) + const encodeDoc = (e: Encoded | PM): PM => codec.encode({ _etag: undefined, ...e }) + const decodeDoc = (e: PM): PM => codec.decode(e) const items_ = yield* seed ?? Effect.sync(() => []) - const defaultValues = _defaultValues ?? {} + const encodedDefaults = toJsonQueryValue(_defaultValues ?? {}) as Partial - const items = new Map([...items_].map((_) => [_[idKey], { _etag: undefined, ...defaultValues, ..._ }] as const)) + const items = new Map( + [...items_].map((_) => { + const encoded = encodeDoc({ ...encodedDefaults, ..._ }) + return [encoded[idKey], encoded] as const + }) + ) const store = Ref.makeUnsafe>(items) const sem = Semaphore.makeUnsafe(1) const withPermit = sem.withPermits(1) const values = Effect.map(Ref.get(store), (s) => s.values()) - const all = Effect.map(values, Array.fromIterable) + const allStored = Effect.map(values, Array.fromIterable) + const all = Effect.map(allStored, (rows) => rows.map(decodeDoc)) const batchSet = (items: NonEmptyReadonlyArray) => Effect - .forEach(items, (i) => Effect.flatMap(s.find(i[idKey]), (current) => updateETag(i, idKey, current))) + .forEach(items, (i) => Effect.flatMap(s.find(i[idKey]), (current) => updateETag(encodeDoc(i), idKey, current))) .pipe( Effect .tap((items) => @@ -368,7 +379,7 @@ export function makeMemoryStoreInt _), + .map((items) => items.map(decodeDoc) as unknown as NonEmptyReadonlyArray), withPermit ) @@ -414,7 +425,7 @@ export function makeMemoryStoreInt Option.fromNullishOr(_.get(id))), + Effect.map((_) => Option.fromNullishOr(_.get(id)).pipe(Option.map(decodeDoc))), annotateDb({ operation: "find", system: "memory", @@ -424,11 +435,16 @@ export function makeMemoryStoreInt - all + filter: (f: FilterArgs) => + allStored .pipe( - Effect.tap(() => logQuery(f, defaultValues)), - Effect.map(memFilter(f)), + Effect.tap(() => logQuery(f, encodedDefaults)), + Effect.map(memFilter({ ...f, filter: f.filter ? jsonifyFilter(f.filter) : f.filter })), + Effect.map((rows): (U extends undefined ? Encoded : Pick)[] => + f.select + ? rows as (U extends undefined ? Encoded : Pick)[] + : rows.map(decodeDoc) as (U extends undefined ? Encoded : Pick)[] + ), annotateDb({ operation: "filter", system: "memory", @@ -441,14 +457,15 @@ export function makeMemoryStoreInt updateETag(e, idKey, current)), + Effect.flatMap((current) => updateETag(encodeDoc(e), idKey, current.pipe(Option.map(encodeDoc)))), Effect - .tap((e) => + .tap((stored) => Ref.get(store).pipe( - Effect.map((_) => new Map([..._, [e[idKey], e]])), + Effect.map((_) => new Map([..._, [stored[idKey], stored]])), Effect.flatMap((_) => Ref.set(store, _)) ) ), + Effect.map(decodeDoc), withPermit, annotateDb({ operation: "set", @@ -523,7 +540,8 @@ export const makeMemoryStore = () => ({ idKey, "primary", seed, - config?.defaultValues + config?.defaultValues, + config?.schema ) const ctx = yield* Effect.context() const stores = new Map([["primary", primary]]) @@ -543,7 +561,7 @@ export const makeMemoryStore = () => ({ if (config?.allowNamespace && !config.allowNamespace(namespace)) { throw new Error(`Namespace ${namespace} not allowed!`) } - return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues) + return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues, config?.schema) .pipe( Effect.orDie, Effect.provide(ctx), diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index f612a2e50..ebd09d301 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -15,6 +15,7 @@ import { SqlClient } from "effect/unstable/sql" import { DatabaseError, OptimisticConcurrencyException } from "../errors.ts" import { InfraLogger } from "../logger.ts" import { annotateDb, type DbSystem } from "../otel.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.ts" import { makeETag } from "./utils.ts" @@ -46,10 +47,13 @@ export class WithNsTransaction export const parseRow = ( row: { id: string; _etag: string | null; data: string }, idKey: PropertyKey, - defaultValues: Partial + defaultValues: Partial, + decode: (doc: PersistenceModelType) => PersistenceModelType = (doc) => doc ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object - return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + return decode( + { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + ) } const parseSelectRow = ( @@ -87,6 +91,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: type PM = PersistenceModelType const tableName = `${prefix}${name}` const defaultValues = config?.defaultValues ?? {} + const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -112,11 +117,11 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: ) const toRow = (e: PM) => { - const newE = makeETag(e) + const newE = makeETag(codec.encode(e)) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + return { id, _etag: newE._etag!, data, item: codec.decode(newE) } } const exec = (query: string, params?: readonly unknown[]) => @@ -209,7 +214,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = ?` return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, codec.decode)) + ), annotateDb({ operation: "all", system, @@ -231,7 +238,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, codec.decode)) : Option.none() }), annotateDb({ @@ -303,7 +310,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, codec.decode) as any as M + ) }) ) ), @@ -419,6 +428,7 @@ function makeSQLiteStorePerNs( type PM = PersistenceModelType const tableName = `${prefix}${name}` const defaultValues = config?.defaultValues ?? {} + const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -430,11 +440,11 @@ function makeSQLiteStorePerNs( })) const toRow = (e: PM) => { - const newE = makeETag(e) + const newE = makeETag(codec.encode(e)) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + return { id, _etag: newE._etag!, data, item: codec.decode(newE) } } const exec = (ns: string, query: string, params?: readonly unknown[]) => @@ -549,7 +559,9 @@ function makeSQLiteStorePerNs( const sqlText = `SELECT id, _etag, data FROM "${tableName}"` return exec(ns, sqlText) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, codec.decode)) + ), annotateDb({ operation: "all", system: "sqlite", @@ -570,7 +582,7 @@ function makeSQLiteStorePerNs( Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, codec.decode)) : Option.none() }), annotateDb({ @@ -641,7 +653,9 @@ function makeSQLiteStorePerNs( } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, codec.decode) as any as M + ) }) ) ), diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index 4fffa4e8c..afba64d89 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -12,6 +12,7 @@ import { SqlClient } from "effect/unstable/sql" import { DatabaseError, OptimisticConcurrencyException } from "../../errors.ts" import { InfraLogger } from "../../logger.ts" import { annotateDb } from "../../otel.ts" +import { makeJsonDocumentCodec } from "../jsonDocument.ts" import { makeETag } from "../utils.ts" import { buildWhereSQLQuery, logQuery, pgDialect } from "./query.ts" @@ -36,10 +37,13 @@ const preserveStoreError = (e: unknown): DatabaseError | OptimisticConcurrencyEx const parseRow = ( row: { id: string; _etag: string | null; data: unknown }, idKey: PropertyKey, - defaultValues: Partial + defaultValues: Partial, + decode: (doc: PersistenceModelType) => PersistenceModelType = (doc) => doc ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object - return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + return decode( + { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + ) } const parseSelectRow = ( @@ -71,6 +75,7 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { type PM = PersistenceModelType const tableName = `${prefix}${name}` const defaultValues = config?.defaultValues ?? {} + const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -96,11 +101,11 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) const toRow = (e: PM) => { - const newE = makeETag(e) + const newE = makeETag(codec.encode(e)) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + return { id, _etag: newE._etag!, data, item: codec.decode(newE) } } const exec = (query: string, params?: readonly unknown[]) => @@ -193,7 +198,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = $1` return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, codec.decode)) + ), annotateDb({ operation: "all", system: "postgresql", @@ -215,7 +222,7 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, codec.decode)) : Option.none() }), annotateDb({ @@ -286,7 +293,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, codec.decode) as any as M + ) }) ) ), diff --git a/packages/infra/src/Store/jsonDocument.ts b/packages/infra/src/Store/jsonDocument.ts new file mode 100644 index 000000000..d0302208b --- /dev/null +++ b/packages/infra/src/Store/jsonDocument.ts @@ -0,0 +1,43 @@ +import type { FieldValues } from "effect-app/Model/filter/types" +import * as S from "effect-app/Schema" +import type { PersistenceModelType } from "effect-app/Store" +import { toJsonQueryValue } from "./utils.ts" + +export interface JsonDocumentCodec { + readonly encode: (doc: PersistenceModelType) => PersistenceModelType + readonly decode: (doc: PersistenceModelType) => PersistenceModelType +} + +const splitEtag = (doc: PersistenceModelType) => { + const { _etag, ...rest } = doc + return { rest: rest as E, _etag } +} + +const joinEtag = ( + rest: E, + _etag: string | undefined +): PersistenceModelType => (_etag === undefined ? rest : { ...rest, _etag }) + +/** + * Encoded document ↔ JSON document. Prefer `Schema.toCodecJson(toEncoded(schema))` + * when the store has a schema; otherwise lower Date/Map/Set structurally. + */ +export const makeJsonDocumentCodec = (schema?: S.Top): JsonDocumentCodec => { + if (schema) { + const codec = S.toCodecJson(S.toEncoded(schema)) as S.Codec + return { + encode: (doc) => { + const { rest, _etag } = splitEtag(doc) + return joinEtag(S.encodeSync(codec)(rest) as E, _etag) + }, + decode: (doc) => { + const { rest, _etag } = splitEtag(doc) + return joinEtag(S.decodeSync(codec)(rest as S.Json), _etag) + } + } + } + return { + encode: (doc) => toJsonQueryValue(doc) as PersistenceModelType, + decode: (doc) => doc + } +} diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index b37fec667..f5cdc07f9 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -11,10 +11,15 @@ import * as Option from "effect-app/Option" import * as S from "effect-app/Schema" import { setupRequestContextFromCurrent } from "effect-app/setupRequest" import { flow, pipe } from "effect/Function" +import * as Redacted from "effect/Redacted" import * as SchemaTransformation from "effect/SchemaTransformation" import * as Struct from "effect/Struct" +import * as fs from "fs" +import * as os from "os" +import * as path from "path" import { inspect } from "util" import { expect, expectTypeOf, it } from "vitest" +import { DiskStoreLayer } from "../src/Store/Disk.js" import { memFilter, MemoryStoreLive } from "../src/Store/Memory.js" import { SomeService } from "./fixtures.js" @@ -181,6 +186,86 @@ it("works with repo", () => Effect.runPromise )) +it("memory store round-trips Date/Set/Map via JSON codecs", () => + Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDoc")({ + id: S.String, + at: S.Date, + tags: S.ReadonlySet(S.String), + meta: S.ReadonlyMap({ key: S.String, value: S.Finite }) + }) {} + const at = new Date("2024-06-01T00:00:00.000Z") + const saved = new Doc({ + id: "d1", + at, + tags: new Set(["a", "b"]), + meta: new Map([["n", 1]]) + }) + const repo = yield* makeRepo("JsonCodecDoc", Doc, { makeInitial: Effect.succeed([saved]) }) + const found = yield* repo.find("d1") + expect(Option.isSome(found)).toBe(true) + if (Option.isSome(found)) { + expect(found.value.at.toISOString()).toBe(at.toISOString()) + expect(found.value.tags).toEqual(new Set(["a", "b"])) + expect(found.value.meta).toEqual(new Map([["n", 1]])) + } + const byDate = yield* repo.query(where("at", at)) + expect(byDate.map((_) => _.id)).toEqual(["d1"]) + const byTag = yield* repo.query(where("tags", "includes", "b")) + expect(byTag.map((_) => _.id)).toEqual(["d1"]) + }) + .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) + +it("disk store round-trips Date/Set/Map via JSON codecs", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "effect-app-disk-json-")) + const diskLive = Layer.merge( + DiskStoreLayer({ url: Redacted.make(`disk://${dir}`), prefix: "", dbName: "test" }, dir), + RepositoryRegistryLive + ) + return Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDiskDoc")({ + id: S.String, + at: S.Date, + tags: S.ReadonlySet(S.String), + meta: S.ReadonlyMap({ key: S.String, value: S.Finite }) + }) {} + const at = new Date("2024-06-01T00:00:00.000Z") + const saved = new Doc({ + id: "d1", + at, + tags: new Set(["a", "b"]), + meta: new Map([["n", 1]]) + }) + const repo = yield* makeRepo("JsonCodecDiskDoc", Doc, { makeInitial: Effect.succeed([saved]) }) + const found = yield* repo.find("d1") + expect(Option.isSome(found)).toBe(true) + if (Option.isSome(found)) { + expect(found.value.at.toISOString()).toBe(at.toISOString()) + expect(found.value.tags).toEqual(new Set(["a", "b"])) + expect(found.value.meta).toEqual(new Map([["n", 1]])) + } + const jsonFile = fs.readdirSync(dir).find((f) => f.endsWith(".json")) + expect(jsonFile).toBeDefined() + const raw = JSON.parse(fs.readFileSync(path.join(dir, jsonFile!), "utf8")) as Array<{ + at: unknown + tags: unknown + meta: unknown + }> + expect(raw[0]?.at).toBe(at.toISOString()) + expect(raw[0]?.tags).toEqual(["a", "b"]) + expect(raw[0]?.meta).toEqual([["n", 1]]) + }) + .pipe( + Effect.provide(diskLive), + setupRequestContextFromCurrent(), + Effect.scoped, + Effect.runPromise + ) + .finally(() => fs.rmSync(dir, { recursive: true, force: true })) +}) + it("collect", () => Effect .gen(function*() { From 4ba9c93f0f17ca11f2364d0af2a14260e69a6322 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:35:38 +0000 Subject: [PATCH 05/10] feat(query): Map hasKey/hasValue/hasKeyValue filter ops JSON-encoded maps are arrays of [key, value] tuples. Add hasKey, hasValue, hasKeyValue (and not*/any/all variants) on the query DSL, implemented in memory, SQLite, Postgres, and Cosmos. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- .changeset/query-map-has-ops.md | 8 + .../effect-app/src/Model/filter/filterApi.ts | 18 ++ packages/effect-app/src/Model/query/dsl.ts | 178 ++++++++++++++++++ packages/infra/src/Store/Cosmos/query.ts | 55 ++++++ packages/infra/src/Store/SQL/query.ts | 140 ++++++++++++++ packages/infra/src/Store/codeFilter.ts | 47 +++++ packages/infra/test/cosmos-query.test.ts | 36 ++++ packages/infra/test/query.test.ts | 52 +++++ packages/infra/test/sql-store.test.ts | 104 ++++++++++ 9 files changed, 638 insertions(+) create mode 100644 .changeset/query-map-has-ops.md diff --git a/.changeset/query-map-has-ops.md b/.changeset/query-map-has-ops.md new file mode 100644 index 000000000..62a7d17bc --- /dev/null +++ b/.changeset/query-map-has-ops.md @@ -0,0 +1,8 @@ +--- +"effect-app": minor +"@effect-app/infra": minor +--- + +Query maps as JSON arrays of `[key, value]` tuples. + +`where("meta", "hasKey" | "hasValue" | "hasKeyValue", ...)` (and `not*` / `*-any` / `*-all` variants) filter `ReadonlyMap` fields. Memory, Disk, SQLite, Postgres, and Cosmos compile those ops against the encoded tuple array. diff --git a/packages/effect-app/src/Model/filter/filterApi.ts b/packages/effect-app/src/Model/filter/filterApi.ts index 83e014bc2..81c5eb250 100644 --- a/packages/effect-app/src/Model/filter/filterApi.ts +++ b/packages/effect-app/src/Model/filter/filterApi.ts @@ -17,6 +17,24 @@ export type OtherOps = | "notIncludes-any" | "includes-all" | "notIncludes-all" + | "hasKey" + | "notHasKey" + | "hasValue" + | "notHasValue" + | "hasKeyValue" + | "notHasKeyValue" + | "hasKey-any" + | "notHasKey-any" + | "hasKey-all" + | "notHasKey-all" + | "hasValue-any" + | "notHasValue-any" + | "hasValue-all" + | "notHasValue-all" + | "hasKeyValue-any" + | "notHasKeyValue-any" + | "hasKeyValue-all" + | "notHasKeyValue-all" | "eq" | "neq" | "gt" diff --git a/packages/effect-app/src/Model/query/dsl.ts b/packages/effect-app/src/Model/query/dsl.ts index 3d114eb73..b91351471 100644 --- a/packages/effect-app/src/Model/query/dsl.ts +++ b/packages/effect-app/src/Model/query/dsl.ts @@ -1161,6 +1161,10 @@ type GetArV = T extends ReadonlySet ? R : T extends readonly (infer R)[] ? R : never +type GetMapK = T extends ReadonlyMap ? K : never +type GetMapV = T extends ReadonlyMap ? V : never +type GetMapEntry = T extends ReadonlyMap ? readonly [K, V] : never + type InValues = readonly T[] | ReadonlySet export type FilterContinuations = { @@ -1264,6 +1268,96 @@ export type FilterContinuations = { V extends FieldPathValue, TFieldValuesRefined extends TFieldValues = TFieldValues, E extends boolean = false + >( + path: TFieldName, + op: "hasKey" | "notHasKey", + value: GetMapK + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasValue" | "notHasValue", + value: GetMapV + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKeyValue" | "notHasKeyValue", + value: GetMapEntry + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKey-any" | "notHasKey-any" | "hasKey-all" | "notHasKey-all", + value: InValues> + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasValue-any" | "notHasValue-any" | "hasValue-all" | "notHasValue-all", + value: InValues> + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKeyValue-any" | "notHasKeyValue-any" | "hasKeyValue-all" | "notHasKeyValue-all", + value: InValues> + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false >(f: { path: TFieldName op: "eq" @@ -1364,6 +1458,90 @@ export type FilterContinuationsWithSubpath = { ): ( current: Query ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKey" | "notHasKey", + value: GetMapK + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasValue" | "notHasValue", + value: GetMapV + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKeyValue" | "notHasKeyValue", + value: GetMapEntry + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKey-any" | "notHasKey-any" | "hasKey-all" | "notHasKey-all", + value: InValues> + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasValue-any" | "notHasValue-any" | "hasValue-all" | "notHasValue-all", + value: InValues> + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKeyValue-any" | "notHasKeyValue-any" | "hasKeyValue-all" | "notHasKeyValue-all", + value: InValues> + ): ( + current: Query + ) => QueryWhere < TFieldValues extends FieldValues, TFieldName extends FieldPath, diff --git a/packages/infra/src/Store/Cosmos/query.ts b/packages/infra/src/Store/Cosmos/query.ts index e7589f059..f4689eee8 100644 --- a/packages/infra/src/Store/Cosmos/query.ts +++ b/packages/infra/src/Store/Cosmos/query.ts @@ -105,6 +105,43 @@ export function buildWhereCosmosQuery3( (x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") }))` + case "hasKey": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = ${v})` + case "notHasKey": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = ${v}))` + case "hasValue": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = ${v})` + case "notHasValue": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = ${v}))` + case "hasKeyValue": + return `ARRAY_CONTAINS(${k}, ${v})` + case "notHasKeyValue": + return `(NOT ARRAY_CONTAINS(${k}, ${v}))` + case "hasKey-any": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[0]))` + case "notHasKey-any": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[0])))` + case "hasKey-all": + return `(NOT EXISTS(SELECT VALUE key FROM key IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = key)))` + case "notHasKey-all": + return `EXISTS(SELECT VALUE key FROM key IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = key))` + case "hasValue-any": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[1]))` + case "notHasValue-any": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[1])))` + case "hasValue-all": + return `(NOT EXISTS(SELECT VALUE val FROM val IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = val)))` + case "notHasValue-all": + return `EXISTS(SELECT VALUE val FROM val IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = val))` + case "hasKeyValue-any": + return `EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE ARRAY_CONTAINS(${k}, pair))` + case "notHasKeyValue-any": + return `(NOT EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE ARRAY_CONTAINS(${k}, pair)))` + case "hasKeyValue-all": + return `(NOT EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE NOT ARRAY_CONTAINS(${k}, pair)))` + case "notHasKeyValue-all": + return `EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE NOT ARRAY_CONTAINS(${k}, pair))` + case "contains": return `CONTAINS(${k}, ${v}, true)` @@ -165,6 +202,24 @@ export function buildWhereCosmosQuery3( "notIncludes-any": "includes-any", "includes-all": "notIncludes-all", "notIncludes-all": "includes-all", + hasKey: "notHasKey", + notHasKey: "hasKey", + hasValue: "notHasValue", + notHasValue: "hasValue", + hasKeyValue: "notHasKeyValue", + notHasKeyValue: "hasKeyValue", + "hasKey-any": "notHasKey-any", + "notHasKey-any": "hasKey-any", + "hasKey-all": "notHasKey-all", + "notHasKey-all": "hasKey-all", + "hasValue-any": "notHasValue-any", + "notHasValue-any": "hasValue-any", + "hasValue-all": "notHasValue-all", + "notHasValue-all": "hasValue-all", + "hasKeyValue-any": "notHasKeyValue-any", + "notHasKeyValue-any": "hasKeyValue-any", + "hasKeyValue-all": "notHasKeyValue-all", + "notHasKeyValue-all": "hasKeyValue-all", in: "notIn", notIn: "in" } satisfies Record diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index 5bb044bf6..a8d6ad980 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -18,6 +18,9 @@ export interface SQLDialect { readonly jsonArrayNotContainsAny: (arrPath: string, valPlaceholders: readonly string[]) => string readonly jsonArrayContainsAll: (arrPath: string, valPlaceholders: readonly string[]) => string readonly jsonArrayNotContainsAll: (arrPath: string, valPlaceholders: readonly string[]) => string + readonly jsonMapHasKey: (arrPath: string, valPlaceholder: string) => string + readonly jsonMapHasValue: (arrPath: string, valPlaceholder: string) => string + readonly jsonMapHasPair: (arrPath: string, valPlaceholder: string) => string readonly caseInsensitiveLike: (expr: string, valPlaceholder: string) => string readonly caseInsensitiveNotLike: (expr: string, valPlaceholder: string) => string readonly jsonColumnType: "JSON" | "JSONB" @@ -46,6 +49,11 @@ export const sqliteDialect: SQLDialect = { `NOT (${ vals.map((v) => `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE value = ${v})`).join(" AND ") })`, + jsonMapHasKey: (arrPath, val) => + `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE json_extract(value, '$[0]') = ${val})`, + jsonMapHasValue: (arrPath, val) => + `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE json_extract(value, '$[1]') = ${val})`, + jsonMapHasPair: (arrPath, val) => `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE value = ${val})`, caseInsensitiveLike: (expr, val) => `LOWER(${expr}) LIKE LOWER(${val})`, caseInsensitiveNotLike: (expr, val) => `LOWER(${expr}) NOT LIKE LOWER(${val})`, jsonColumnType: "JSON", @@ -113,6 +121,27 @@ export const pgDialect: SQLDialect = { : `data${parts.map((p) => `->'${p}'`).join("")}` return `NOT (${vals.map((v) => `${jsonPath} @> ${v}::jsonb`).join(" AND ")})` }, + jsonMapHasKey: (arrPath, val) => { + const parts = arrPath.split(".") + const jsonPath = parts.length === 1 + ? `data->'${parts[0]}'` + : `data${parts.map((p) => `->'${p}'`).join("")}` + return `EXISTS(SELECT 1 FROM jsonb_array_elements(${jsonPath}) e WHERE e->0 = ${val}::jsonb)` + }, + jsonMapHasValue: (arrPath, val) => { + const parts = arrPath.split(".") + const jsonPath = parts.length === 1 + ? `data->'${parts[0]}'` + : `data${parts.map((p) => `->'${p}'`).join("")}` + return `EXISTS(SELECT 1 FROM jsonb_array_elements(${jsonPath}) e WHERE e->1 = ${val}::jsonb)` + }, + jsonMapHasPair: (arrPath, val) => { + const parts = arrPath.split(".") + const jsonPath = parts.length === 1 + ? `data->'${parts[0]}'` + : `data${parts.map((p) => `->'${p}'`).join("")}` + return `${jsonPath} @> jsonb_build_array(${val}::jsonb)` + }, caseInsensitiveLike: (expr, val) => `${expr} ILIKE ${val}`, caseInsensitiveNotLike: (expr, val) => `${expr} NOT ILIKE ${val}`, jsonColumnType: "JSONB", @@ -150,6 +179,14 @@ const dottedToJsonPath = (path: string) => .filter((p) => p !== "-1") .join(".") +const mapItemParam = (dialect: SQLDialect, value: unknown) => + dialect.jsonColumnType === "JSONB" ? dialect.serializeJsonValue(value) : value + +const mapPairParam = (dialect: SQLDialect, value: unknown) => { + const encoded = dialect.serializeJsonValue(value) + return typeof encoded === "string" ? encoded : JSON.stringify(value) +} + const sqlStringLiteral = (value: string) => `'${value.replaceAll("'", "''")}'` export function buildWhereSQLQuery( @@ -278,6 +315,109 @@ export function buildWhereSQLQuery( return dialect.jsonArrayNotContainsAll(arrPath, placeholders) } + case "hasKey": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return dialect.jsonMapHasKey(arrPath, v) + } + case "notHasKey": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return `NOT (${dialect.jsonMapHasKey(arrPath, v)})` + } + case "hasValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return dialect.jsonMapHasValue(arrPath, v) + } + case "notHasValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return `NOT (${dialect.jsonMapHasValue(arrPath, v)})` + } + case "hasKeyValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapPairParam(dialect, x.value)) + return dialect.jsonMapHasPair(arrPath, v) + } + case "notHasKeyValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapPairParam(dialect, x.value)) + return `NOT (${dialect.jsonMapHasPair(arrPath, v)})` + } + case "hasKey-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))) + return `(${parts.join(" OR ")})` + } + case "notHasKey-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))) + return `NOT (${parts.join(" OR ")})` + } + case "hasKey-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + } + case "notHasKey-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return `NOT (${ + vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + })` + } + case "hasValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))) + return `(${parts.join(" OR ")})` + } + case "notHasValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))) + return `NOT (${parts.join(" OR ")})` + } + case "hasValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + } + case "notHasValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return `NOT (${ + vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + })` + } + case "hasKeyValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))) + return `(${parts.join(" OR ")})` + } + case "notHasKeyValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))) + return `NOT (${parts.join(" OR ")})` + } + case "hasKeyValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))).join(" AND ") + } + case "notHasKeyValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return `NOT (${ + vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))).join(" AND ") + })` + } + case "contains": { const v = addParam(`%${x.value}%`) return dialect.caseInsensitiveLike(k, v) diff --git a/packages/infra/src/Store/codeFilter.ts b/packages/infra/src/Store/codeFilter.ts index 8a04fb5fe..5de521c4b 100644 --- a/packages/infra/src/Store/codeFilter.ts +++ b/packages/infra/src/Store/codeFilter.ts @@ -10,6 +10,17 @@ import { compare, get, greaterThan, greaterThanExclusive, lowerThan, lowerThanEx const vAsArr = (v: unknown) => toJsonQueryValue(v) as any[] +const mapEntries = (value: unknown): readonly [unknown, unknown][] => { + const json = toJsonQueryValue(value) + if (!Array.isArray(json)) return [] + return json.filter((entry): entry is [unknown, unknown] => Array.isArray(entry) && entry.length >= 2) +} + +const pairEq = (entry: readonly [unknown, unknown], pair: unknown) => { + const json = toJsonQueryValue(pair) + return Array.isArray(json) && json.length >= 2 && compare(entry[0], json[0]) && compare(entry[1], json[1]) +} + const filterStatement = (x: any, p: FilterR) => { const k = toJsonQueryValue(get(x, p.path)) const v = toJsonQueryValue(p.value) @@ -38,6 +49,42 @@ const filterStatement = (x: any, p: FilterR) => { return (vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) case "notIncludes-all": return !(vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) + case "hasKey": + return mapEntries(k).some(([key]) => compare(key, v)) + case "notHasKey": + return !mapEntries(k).some(([key]) => compare(key, v)) + case "hasValue": + return mapEntries(k).some(([, val]) => compare(val, v)) + case "notHasValue": + return !mapEntries(k).some(([, val]) => compare(val, v)) + case "hasKeyValue": + return mapEntries(k).some((entry) => pairEq(entry, v)) + case "notHasKeyValue": + return !mapEntries(k).some((entry) => pairEq(entry, v)) + case "hasKey-any": + return vAsArr(p.value).some((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "notHasKey-any": + return !vAsArr(p.value).some((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "hasKey-all": + return vAsArr(p.value).every((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "notHasKey-all": + return !vAsArr(p.value).every((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "hasValue-any": + return vAsArr(p.value).some((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "notHasValue-any": + return !vAsArr(p.value).some((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "hasValue-all": + return vAsArr(p.value).every((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "notHasValue-all": + return !vAsArr(p.value).every((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "hasKeyValue-any": + return vAsArr(p.value).some((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) + case "notHasKeyValue-any": + return !vAsArr(p.value).some((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) + case "hasKeyValue-all": + return vAsArr(p.value).every((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) + case "notHasKeyValue-all": + return !vAsArr(p.value).every((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) case "contains": return (k as string).toLowerCase().includes((v as string).toLowerCase()) case "endsWith": diff --git a/packages/infra/test/cosmos-query.test.ts b/packages/infra/test/cosmos-query.test.ts index 7c4535322..c3462e351 100644 --- a/packages/infra/test/cosmos-query.test.ts +++ b/packages/infra/test/cosmos-query.test.ts @@ -38,6 +38,42 @@ describe("cosmos query filter: native Encoded values", () => { ) }) + it("emits EXISTS over Map JSON tuples for hasKey / hasValue / hasKeyValue", () => { + const byKey = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "n" }], + "Orders", + {} + ) + expect(byKey.query).toContain("EXISTS(SELECT VALUE p FROM p IN f[\"meta\"] WHERE p[0] = @v0)") + expect(byKey.parameters).toEqual(expect.arrayContaining([{ name: "@v0", value: "n" }])) + + const byValue = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasValue", value: 1 }], + "Orders", + {} + ) + expect(byValue.query).toContain("p[1] = @v0") + + const byPair = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "Orders", + {} + ) + expect(byPair.query).toContain("ARRAY_CONTAINS") + expect(byPair.parameters).toEqual(expect.arrayContaining([{ name: "@v0", value: ["n", 1] }])) + + const anyKey = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasKey-any", value: ["n", "x"] }], + "Orders", + {} + ) + expect(anyKey.query).toContain("ARRAY_CONTAINS(@v0, p[0])") + }) + it("binds includes Date as ISO string", () => { const result = buildWhereCosmosQuery3( "id", diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index f5cdc07f9..7028e1461 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -214,6 +214,10 @@ it("memory store round-trips Date/Set/Map via JSON codecs", () => expect(byDate.map((_) => _.id)).toEqual(["d1"]) const byTag = yield* repo.query(where("tags", "includes", "b")) expect(byTag.map((_) => _.id)).toEqual(["d1"]) + const byKey = yield* repo.query(where("meta", "hasKey", "n")) + expect(byKey.map((_) => _.id)).toEqual(["d1"]) + const byPair = yield* repo.query(where("meta", "hasKeyValue", ["n", 1])) + expect(byPair.map((_) => _.id)).toEqual(["d1"]) }) .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) @@ -1456,6 +1460,28 @@ it("does not allow string queries on arrays", () => expectTypeOf(n4).toEqualTypeOf>() expectTypeOf(n5).toEqualTypeOf>() expectTypeOf(n6).toEqualTypeOf>() + + type WithMap = { + readonly id: string + readonly meta: ReadonlyMap + } + const mapped = make() + const m1 = mapped.pipe(where("meta", "hasKey", "n")) + const m2 = mapped.pipe(where("meta", "hasValue", 1)) + const m3 = mapped.pipe(where("meta", "hasKeyValue", ["n", 1] as const)) + const m4 = mapped.pipe(where("meta", "hasKey-any", ["n", "x"])) + const m5 = mapped.pipe(where("meta", "hasValue-all", new Set([1, 2]))) + const m6 = mapped.pipe(where("meta", "hasKeyValue-any", [["n", 1] as const, ["x", 2] as const])) + expectTypeOf(m1).toEqualTypeOf>() + expectTypeOf(m2).toEqualTypeOf>() + expectTypeOf(m3).toEqualTypeOf>() + expectTypeOf(m4).toEqualTypeOf>() + expectTypeOf(m5).toEqualTypeOf>() + expectTypeOf(m6).toEqualTypeOf>() + // @ts-expect-error cannot hasKey on a string field + mapped.pipe(where("id", "hasKey", "n")) + // @ts-expect-error hasKey value must be the map key type + mapped.pipe(where("meta", "hasKey", 1)) }) .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) @@ -2200,6 +2226,32 @@ it("codeFilter: Date array / Set includes and in", () => { expect(run(make().pipe(where("tag", "in", new Set(["a"]))))).toEqual(["1"]) }) +it("codeFilter: Map hasKey / hasValue / hasKeyValue", () => { + type MapRow = { + readonly id: string + readonly meta: ReadonlyMap + } + const rows: MapRow[] = [ + { id: "1", meta: new Map([["n", 1], ["x", 2]]) }, + { id: "2", meta: new Map([["n", 9]]) }, + { id: "3", meta: new Map([["z", 2]]) } + ] + const run = (q: any) => (memFilter(toFilter(q))(rows) as MapRow[]).map((_) => _.id) + expect(run(make().pipe(where("meta", "hasKey", "n"))).sort()).toEqual(["1", "2"]) + expect(run(make().pipe(where("meta", "notHasKey", "n")))).toEqual(["3"]) + expect(run(make().pipe(where("meta", "hasValue", 2))).sort()).toEqual(["1", "3"]) + expect(run(make().pipe(where("meta", "notHasValue", 2)))).toEqual(["2"]) + expect(run(make().pipe(where("meta", "hasKeyValue", ["n", 1])))).toEqual(["1"]) + expect(run(make().pipe(where("meta", "notHasKeyValue", ["n", 1]))).sort()).toEqual(["2", "3"]) + expect(run(make().pipe(where("meta", "hasKey-any", ["z", "missing"])))).toEqual(["3"]) + expect(run(make().pipe(where("meta", "hasKey-all", ["n", "x"])))).toEqual(["1"]) + expect(run(make().pipe(where("meta", "notHasKey-all", ["n", "x"]))).sort()).toEqual(["2", "3"]) + expect(run(make().pipe(where("meta", "hasValue-any", [9, 99])))).toEqual(["2"]) + expect(run(make().pipe(where("meta", "hasValue-all", [1, 2])))).toEqual(["1"]) + expect(run(make().pipe(where("meta", "hasKeyValue-any", [["n", 9], ["missing", 0]])))).toEqual(["2"]) + expect(run(make().pipe(where("meta", "hasKeyValue-all", [["n", 1], ["x", 2]])))).toEqual(["1"]) +}) + it("codeFilter: in / notIn", () => { expect(runCF(make().pipe(where("tag", "in", ["x", "z"]))).sort()).toEqual(["1", "3"]) expect(runCF(make().pipe(where("tag", "notIn", ["x", "z"]))).sort()).toEqual(["2", "4"]) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 59adc1c25..a9c0e87e0 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -57,6 +57,72 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("2024-01-01T00:00:00.000Z") }) + it("where hasKey / hasValue / hasKeyValue on Map JSON tuples", () => { + const key = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "n" }], + "users", + {} + ) + expect(key.sql).toContain("json_extract(value, '$[0]')") + expect(key.params).toContain("n") + + const value = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasValue", value: 1 }], + "users", + {} + ) + expect(value.sql).toContain("json_extract(value, '$[1]')") + expect(value.params).toContain(1) + + const pair = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "users", + {} + ) + expect(pair.sql).toContain("json_each") + expect(pair.params).toContain(JSON.stringify(["n", 1])) + + const anyKeys = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey-any", value: ["n", "x"] }], + "users", + {} + ) + expect(anyKeys.sql).toContain(" OR ") + expect(anyKeys.params).toEqual(expect.arrayContaining(["n", "x"])) + }) + + it("pg where hasKey / hasKeyValue uses jsonb tuple elements", () => { + const key = buildWhereSQLQuery( + pgDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "n" }], + "users", + {} + ) + expect(key.sql).toContain("jsonb_array_elements") + expect(key.sql).toContain("e->0") + expect(key.params).toContain(JSON.stringify("n")) + + const pair = buildWhereSQLQuery( + pgDialect, + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "users", + {} + ) + expect(pair.sql).toContain("@>") + expect(pair.sql).toContain("jsonb_build_array") + expect(pair.params).toContain(JSON.stringify(["n", 1])) + }) + it("where includes-any Date[] binds ISO strings", () => { const result = buildWhereSQLQuery( sqliteDialect, @@ -1508,6 +1574,44 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { expect((JSON.parse(rows[0].data) as any).name).toBe("Alice") })) + it("where hasKey / hasValue / hasKeyValue match Map tuple JSON", () => + withDb((db) => { + db.exec(`CREATE TABLE "t" (id TEXT PRIMARY KEY, _etag TEXT, data JSON NOT NULL)`) + db + .prepare(`INSERT INTO "t" (id, _etag, data) VALUES (?, ?, ?)`) + .run("1", "e", JSON.stringify({ meta: [["n", 1], ["x", 2]] })) + db + .prepare(`INSERT INTO "t" (id, _etag, data) VALUES (?, ?, ?)`) + .run("2", "e", JSON.stringify({ meta: [["n", 9]] })) + + const byKey = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "x" }], + "t", + {} + ) + expect(query(db, byKey.sql, byKey.params).map((r) => r.id)).toEqual(["1"]) + + const byValue = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasValue", value: 9 }], + "t", + {} + ) + expect(query(db, byValue.sql, byValue.params).map((r) => r.id)).toEqual(["2"]) + + const byPair = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "t", + {} + ) + expect(query(db, byPair.sql, byPair.params).map((r) => r.id)).toEqual(["1"]) + })) + it("where neq boolean works", () => withDb((db) => { db.exec(`CREATE TABLE "t" (id TEXT PRIMARY KEY, _etag TEXT, data JSON NOT NULL)`) From bc7c0a446817e89c686feb71d41146bdda0b91ff Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:00:25 +0000 Subject: [PATCH 06/10] fix(store): JSON-lower Encoded defaults before document decode Cosmos/SQL merged StoreConfig.defaultValues (now native Date/Map/Set) into stored JSON before toCodecJson decode. Lower defaults the same way Memory already does. Also use globalThis.Date in schema helpers, Reflect for Error.stackTraceLimit, and Sendgrid send ReturnType so linked-source app typechecks. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- packages/effect-app/src/Context.ts | 6 +++--- packages/effect-app/src/Schema/ext.ts | 4 ++-- packages/infra/src/Emailer/Sendgrid.ts | 4 ++-- packages/infra/src/Store/Cosmos.ts | 5 +++-- packages/infra/src/Store/SQL.ts | 9 +++++---- packages/infra/src/Store/SQL/Pg.ts | 7 ++++--- packages/infra/test/sql-store.test.ts | 10 ++++++++++ 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/packages/effect-app/src/Context.ts b/packages/effect-app/src/Context.ts index 8846a8fa8..16e8609c8 100644 --- a/packages/effect-app/src/Context.ts +++ b/packages/effect-app/src/Context.ts @@ -48,10 +48,10 @@ export function assignTag ({ * omitted from `.make(...)` input. NOT applied during decode — cannot be * used to JIT-migrate database fields. See file-level note. */ - withConstructorDefault: s.pipe(S.withConstructorDefault(Effect.sync(() => new global.Date()))), + withConstructorDefault: s.pipe(S.withConstructorDefault(Effect.sync(() => new globalThis.Date()))), /** * Decode-time default `new Date()`. **Discouraged for persisted data:** a * missing field may be data corruption, not an old-shape document; silently @@ -127,7 +127,7 @@ const dateHelpers = (s: S.Date) => ({ * preferably versioned migration over a decode-time fallback. See * file-level note. */ - withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new global.Date()))) + withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new globalThis.Date()))) }) /** Like the default Schema `Date` (Encoded is `Date`) with default helpers. */ diff --git a/packages/infra/src/Emailer/Sendgrid.ts b/packages/infra/src/Emailer/Sendgrid.ts index 25744fa4f..c11dfaaf7 100644 --- a/packages/infra/src/Emailer/Sendgrid.ts +++ b/packages/infra/src/Emailer/Sendgrid.ts @@ -53,8 +53,8 @@ const makeSendgrid = ( const ret = yield* Effect .callback< - [sgMail.ClientResponse, Record], - Error | sgMail.ResponseError + Awaited>, + Error >( (resume) => void sgMail.send( diff --git a/packages/infra/src/Store/Cosmos.ts b/packages/infra/src/Store/Cosmos.ts index d1a336c42..5c0f8e458 100644 --- a/packages/infra/src/Store/Cosmos.ts +++ b/packages/infra/src/Store/Cosmos.ts @@ -20,6 +20,7 @@ import { InfraLogger } from "../logger.ts" import { annotateCosmosResponse, annotateDb } from "../otel.ts" import { buildWhereCosmosQuery3, logQuery } from "./Cosmos/query.ts" import { makeJsonDocumentCodec } from "./jsonDocument.ts" +import { toJsonQueryValue } from "./utils.ts" const makeMapId = (idKey: IdKey) => ({ [idKey]: id, ...e }: Encoded) => ({ @@ -98,7 +99,8 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { const mapId = makeMapId(idKey) const mapReverseId = makeReverseMapId(idKey) const codec = makeJsonDocumentCodec(config?.schema) - const fromStored = (raw: Encoded) => codec.decode({ ...config?.defaultValues, ...mapReverseId(raw as any) }) + const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial + const fromStored = (raw: Encoded) => codec.decode({ ...defaultValues, ...mapReverseId(raw as any) }) type PM = PersistenceModelType type PMCosmos = PersistenceModelType & { id: string }> const containerId = `${prefix}${name}` @@ -131,7 +133,6 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { return namespace })) - const defaultValues = config?.defaultValues ?? {} const container = db.container(containerId) const bulk = container.items.bulk.bind(container.items) const execBatch = container.items.batch.bind(container.items) diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index ebd09d301..16cda28fe 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -17,7 +17,7 @@ import { InfraLogger } from "../logger.ts" import { annotateDb, type DbSystem } from "../otel.ts" import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.ts" -import { makeETag } from "./utils.ts" +import { makeETag, toJsonQueryValue } from "./utils.ts" const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e) const sqlIsTransient = (e: unknown) => @@ -51,8 +51,9 @@ export const parseRow = ( decode: (doc: PersistenceModelType) => PersistenceModelType = (doc) => doc ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object + const jsonDefaults = toJsonQueryValue(defaultValues) as Partial return decode( - { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + { ...jsonDefaults, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType ) } @@ -90,7 +91,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} + const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace @@ -427,7 +428,7 @@ function makeSQLiteStorePerNs( ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} + const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index afba64d89..fec2bc78b 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -13,7 +13,7 @@ import { DatabaseError, OptimisticConcurrencyException } from "../../errors.ts" import { InfraLogger } from "../../logger.ts" import { annotateDb } from "../../otel.ts" import { makeJsonDocumentCodec } from "../jsonDocument.ts" -import { makeETag } from "../utils.ts" +import { makeETag, toJsonQueryValue } from "../utils.ts" import { buildWhereSQLQuery, logQuery, pgDialect } from "./query.ts" const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e) @@ -41,8 +41,9 @@ const parseRow = ( decode: (doc: PersistenceModelType) => PersistenceModelType = (doc) => doc ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object + const jsonDefaults = toJsonQueryValue(defaultValues) as Partial return decode( - { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + { ...jsonDefaults, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType ) } @@ -74,7 +75,7 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} + const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index a9c0e87e0..ed65b4883 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -1863,4 +1863,14 @@ describe("parseRow reconstructs full object from row", () => { expect(reconstructed.tags).toEqual(["admin"]) expect(reconstructed._etag).toBe(newE._etag) }) + + it("lowers Date defaultValues to ISO before merge", () => { + const result: any = parseRow( + { id: "1", _etag: "e1", data: JSON.stringify({ name: "Alice" }) }, + "id", + { at: new Date("2024-06-01T00:00:00.000Z") } + ) + expect(result.at).toBe("2024-06-01T00:00:00.000Z") + expect(result.name).toBe("Alice") + }) }) From 98f43840c3c2260df95b4a2e22902eb8d1c7cd3a Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:40:34 +0000 Subject: [PATCH 07/10] revert: drop unrelated Context/Sendgrid typecheck workarounds Restore Error.stackTraceLimit assignment and Sendgrid ClientResponse callback types. They are unrelated to native Encoded query work. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- packages/effect-app/src/Context.ts | 6 +++--- packages/infra/src/Emailer/Sendgrid.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/effect-app/src/Context.ts b/packages/effect-app/src/Context.ts index 16e8609c8..8846a8fa8 100644 --- a/packages/effect-app/src/Context.ts +++ b/packages/effect-app/src/Context.ts @@ -48,10 +48,10 @@ export function assignTag>, - Error + [sgMail.ClientResponse, Record], + Error | sgMail.ResponseError >( (resume) => void sgMail.send( From b9932f8a00fd3305ac5387f289d1f1006f4ae657 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 07:15:22 +0000 Subject: [PATCH 08/10] feat(store): JsonValues for app native Encoded types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Date/Map/Set stay built-in. App schemas (DateOnly, …) register via JsonValues, StoreConfig.jsonValues, or registerJsonSchema so query params and schemaless documents lower through toCodecJson without special-casing those types in effect-app. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- .changeset/native-encoded-query-json.md | 2 +- .changeset/store-json-values.md | 8 +++ packages/effect-app/src/Store.ts | 8 +++ packages/infra/src/Store/Cosmos.ts | 8 +-- packages/infra/src/Store/Cosmos/query.ts | 10 ++-- packages/infra/src/Store/Disk.ts | 14 ++++-- packages/infra/src/Store/Memory.ts | 17 ++++--- packages/infra/src/Store/SQL.ts | 15 ++++-- packages/infra/src/Store/SQL/Pg.ts | 9 ++-- packages/infra/src/Store/SQL/query.ts | 10 ++-- packages/infra/src/Store/index.ts | 2 + packages/infra/src/Store/jsonValues.ts | 51 +++++++++++++++++++ packages/infra/src/Store/utils.ts | 60 ++++++++++++++++++----- packages/infra/test/query.test.ts | 62 ++++++++++++++++++++++++ packages/infra/test/sql-store.test.ts | 26 ++++++++++ 15 files changed, 260 insertions(+), 42 deletions(-) create mode 100644 .changeset/store-json-values.md create mode 100644 packages/infra/src/Store/jsonValues.ts diff --git a/.changeset/native-encoded-query-json.md b/.changeset/native-encoded-query-json.md index 2686c0041..a7a9b298a 100644 --- a/.changeset/native-encoded-query-json.md +++ b/.changeset/native-encoded-query-json.md @@ -6,4 +6,4 @@ Stop forcing Date/Map/Set Encoded shapes to JSON. -`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values, including array ops (`includes` / `in` / `includes-any`) on `Date[]` and `ReadonlySet` fields. Memory, Disk, SQL, and Cosmos convert Encoded Date/Map/Set through `Schema.toCodecJson` on write/read; query parameters are lowered the same way. +`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values, including array ops (`includes` / `in` / `includes-any`) on `Date[]` and `ReadonlySet` fields. Memory, Disk, SQL, and Cosmos convert Encoded Date/Map/Set through `Schema.toCodecJson` on write/read; query parameters are lowered the same way. App schemas (DateOnly, …) register the same way via `JsonValues` / `StoreConfig.jsonValues`. diff --git a/.changeset/store-json-values.md b/.changeset/store-json-values.md new file mode 100644 index 000000000..888b1ba0c --- /dev/null +++ b/.changeset/store-json-values.md @@ -0,0 +1,8 @@ +--- +"effect-app": minor +"@effect-app/infra": minor +--- + +App schemas can plug native Encoded values into JSON stores without baking them into effect-app. + +`StoreConfig.jsonValues` and `JsonValues` (a Context service) take schemas whose Encoded form is a native value (DateOnly, branded money, …). Query params and schemaless documents lower through `Schema.toCodecJson(toEncoded(schema))`. Date/Map/Set stay built in. Use `registerJsonSchema` for process-wide registration. diff --git a/packages/effect-app/src/Store.ts b/packages/effect-app/src/Store.ts index e554b685b..60e950e43 100644 --- a/packages/effect-app/src/Store.ts +++ b/packages/effect-app/src/Store.ts @@ -51,6 +51,14 @@ export interface StoreConfig { * `Schema.toCodecJson(Schema.toEncoded(schema))` so Date/Map/Set round-trip. */ schema?: SchemaTop + /** + * Extra schemas whose Encoded form is a native (non-JSON) value — app types + * such as DateOnly. Adapters lower query params and schemaless documents + * through `toCodecJson(toEncoded(schema))`. Date/Map/Set are built in. + * Prefer providing these app-wide via `JsonValues` instead of repeating + * them on every store. + */ + jsonValues?: readonly SchemaTop[] } export type SupportedValues = string | boolean | number | null diff --git a/packages/infra/src/Store/Cosmos.ts b/packages/infra/src/Store/Cosmos.ts index 5c0f8e458..9119a1369 100644 --- a/packages/infra/src/Store/Cosmos.ts +++ b/packages/infra/src/Store/Cosmos.ts @@ -20,7 +20,7 @@ import { InfraLogger } from "../logger.ts" import { annotateCosmosResponse, annotateDb } from "../otel.ts" import { buildWhereCosmosQuery3, logQuery } from "./Cosmos/query.ts" import { makeJsonDocumentCodec } from "./jsonDocument.ts" -import { toJsonQueryValue } from "./utils.ts" +import { makeJsonLower } from "./utils.ts" const makeMapId = (idKey: IdKey) => ({ [idKey]: id, ...e }: Encoded) => ({ @@ -99,7 +99,8 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { const mapId = makeMapId(idKey) const mapReverseId = makeReverseMapId(idKey) const codec = makeJsonDocumentCodec(config?.schema) - const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial + const json = yield* makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial const fromStored = (raw: Encoded) => codec.decode({ ...defaultValues, ...mapReverseId(raw as any) }) type PM = PersistenceModelType type PMCosmos = PersistenceModelType & { id: string }> @@ -496,7 +497,8 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { | undefined, f.order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined, skip, - limit + limit, + json ) ), ns: resolveNamespace diff --git a/packages/infra/src/Store/Cosmos/query.ts b/packages/infra/src/Store/Cosmos/query.ts index f4689eee8..266e0a821 100644 --- a/packages/infra/src/Store/Cosmos/query.ts +++ b/packages/infra/src/Store/Cosmos/query.ts @@ -9,7 +9,7 @@ import type { SupportedValues } from "effect-app/Store" import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.ts" import { isRelationCheck } from "../codeFilter.ts" -import { jsonifyFilter, toJsonQueryValue } from "../utils.ts" +import { jsonifyFilter, type JsonLower, toJsonQueryValue } from "../utils.ts" export function logQuery(q: { query: string @@ -61,10 +61,12 @@ export function buildWhereCosmosQuery3( >, order?: NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }>, skip?: number, - limit?: number + limit?: number, + json?: JsonLower ) { - filter = jsonifyFilter(filter) - defaultValues = toJsonQueryValue(defaultValues) as Record + const toJson = json?.toJson ?? toJsonQueryValue + filter = (json?.jsonifyFilter ?? jsonifyFilter)(filter) + defaultValues = toJson(defaultValues) as Record const statement = (x: FilterR, i: number) => { if (x.path === idKey) { x = { ...x, path: "id" } diff --git a/packages/infra/src/Store/Disk.ts b/packages/infra/src/Store/Disk.ts index 862e33dbb..090112c54 100644 --- a/packages/infra/src/Store/Disk.ts +++ b/packages/infra/src/Store/Disk.ts @@ -12,6 +12,7 @@ import * as Semaphore from "effect/Semaphore" import { annotateDb } from "../otel.ts" import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { makeMemoryStoreInt } from "./Memory.ts" +import { type JsonLower, makeJsonLower } from "./utils.ts" function makeDiskStoreInt( prefix: string, @@ -21,7 +22,8 @@ function makeDiskStoreInt, E, R>, defaultValues?: Partial, - schema?: StoreConfig["schema"] + schema?: StoreConfig["schema"], + json?: JsonLower ) { type PM = PersistenceModelType const codec = makeJsonDocumentCodec(schema) @@ -121,7 +123,8 @@ function makeDiskStoreInt, E, R>, config?: StoreConfig ) { + const json = yield* makeJsonLower(config) const primary = yield* makeDiskStoreInt( prefix, idKey, @@ -187,7 +191,8 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) { name, seed, config?.defaultValues, - config?.schema + config?.schema, + json ) .pipe( Effect.orDie @@ -219,7 +224,8 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) { name, seed, config?.defaultValues, - config?.schema + config?.schema, + json ) .pipe( Effect.orDie, diff --git a/packages/infra/src/Store/Memory.ts b/packages/infra/src/Store/Memory.ts index e6fffe584..152c980df 100644 --- a/packages/infra/src/Store/Memory.ts +++ b/packages/infra/src/Store/Memory.ts @@ -19,7 +19,7 @@ import { InfraLogger } from "../logger.ts" import { annotateDb } from "../otel.ts" import { codeFilter, codeFilter3_ } from "./codeFilter.ts" import { makeJsonDocumentCodec } from "./jsonDocument.ts" -import { get, jsonifyFilter, makeUpdateETag, toJsonQueryValue } from "./utils.ts" +import { get, jsonifyFilter, type JsonLower, makeJsonLower, makeUpdateETag, toJsonQueryValue } from "./utils.ts" export { get } from "./utils.ts" @@ -334,7 +334,8 @@ export function makeMemoryStoreInt, E, R>, _defaultValues?: Partial, - schema?: StoreConfig["schema"] + schema?: StoreConfig["schema"], + json?: JsonLower ) { type PM = PersistenceModelType return Effect.gen(function*() { @@ -343,7 +344,9 @@ export function makeMemoryStoreInt codec.encode({ _etag: undefined, ...e }) const decodeDoc = (e: PM): PM => codec.decode(e) const items_ = yield* seed ?? Effect.sync(() => []) - const encodedDefaults = toJsonQueryValue(_defaultValues ?? {}) as Partial + const toJson = json?.toJson ?? toJsonQueryValue + const lowerFilter = json?.jsonifyFilter ?? jsonifyFilter + const encodedDefaults = toJson(_defaultValues ?? {}) as Partial const items = new Map( [...items_].map((_) => { @@ -439,7 +442,7 @@ export function makeMemoryStoreInt logQuery(f, encodedDefaults)), - Effect.map(memFilter({ ...f, filter: f.filter ? jsonifyFilter(f.filter) : f.filter })), + Effect.map(memFilter({ ...f, filter: f.filter ? lowerFilter(f.filter) : f.filter })), Effect.map((rows): (U extends undefined ? Encoded : Pick)[] => f.select ? rows as (U extends undefined ? Encoded : Pick)[] @@ -535,13 +538,15 @@ export const makeMemoryStore = () => ({ seed?: Effect.Effect, E, R>, config?: StoreConfig ) { + const json = yield* makeJsonLower(config) const primary = yield* makeMemoryStoreInt( modelName, idKey, "primary", seed, config?.defaultValues, - config?.schema + config?.schema, + json ) const ctx = yield* Effect.context() const stores = new Map([["primary", primary]]) @@ -561,7 +566,7 @@ export const makeMemoryStore = () => ({ if (config?.allowNamespace && !config.allowNamespace(namespace)) { throw new Error(`Namespace ${namespace} not allowed!`) } - return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues, config?.schema) + return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues, config?.schema, json) .pipe( Effect.orDie, Effect.provide(ctx), diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index 16cda28fe..a79b11579 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -17,7 +17,7 @@ import { InfraLogger } from "../logger.ts" import { annotateDb, type DbSystem } from "../otel.ts" import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.ts" -import { makeETag, toJsonQueryValue } from "./utils.ts" +import { makeETag, makeJsonLower, toJsonQueryValue } from "./utils.ts" const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e) const sqlIsTransient = (e: unknown) => @@ -91,7 +91,8 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial + const json = yield* makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace @@ -289,7 +290,8 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: .skip, f .limit, - ns + ns, + json ) }) .pipe( @@ -428,7 +430,8 @@ function makeSQLiteStorePerNs( ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial + const json = yield* makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace @@ -632,7 +635,9 @@ function makeSQLiteStorePerNs( f .skip, f - .limit + .limit, + undefined, + json ) ) .pipe( diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index fec2bc78b..56d9a75d3 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -13,7 +13,7 @@ import { DatabaseError, OptimisticConcurrencyException } from "../../errors.ts" import { InfraLogger } from "../../logger.ts" import { annotateDb } from "../../otel.ts" import { makeJsonDocumentCodec } from "../jsonDocument.ts" -import { makeETag, toJsonQueryValue } from "../utils.ts" +import { makeETag, makeJsonLower, toJsonQueryValue } from "../utils.ts" import { buildWhereSQLQuery, logQuery, pgDialect } from "./query.ts" const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e) @@ -75,7 +75,8 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = toJsonQueryValue(config?.defaultValues ?? {}) as Partial + const json = yield* makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace @@ -264,7 +265,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { | undefined, f.order, f.skip, - f.limit + f.limit, + ns, + json ) const nsPlaceholder = pgDialect.placeholder(q.params.length + 1) const hasWhere = q.sql.includes("WHERE") diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index a8d6ad980..c41dc6abe 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -6,7 +6,7 @@ import type { AggregateIrExpression, ComputedProjectionIrExpression, ComputedPro import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.ts" import { isRelationCheck } from "../codeFilter.ts" -import { jsonifyFilter, toJsonQueryValue } from "../utils.ts" +import { jsonifyFilter, type JsonLower, toJsonQueryValue } from "../utils.ts" export interface SQLDialect { readonly jsonExtract: (path: string) => string @@ -213,10 +213,12 @@ export function buildWhereSQLQuery( order?: NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }>, skip?: number, limit?: number, - namespace?: string + namespace?: string, + json?: JsonLower ) { - filter = jsonifyFilter(filter) - defaultValues = toJsonQueryValue(defaultValues) as Record + const toJson = json?.toJson ?? toJsonQueryValue + filter = (json?.jsonifyFilter ?? jsonifyFilter)(filter) + defaultValues = toJson(defaultValues) as Record const params: unknown[] = [] let paramIndex = 1 diff --git a/packages/infra/src/Store/index.ts b/packages/infra/src/Store/index.ts index 030a41586..0254a3071 100644 --- a/packages/infra/src/Store/index.ts +++ b/packages/infra/src/Store/index.ts @@ -12,6 +12,8 @@ import { MemoryStoreLive } from "./Memory.ts" import { SQLiteStoreLayer } from "./SQL.ts" import { PgStoreLayer } from "./SQL/Pg.ts" +export { JsonValues, JsonValuesLayer, registerJsonSchema, registerJsonValue } from "./jsonValues.ts" + export function StoreMakerLayer( cfg: StorageConfig, options?: { makeSqlClientLayer?: (namespace: string) => Layer.Layer } diff --git a/packages/infra/src/Store/jsonValues.ts b/packages/infra/src/Store/jsonValues.ts new file mode 100644 index 000000000..266e431c8 --- /dev/null +++ b/packages/infra/src/Store/jsonValues.ts @@ -0,0 +1,51 @@ +import * as Context from "effect-app/Context" +import * as Layer from "effect-app/Layer" +import * as S from "effect-app/Schema" + +/** + * Native Encoded value that is not JSON, plus how to lower it for document-DB + * adapters. Date / Map / Set are built in; app schemas (DateOnly, money, …) + * register here instead of being special-cased in effect-app. + */ +export interface JsonValueHandler { + readonly is: (u: unknown) => boolean + readonly toJson: (u: unknown) => unknown +} + +const registered: JsonValueHandler[] = [] + +/** Process-wide handler. Prefer {@link JsonValues} / StoreConfig.jsonValues. */ +export const registerJsonValue = (handler: JsonValueHandler) => { + registered.push(handler) +} + +/** + * Register a schema whose Encoded form is a native (non-JSON) value. + * Uses `Schema.is(toEncoded(schema))` and `toCodecJson(toEncoded(schema))`. + */ +export const registerJsonSchema = (schema: S.Top) => { + registered.push(handlerForSchema(schema)) +} + +export const jsonValueHandlers = () => registered + +export const handlerForSchema = (schema: S.Top): JsonValueHandler => { + const encoded = S.toEncoded(schema) + const json = S.toCodecJson(encoded) + return { + is: (u) => S.is(encoded)(u), + toJson: (u) => S.encodeSync(json)(u) + } +} + +export const jsonHandlersFromSchemas = (schemas: readonly S.Top[]): JsonValueHandler[] => schemas.map(handlerForSchema) + +/** + * App-provided schemas whose Encoded values are native (DateOnly, branded + * money, …). Store adapters merge this with `StoreConfig.jsonValues`. + */ +export class JsonValues extends Context.Service()("effect-app/Store/JsonValues") {} + +export const JsonValuesLayer = (schemas: readonly S.Top[]) => Layer.succeed(JsonValues, { schemas }) diff --git a/packages/infra/src/Store/utils.ts b/packages/infra/src/Store/utils.ts index 0204ccb39..acce99523 100644 --- a/packages/infra/src/Store/utils.ts +++ b/packages/infra/src/Store/utils.ts @@ -5,52 +5,88 @@ import * as Option from "effect-app/Option" import * as S from "effect-app/Schema" import type { PersistenceModelType, SupportedValues2 } from "effect-app/Store" import { OptimisticConcurrencyException } from "../errors.ts" +import { jsonHandlersFromSchemas, type JsonValueHandler, jsonValueHandlers, JsonValues } from "./jsonValues.ts" const dateJson = S.toCodecJson(S.Date) +const applyHandlers = (value: unknown, extra: readonly JsonValueHandler[]): unknown => { + for (const h of extra) { + if (h.is(value)) return toJsonQueryValue(h.toJson(value), extra) + } + for (const h of jsonValueHandlers()) { + if (h.is(value)) return toJsonQueryValue(h.toJson(value), extra) + } + return undefined +} + /** - * Lower Date / Map / Set query and document values to JSON, matching - * `Schema.toCodecJson` of those declarations so document-DB adapters can bind - * native Encoded values as JSON parameters. + * Lower Date / Map / Set (and app-registered native Encoded values) to JSON, + * matching `Schema.toCodecJson` so document-DB adapters can bind them as + * JSON parameters. App schemas register via `registerJsonSchema` / + * `JsonValues` / `StoreConfig.jsonValues` instead of being special-cased here. */ -export function toJsonQueryValue(value: unknown): unknown { +export function toJsonQueryValue(value: unknown, extra: readonly JsonValueHandler[] = []): unknown { + const handled = applyHandlers(value, extra) + if (handled !== undefined) return handled if (value instanceof Date) { return S.encodeSync(dateJson)(value) } if (value instanceof Map) { - return [...value.entries()].map(([k, v]) => [toJsonQueryValue(k), toJsonQueryValue(v)]) + return [...value.entries()].map(([k, v]) => [toJsonQueryValue(k, extra), toJsonQueryValue(v, extra)]) } if (value instanceof Set) { - return [...value].map(toJsonQueryValue) + return [...value].map((v) => toJsonQueryValue(v, extra)) } if (Array.isArray(value)) { - return value.map(toJsonQueryValue) + return value.map((v) => toJsonQueryValue(v, extra)) } if (value !== null && typeof value === "object") { const proto = Object.getPrototypeOf(value) if (proto === Object.prototype || proto === null) { const out: Record = {} for (const [k, v] of Object.entries(value)) { - out[k] = toJsonQueryValue(v) + out[k] = toJsonQueryValue(v, extra) } return out } const toJSON = (value as { toJSON?: () => unknown }).toJSON if (typeof toJSON === "function") { - return toJsonQueryValue(toJSON.call(value)) + return toJsonQueryValue(toJSON.call(value), extra) } } return value } -export function jsonifyFilter(filter: readonly FilterResult[]): FilterResult[] { +export function jsonifyFilter( + filter: readonly FilterResult[], + extra: readonly JsonValueHandler[] = [] +): FilterResult[] { return filter.map((r) => r.t === "and-scope" || r.t === "or-scope" || r.t === "where-scope" - ? { ...r, result: jsonifyFilter(r.result) } - : { ...r, value: toJsonQueryValue(r.value) } + ? { ...r, result: jsonifyFilter(r.result, extra) } + : { ...r, value: toJsonQueryValue(r.value, extra) } ) } +export type JsonLower = { + readonly toJson: (value: unknown) => unknown + readonly jsonifyFilter: (filter: readonly FilterResult[]) => FilterResult[] +} + +export const makeJsonLower = Effect.fnUntraced(function*(config?: { + readonly jsonValues?: readonly S.Top[] +}) { + const provided = yield* Effect.serviceOption(JsonValues) + const extra = jsonHandlersFromSchemas([ + ...(Option.isSome(provided) ? provided.value.schemas : []), + ...(config?.jsonValues ?? []) + ]) + return { + toJson: (value: unknown) => toJsonQueryValue(value, extra), + jsonifyFilter: (filter: readonly FilterResult[]) => jsonifyFilter(filter, extra) + } satisfies JsonLower +}) + /** Traverse an object by a dot-separated path string, e.g. `"a.b.c"`. */ export function get(obj: any, path: string): any { return path.split(".").reduce((res: any, key: string) => (res != null ? res[key] : res), obj) diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index 7028e1461..c22a57e13 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -12,6 +12,7 @@ import * as S from "effect-app/Schema" import { setupRequestContextFromCurrent } from "effect-app/setupRequest" import { flow, pipe } from "effect/Function" import * as Redacted from "effect/Redacted" +import * as Getter from "effect/SchemaGetter" import * as SchemaTransformation from "effect/SchemaTransformation" import * as Struct from "effect/Struct" import * as fs from "fs" @@ -20,6 +21,7 @@ import * as path from "path" import { inspect } from "util" import { expect, expectTypeOf, it } from "vitest" import { DiskStoreLayer } from "../src/Store/Disk.js" +import { JsonValuesLayer } from "../src/Store/jsonValues.js" import { memFilter, MemoryStoreLive } from "../src/Store/Memory.js" import { SomeService } from "./fixtures.js" @@ -221,6 +223,66 @@ it("memory store round-trips Date/Set/Map via JSON codecs", () => }) .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) +class Day { + constructor(readonly ymd: string) {} +} + +const DayFromSelf = S.declare((u): u is Day => u instanceof Day, { + expected: "Day", + toCodecJson: () => + S.link()( + S.String, + { + decode: Getter.transform((s: string) => new Day(s)), + encode: Getter.transform((d: Day) => d.ymd) + } + ) +}) + +it("memory store round-trips app native Encoded values via JsonValues", () => + Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDayDoc")({ + id: S.String, + day: DayFromSelf + }) {} + const day = new Day("2024-06-01") + const saved = new Doc({ id: "d1", day }) + const repo = yield* makeRepo("JsonCodecDayDoc", Doc, { makeInitial: Effect.succeed([saved]) }) + const found = yield* repo.find("d1") + expect(Option.isSome(found)).toBe(true) + if (Option.isSome(found)) { + expect(found.value.day).toBeInstanceOf(Day) + expect(found.value.day.ymd).toBe("2024-06-01") + } + const byDay = yield* repo.query(where("day", day)) + expect(byDay.map((_) => _.id)).toEqual(["d1"]) + }) + .pipe( + Effect.provide(Layer.mergeAll(TestStoreLive, JsonValuesLayer([DayFromSelf]))), + setupRequestContextFromCurrent(), + Effect.scoped, + Effect.runPromise + )) + +it("memory store round-trips app native Encoded values via StoreConfig.jsonValues", () => + Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDayDocConfig")({ + id: S.String, + day: DayFromSelf + }) {} + const day = new Day("2024-07-04") + const saved = new Doc({ id: "d2", day }) + const repo = yield* makeRepo("JsonCodecDayDocConfig", Doc, { + makeInitial: Effect.succeed([saved]), + config: { jsonValues: [DayFromSelf] } + }) + const byDay = yield* repo.query(where("day", day)) + expect(byDay.map((_) => _.id)).toEqual(["d2"]) + }) + .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) + it("disk store round-trips Date/Set/Map via JSON codecs", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "effect-app-disk-json-")) const diskLive = Layer.merge( diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index ed65b4883..b43ec1898 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -35,6 +35,32 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("2024-01-01T00:00:00.000Z") }) + it("where eq app native Encoded binds via jsonValues", () => { + class Day { + constructor(readonly ymd: string) {} + } + const day = new Day("2024-06-01") + const json = { + toJson: (v: unknown) => v instanceof Day ? v.ymd : v, + jsonifyFilter: (filter: readonly { t: string; path?: string; op?: string; value?: unknown }[]) => + filter.map((r) => r.t === "where" ? { ...r, value: r.value instanceof Day ? r.value.ymd : r.value } : r) + } + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "day", op: "eq", value: day }], + "users", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + json as never + ) + expect(result.params).toContain("2024-06-01") + }) + it("where in Set binds array values", () => { const result = buildWhereSQLQuery( sqliteDialect, From d8a495e9dc235aeb019376b4d6f0183a19aa54ac Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 07:15:44 +0000 Subject: [PATCH 09/10] fix(test): avoid parameter properties in JsonValues fixtures Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- packages/infra/test/query.test.ts | 5 ++++- packages/infra/test/sql-store.test.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index c22a57e13..586db6db0 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -224,7 +224,10 @@ it("memory store round-trips Date/Set/Map via JSON codecs", () => .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) class Day { - constructor(readonly ymd: string) {} + readonly ymd: string + constructor(ymd: string) { + this.ymd = ymd + } } const DayFromSelf = S.declare((u): u is Day => u instanceof Day, { diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index b43ec1898..7ec7ebf1f 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -37,7 +37,10 @@ describe("SQL query builder (SQLite dialect)", () => { it("where eq app native Encoded binds via jsonValues", () => { class Day { - constructor(readonly ymd: string) {} + readonly ymd: string + constructor(ymd: string) { + this.ymd = ymd + } } const day = new Day("2024-06-01") const json = { From c1f3c7dd6b2e0a8acc2965058d458940b04fa9e9 Mon Sep 17 00:00:00 2001 From: "omegent-app[bot]" <306514130+omegent-app[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 07:39:38 +0000 Subject: [PATCH 10/10] fix(infra): declare @sentry/node as a runtime dependency errorReporter.ts imports @sentry/node statically. Linked --prod Docker installs skip peer/dev copies, so Node cannot resolve it from source. Co-authored-by: Patrick Roza <42661+patroza@users.noreply.github.com> --- .changeset/infra-sentry-node-dep.md | 5 +++++ packages/infra/package.json | 2 +- pnpm-lock.yaml | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 .changeset/infra-sentry-node-dep.md diff --git a/.changeset/infra-sentry-node-dep.md b/.changeset/infra-sentry-node-dep.md new file mode 100644 index 000000000..30f66e1f3 --- /dev/null +++ b/.changeset/infra-sentry-node-dep.md @@ -0,0 +1,5 @@ +--- +"@effect-app/infra": patch +--- + +Declare `@sentry/node` as a runtime dependency of `@effect-app/infra`. `errorReporter.ts` imports it statically, so `pnpm install --prod` of linked source (Docker) must install it next to the package, not only as a peer of the app. diff --git a/packages/infra/package.json b/packages/infra/package.json index 5906dcd82..dcc58d1b8 100644 --- a/packages/infra/package.json +++ b/packages/infra/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@faker-js/faker": "^8.4.1", + "@sentry/node": "10.55.0", "effect-app": "workspace:*", "fast-check": "^4.9.0", "jose": "^6.2.3", @@ -21,7 +22,6 @@ "@azure/cosmos": "^4.9.3", "@azure/service-bus": "^7.9.5", "@effect/sql-sqlite-node": "4.0.0-beta.107", - "@sentry/node": "10.55.0", "@sentry/opentelemetry": "10.55.0", "@types/better-sqlite3": "^7.6.13", "@types/node": "25.9.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04496f088..428be274a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -342,6 +342,9 @@ importers: "@sendgrid/mail": specifier: ^8.1.6 version: 8.1.6(patch_hash=3d2dd1cd5f52d3eef1772e4be48776258a596f9277c00adf0a471f54fcd3e21a)(debug@4.4.3(supports-color@8.1.1)) + "@sentry/node": + specifier: 10.55.0 + version: 10.55.0(supports-color@8.1.1) effect: specifier: ^4.0.0-beta.107 version: 4.0.0-beta.107 @@ -373,9 +376,6 @@ importers: "@effect/sql-sqlite-node": specifier: 4.0.0-beta.107 version: 4.0.0-beta.107(effect@4.0.0-beta.107) - "@sentry/node": - specifier: 10.55.0 - version: 10.55.0(supports-color@8.1.1) "@sentry/opentelemetry": specifier: 10.55.0 version: 10.55.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/semantic-conventions@1.40.0)