diff --git a/.changeset/infra-sentry-node-dep.md b/.changeset/infra-sentry-node-dep.md new file mode 100644 index 000000000..30f66e1f3 --- /dev/null +++ b/.changeset/infra-sentry-node-dep.md @@ -0,0 +1,5 @@ +--- +"@effect-app/infra": patch +--- + +Declare `@sentry/node` as a runtime dependency of `@effect-app/infra`. `errorReporter.ts` imports it statically, so `pnpm install --prod` of linked source (Docker) must install it next to the package, not only as a peer of the app. diff --git a/.changeset/native-encoded-query-json.md b/.changeset/native-encoded-query-json.md new file mode 100644 index 000000000..e9c963a39 --- /dev/null +++ b/.changeset/native-encoded-query-json.md @@ -0,0 +1,9 @@ +--- +"effect-app": minor +"@effect-app/infra": minor +"@effect-app/vue-components": minor +--- + +Stop forcing Date/Map/Set Encoded shapes to JSON. + +`Schema.Date` / `ReadonlySet` / `ReadonlyMap` now keep native Encoded types (`Date`, `Set`, `Map`). Use `DateFromString`, `ReadonlySetFromArray`, and `ReadonlyMapFromArray` when the Encoded form must be JSON. The query DSL accepts those native values, including array ops (`includes` / `in` / `includes-any`) on `Date[]` and `ReadonlySet` fields. Memory, Disk, SQL, and Cosmos convert Encoded Date/Map/Set through `Schema.toCodecJson` on write/read; query parameters and defaults lower the same way from the store schema. App types such as DateOnly stay native Encoded and JSON-lower via that schema — not a type registry. diff --git a/.changeset/query-map-has-ops.md b/.changeset/query-map-has-ops.md new file mode 100644 index 000000000..62a7d17bc --- /dev/null +++ b/.changeset/query-map-has-ops.md @@ -0,0 +1,8 @@ +--- +"effect-app": minor +"@effect-app/infra": minor +--- + +Query maps as JSON arrays of `[key, value]` tuples. + +`where("meta", "hasKey" | "hasValue" | "hasKeyValue", ...)` (and `not*` / `*-any` / `*-all` variants) filter `ReadonlyMap` fields. Memory, Disk, SQLite, Postgres, and Cosmos compile those ops against the encoded tuple array. diff --git a/.changeset/store-json-values.md b/.changeset/store-json-values.md new file mode 100644 index 000000000..03fb1280e --- /dev/null +++ b/.changeset/store-json-values.md @@ -0,0 +1,8 @@ +--- +"effect-app": minor +"@effect-app/infra": minor +--- + +JSON stores lower native Encoded values (Date, Map, Set, and app types such as DateOnly) through the store's document schema. + +`makeRepo` already passes that schema. Adapters encode documents, query parameters, and defaults with `Schema.toCodecJson(toEncoded(schema))` at the field path. No type registry. Schemaless stores still lower Date/Map/Set structurally. diff --git a/packages/effect-app/src/Model/Repository/internal/internal.ts b/packages/effect-app/src/Model/Repository/internal/internal.ts index 33db662b3..0bb5cba8b 100644 --- a/packages/effect-app/src/Model/Repository/internal/internal.ts +++ b/packages/effect-app/src/Model/Repository/internal/internal.ts @@ -581,7 +581,7 @@ export function makeRepoInternal< .pipe( Effect.andThen( (items) => - S.decodeEffectConcurrently(S.Array(a.schema ?? schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(a.schema ?? schema)))(items).pipe( provideRctx, timeSchema("decode", name, "aggregate", items.length) ) @@ -593,7 +593,7 @@ export function makeRepoInternal< .pipe( Effect.andThen( (items) => - S.decodeEffectConcurrently(S.Array(a.schema ?? schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(a.schema ?? schema)))(items).pipe( provideRctx, timeSchema("decode", name, "project", items.length) ) @@ -604,7 +604,7 @@ export function makeRepoInternal< // TODO: mapFrom but need to support per field and dependencies .pipe( Effect.flatMap((items) => - S.decodeEffectConcurrently(S.Array(a.schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(a.schema)))(items).pipe( Effect.map(Array.getSomes), provideRctx, timeSchema("decode", name, "collect", items.length) @@ -737,7 +737,7 @@ export function makeRepoInternal< queryRaw(schema: S.Codec, q: Q.RawQuery) { return store.queryRaw(q).pipe( Effect.flatMap((items) => - S.decodeEffectConcurrently(S.Array(schema))(items).pipe( + S.decodeEffectConcurrently(S.Array(S.toCodecJson(schema)))(items as readonly S.Json[]).pipe( timeSchema("decode", name, undefined, items.length) ) ), @@ -887,6 +887,7 @@ export function makeStore() { : undefined, { ...config, + schema, partitionValue: config?.partitionValue ?? ((_) => "primary") /*(isIntegrationEvent(r) ? r.companyId : r.id*/ } diff --git a/packages/effect-app/src/Model/filter/filterApi.ts b/packages/effect-app/src/Model/filter/filterApi.ts index aba49c20b..81c5eb250 100644 --- a/packages/effect-app/src/Model/filter/filterApi.ts +++ b/packages/effect-app/src/Model/filter/filterApi.ts @@ -17,6 +17,24 @@ export type OtherOps = | "notIncludes-any" | "includes-all" | "notIncludes-all" + | "hasKey" + | "notHasKey" + | "hasValue" + | "notHasValue" + | "hasKeyValue" + | "notHasKeyValue" + | "hasKey-any" + | "notHasKey-any" + | "hasKey-all" + | "notHasKey-all" + | "hasValue-any" + | "notHasValue-any" + | "hasValue-all" + | "notHasValue-all" + | "hasKeyValue-any" + | "notHasKeyValue-any" + | "hasKeyValue-all" + | "notHasKeyValue-all" | "eq" | "neq" | "gt" @@ -44,7 +62,7 @@ export type FilterR = { op: Ops path: string - value: string // ToDO: Value[] + value: unknown } export type FilterResult = diff --git a/packages/effect-app/src/Model/query/dsl.ts b/packages/effect-app/src/Model/query/dsl.ts index d9efe2d04..b91351471 100644 --- a/packages/effect-app/src/Model/query/dsl.ts +++ b/packages/effect-app/src/Model/query/dsl.ts @@ -1157,7 +1157,15 @@ export const aggregate: { return new Project({ current, schema, mode: "aggregate", aggregateMap } as any) } -type GetArV = T extends readonly (infer R)[] ? R : never +type GetArV = T extends ReadonlySet ? R + : T extends readonly (infer R)[] ? R + : never + +type GetMapK = T extends ReadonlyMap ? K : never +type GetMapV = T extends ReadonlyMap ? V : never +type GetMapEntry = T extends ReadonlyMap ? readonly [K, V] : never + +type InValues = readonly T[] | ReadonlySet export type FilterContinuations = { < @@ -1207,13 +1215,12 @@ export type FilterContinuations = { < TFieldValues extends FieldValues, TFieldName extends FieldPath, - const V extends readonly FieldPathValue[], TFieldValuesRefined extends TFieldValues = TFieldValues, E extends boolean = false >( path: TFieldName, op: "in" | "notIn", - value: V + value: InValues> ): ( current: IsCurrentInitial extends true ? Query : QueryWhere @@ -1249,7 +1256,97 @@ export type FilterContinuations = { | "notIncludes-any" | "includes-all" | "notIncludes-all", - value: readonly GetArV[] + value: InValues> + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKey" | "notHasKey", + value: GetMapK + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasValue" | "notHasValue", + value: GetMapV + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKeyValue" | "notHasKeyValue", + value: GetMapEntry + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKey-any" | "notHasKey-any" | "hasKey-all" | "notHasKey-all", + value: InValues> + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasValue-any" | "notHasValue-any" | "hasValue-all" | "notHasValue-all", + value: InValues> + ): ( + current: IsCurrentInitial extends true ? Query + : QueryWhere + ) => IsCurrentInitial extends true ? QueryWhere + : QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + V extends FieldPathValue, + TFieldValuesRefined extends TFieldValues = TFieldValues, + E extends boolean = false + >( + path: TFieldName, + op: "hasKeyValue-any" | "notHasKeyValue-any" | "hasKeyValue-all" | "notHasKeyValue-all", + value: InValues> ): ( current: IsCurrentInitial extends true ? Query : QueryWhere @@ -1318,12 +1415,12 @@ export type FilterContinuationsWithSubpath = { TFieldName extends FieldPath, TFieldValuesSub extends TFieldValues[TFieldName][number], TFieldNameSub extends FieldPath, - const V extends readonly FieldPathValue[] + V extends FieldPathValue >( subPath: TFieldName, restPath: TFieldNameSub, op: "in" | "notIn", - value: V + value: InValues ): ( current: Query ) => QueryWhere @@ -1357,7 +1454,91 @@ export type FilterContinuationsWithSubpath = { | "notIncludes-any" | "includes-all" | "notIncludes-all", - value: readonly GetArV[] + value: InValues> + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKey" | "notHasKey", + value: GetMapK + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasValue" | "notHasValue", + value: GetMapV + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKeyValue" | "notHasKeyValue", + value: GetMapEntry + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKey-any" | "notHasKey-any" | "hasKey-all" | "notHasKey-all", + value: InValues> + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasValue-any" | "notHasValue-any" | "hasValue-all" | "notHasValue-all", + value: InValues> + ): ( + current: Query + ) => QueryWhere + < + TFieldValues extends FieldValues, + TFieldName extends FieldPath, + TFieldValuesSub extends TFieldValues[TFieldName][number], + TFieldNameSub extends FieldPath, + V extends FieldPathValue + >( + subPath: TFieldName, + restPath: TFieldNameSub, + op: "hasKeyValue-any" | "notHasKeyValue-any" | "hasKeyValue-all" | "notHasKeyValue-all", + value: InValues> ): ( current: Query ) => QueryWhere diff --git a/packages/effect-app/src/Schema/ext.ts b/packages/effect-app/src/Schema/ext.ts index 0311be652..232dbf2b0 100644 --- a/packages/effect-app/src/Schema/ext.ts +++ b/packages/effect-app/src/Schema/ext.ts @@ -82,8 +82,6 @@ export const withDefaultParseOptions = ( return (input: any, options?: SchemaAST.ParseOptions) => run(input, { ...defaultParseOptions, ...options }) }) as Decode -// TODO: v4 migration - Date is no longer by default encoded to string. - const DateString = S.String.annotate({ identifier: "DateOrInvalid", description: "an ISO 8601 date string that will be decoded as a Date (may be invalid)", @@ -107,18 +105,21 @@ export interface DateFromString extends S.decodeTo {} * Encoding: * - A `Date` is encoded as a `string`. * + * Use this when the Encoded form must be JSON (`string`). Domain models should + * prefer {@link Date}, whose Encoded form is `Date`; JSON stores convert via + * `Schema.toCodecJson`. + * * @since 4.0.0 */ export const DateFromString: DateFromString = DateString.pipe(S.decodeTo(S.Date, SchemaTransformation.dateFromString)) -/** Like the default Schema `Date` but from String, with default helpers. */ -export const Date = extendM(DateFromString, (s) => ({ +const dateHelpers = (s: S.Date) => ({ /** * Construction-only default `new Date()`. Applied only when the field is * omitted from `.make(...)` input. NOT applied during decode — cannot be * used to JIT-migrate database fields. See file-level note. */ - withConstructorDefault: s.pipe(S.withConstructorDefault(Effect.sync(() => new global.Date()))), + withConstructorDefault: s.pipe(S.withConstructorDefault(Effect.sync(() => new globalThis.Date()))), /** * Decode-time default `new Date()`. **Discouraged for persisted data:** a * missing field may be data corruption, not an old-shape document; silently @@ -126,38 +127,18 @@ export const Date = extendM(DateFromString, (s) => ({ * preferably versioned migration over a decode-time fallback. See * file-level note. */ - withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new global.Date()))) -})) - -const DateValidString = S.String.annotate({ - identifier: "Date", - description: "a valid ISO 8601 date string that will be decoded as a Date", - format: "date-time" + withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new globalThis.Date()))) }) -// Schema.Date rejects invalid Dates since beta.91+; no separate isDateValid check needed. -const DateValidFromString = DateValidString - .pipe( - S.decodeTo(S.Date, SchemaTransformation.dateFromString) - ) +/** Like the default Schema `Date` (Encoded is `Date`) with default helpers. */ +export const Date = extendM(S.Date, dateHelpers) -/** Like the default Schema `Date` (valid only) but from String, with default helpers. */ -export const DateValid = extendM(DateValidFromString, (s) => ({ - /** - * Construction-only default `new Date()`. Applied only when the field is - * omitted from `.make(...)` input. NOT applied during decode — cannot be - * used to JIT-migrate database fields. See file-level note. - */ - withConstructorDefault: s.pipe(S.withConstructorDefault(Effect.sync(() => new global.Date()))), - /** - * Decode-time default `new Date()`. **Discouraged for persisted data:** a - * missing field may be data corruption, not an old-shape document; silently - * substituting `new Date()` hides the problem. Prefer an explicit, - * preferably versioned migration over a decode-time fallback. See - * file-level note. - */ - withDecodingDefaultType: s.pipe(S.withDecodingDefaultType(Effect.sync(() => new global.Date()))) -})) +/** + * Alias of {@link Date}. Core `Schema.Date` already rejects invalid Dates. + * + * @deprecated Use {@link Date}. + */ +export const DateValid = Date /** Like the default Schema `Boolean` but with default helpers. */ export const Boolean = Object.assign(S.Boolean, { @@ -337,10 +318,10 @@ export const ReadonlyMapFromArray = (value: ValueSchema) => pipe( - ReadonlySetFromArray(value), + S.ReadonlySet(value), (s) => Object.assign(s, { /** @@ -365,13 +346,13 @@ export const ReadonlySet = (value: ValueSchema) => }) ) -/** Like the default Schema `ReadonlyMap` but from Array, with default helpers. */ +/** Like the default Schema `ReadonlyMap` (Encoded is `Map`) with default helpers. */ export const ReadonlyMap = (pair: { readonly key: KeySchema readonly value: ValueSchema }) => pipe( - ReadonlyMapFromArray(pair), + S.ReadonlyMap(pair.key, pair.value), (s) => Object.assign(s, { /** @@ -503,9 +484,9 @@ export type WithDefaults = ( // export type UnionToIntersection3 = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I // : never -/** Union of core `Schema.Date` (Date objects) and string-encoded `Date`, with default helpers. */ +/** Union of core `Schema.Date` (Date objects) and string-encoded `DateFromString`, with default helpers. */ export const inputDate = extendM( - S.Union([S.Date, Date]), + S.Union([S.Date, DateFromString]), (s) => ({ /** * Construction-only default `new Date()`. Applied only when the field is diff --git a/packages/effect-app/src/Store.ts b/packages/effect-app/src/Store.ts index eea19b6c4..3bf25fba9 100644 --- a/packages/effect-app/src/Store.ts +++ b/packages/effect-app/src/Store.ts @@ -11,7 +11,7 @@ import type { FieldPath } from "./Model/filter/types/path/index.ts" import type { AggregateIrExpression, ComputedProjectionIrExpression, RawQuery } from "./Model/query.ts" import type * as Option from "./Option.ts" import * as RequestScopedDependencies from "./RequestScopedDependencies.ts" -import { NonEmptyString255 } from "./Schema.ts" +import { NonEmptyString255, type Top as SchemaTop } from "./Schema.ts" /** * Adapter-neutral unique-key definition for stores that support unique indexes, @@ -46,6 +46,13 @@ export interface StoreConfig { * Unique indexes, mainly for CosmosDB */ uniqueKeys?: UniqueKey[] + /** + * Domain schema whose Encoded shape is stored. JSON adapters use + * `Schema.toCodecJson(Schema.toEncoded(schema))` for documents, query + * values, and defaults. Date/Map/Set (and app types such as DateOnly) + * stay native on Encoded and lower only on this JSON boundary. + */ + schema?: SchemaTop } export type SupportedValues = string | boolean | number | null diff --git a/packages/effect-app/test/schema.test.ts b/packages/effect-app/test/schema.test.ts index 976220a7a..4d5083f77 100644 --- a/packages/effect-app/test/schema.test.ts +++ b/packages/effect-app/test/schema.test.ts @@ -265,14 +265,13 @@ test("TaggedUnion match dispatches on _tag", () => { S.TaggedStruct("A", { a: S.String }), S.TaggedStruct("B", { b: S.Finite }) ]) - type T = S.Schema.Type const matcher = schema.match({ A: (v) => `got A: ${v.a}`, B: (v) => `got B: ${v.b}` }) - expect(matcher({ _tag: "A", a: "hello" } as T)).toBe("got A: hello") - expect(matcher({ _tag: "B", b: 42 } as T)).toBe("got B: 42") + expect(matcher({ _tag: "A", a: "hello" })).toBe("got A: hello") + expect(matcher({ _tag: "B", b: 42 })).toBe("got B: 42") }) test("TaggedUnion with single member", () => { @@ -319,8 +318,7 @@ test("TaggedUnion with encodeKeys renaming a non-tag key", () => { expect(decoded2).toEqual({ _tag: "B", lastName: 42 }) // encode back to snake_case - type T = S.Schema.Type - const encoded = S.encodeSync(schema)({ _tag: "A", firstName: "Alice" } as T) + const encoded = S.encodeSync(schema)({ _tag: "A", firstName: "Alice" }) expect(encoded).toEqual({ _tag: "A", first_name: "Alice" }) // guards work on decoded values @@ -387,7 +385,7 @@ describe("ReadonlySetFromArray", () => { describe("ReadonlyMapFromArray", () => { test("decodes an array of tuples to a Map", () => { - const schema = S.ReadonlyMap({ key: S.String, value: S.Finite }) + const schema = S.ReadonlyMapFromArray({ key: S.String, value: S.Finite }) const decoded = S.decodeUnknownSync(schema)([["a", 1], ["b", 2]]) expect(decoded).toEqual(new Map([["a", 1], ["b", 2]])) }) @@ -445,11 +443,18 @@ describe("ReadonlySet (with withConstructorDefault)", () => { expect(made.items).toEqual(new Set()) }) - test("decodes array with NumberFromString values", () => { + test("decodes a Set with NumberFromString values", () => { const schema = S.ReadonlySet(S.NumberFromString) - const decoded = S.decodeUnknownSync(schema)(["1", "2"]) + const decoded = S.decodeUnknownSync(schema)(new Set(["1", "2"])) expect(decoded).toEqual(new Set([1, 2])) }) + + test("Encoded is a Set, not an array", () => { + const schema = S.ReadonlySet(S.String) + const encoded = S.encodeSync(schema)(new Set(["a"])) + expect(encoded).toEqual(new Set(["a"])) + expectTypeOf(encoded).toEqualTypeOf>() + }) }) describe("ReadonlyMap (with withConstructorDefault)", () => { @@ -460,11 +465,18 @@ describe("ReadonlyMap (with withConstructorDefault)", () => { expect(made.items).toEqual(new Map()) }) - test("decodes array of tuples with NumberFromString keys", () => { + test("decodes a Map with NumberFromString keys", () => { const schema = S.ReadonlyMap({ key: S.NumberFromString, value: S.String }) - const decoded = S.decodeUnknownSync(schema)([["1", "one"]]) + const decoded = S.decodeUnknownSync(schema)(new Map([["1", "one"]])) expect(decoded).toEqual(new Map([[1, "one"]])) }) + + test("Encoded is a Map, not an array of tuples", () => { + const schema = S.ReadonlyMap({ key: S.String, value: S.Finite }) + const encoded = S.encodeSync(schema)(new Map([["a", 1]])) + expect(encoded).toEqual(new Map([["a", 1]])) + expectTypeOf(encoded).toEqualTypeOf>() + }) }) describe("JSON Schema", () => { @@ -506,8 +518,18 @@ describe("JSON Schema", () => { }) }) - test("Date has identifier DateOrInvalid and ISO 8601 description", () => { + test("Date Encoded is Date; JSON codec encodes ISO strings", () => { + const d = new Date("2024-01-01T00:00:00.000Z") + expect(S.decodeUnknownSync(S.Date)(d)).toBe(d) + expect(S.encodeSync(S.Date)(d)).toBe(d) + expect(S.encodeSync(S.toCodecJson(S.Date))(d)).toBe("2024-01-01T00:00:00.000Z") const doc = S.toJsonSchemaDocument(S.Date) + expect(doc.dialect).toBe("draft-2020-12") + expect(doc.schema).toEqual({ type: "string" }) + }) + + test("DateFromString keeps string Encoded", () => { + const doc = S.toJsonSchemaDocument(S.DateFromString) expect(doc).toStrictEqual({ dialect: "draft-2020-12", schema: { "$ref": "#/$defs/DateOrInvalid" }, @@ -521,21 +543,6 @@ describe("JSON Schema", () => { }) }) - test("DateValid has identifier Date and ISO 8601 description", () => { - const doc = S.toJsonSchemaDocument(S.DateValid) - expect(doc).toStrictEqual({ - dialect: "draft-2020-12", - schema: { "$ref": "#/$defs/Date" }, - definitions: { - Date: { - type: "string", - description: "a valid ISO 8601 date string that will be decoded as a Date", - format: "date-time" - } - } - }) - }) - test("PhoneNumber has format phone", () => { const doc = specialJsonSchemaDocument(S.PhoneNumber) expect(doc).toStrictEqual({ diff --git a/packages/infra/examples/query.ts b/packages/infra/examples/query.ts index 37d0d2652..71ef736f3 100644 --- a/packages/infra/examples/query.ts +++ b/packages/infra/examples/query.ts @@ -76,7 +76,7 @@ const program = Effect.gen(function*() { and("_tag", "Something"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z"), // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")), and("_tag", "Something") ), order("displayName"), @@ -90,7 +90,7 @@ const program = Effect.gen(function*() { and("_tag", "Something"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z"), // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")), and("_tag", "Something") ), order("displayName"), @@ -112,7 +112,7 @@ expectTypeOf(test1).toEqualTypeOf< readonly _tag: "Something" readonly id: string readonly displayName: string - readonly n: string + readonly n: Date readonly union: { readonly _tag: "string" readonly value: string @@ -129,7 +129,7 @@ expectTypeOf(testneq1).toEqualTypeOf< readonly _tag: "Something" readonly id: string readonly displayName: string - readonly n: string + readonly n: Date readonly union: { readonly _tag: "number" readonly value: number diff --git a/packages/infra/package.json b/packages/infra/package.json index 5906dcd82..dcc58d1b8 100644 --- a/packages/infra/package.json +++ b/packages/infra/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@faker-js/faker": "^8.4.1", + "@sentry/node": "10.55.0", "effect-app": "workspace:*", "fast-check": "^4.9.0", "jose": "^6.2.3", @@ -21,7 +22,6 @@ "@azure/cosmos": "^4.9.3", "@azure/service-bus": "^7.9.5", "@effect/sql-sqlite-node": "4.0.0-beta.107", - "@sentry/node": "10.55.0", "@sentry/opentelemetry": "10.55.0", "@types/better-sqlite3": "^7.6.13", "@types/node": "25.9.1", diff --git a/packages/infra/src/Store/Cosmos.ts b/packages/infra/src/Store/Cosmos.ts index fbde39270..c47f9f0af 100644 --- a/packages/infra/src/Store/Cosmos.ts +++ b/packages/infra/src/Store/Cosmos.ts @@ -19,6 +19,8 @@ import { DatabaseError, OptimisticConcurrencyException } from "../errors.ts" import { InfraLogger } from "../logger.ts" import { annotateCosmosResponse, annotateDb } from "../otel.ts" import { buildWhereCosmosQuery3, logQuery } from "./Cosmos/query.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" +import { makeJsonLower } from "./utils.ts" const makeMapId = (idKey: IdKey) => ({ [idKey]: id, ...e }: Encoded) => ({ @@ -96,6 +98,10 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) { const mapId = makeMapId(idKey) const mapReverseId = makeReverseMapId(idKey) + const codec = makeJsonDocumentCodec(config?.schema) + const json = makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial + const fromStored = (raw: Encoded) => codec.decode({ ...defaultValues, ...mapReverseId(raw as any) }) type PM = PersistenceModelType type PMCosmos = PersistenceModelType & { id: string }> const containerId = `${prefix}${name}` @@ -128,7 +134,6 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { return namespace })) - const defaultValues = config?.defaultValues ?? {} const container = db.container(containerId) const bulk = container.items.bulk.bind(container.items) const execBatch = container.items.batch.bind(container.items) @@ -205,7 +210,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { dropUndefinedT({ operationType: "Create" as const, resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) } @@ -217,7 +222,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { operationType: "Replace" as const, id: x[idKey], resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) }, @@ -314,7 +319,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { onNone: () => ({ operationType: "Create" as const, resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) } @@ -325,7 +330,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { operationType: "Replace" as const, id: x[idKey], resourceBody: { - ...Struct.omit(x, ["_etag", idKey]), + ...Struct.omit(codec.encode(x), ["_etag", idKey]), id: x[idKey], _partitionKey: nsPartitionValue(ns, x) }, @@ -447,7 +452,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { container.items.query(q, { partitionKey: nsBasePartitionKey(ns) }).fetchAll() ) yield* annotateFeed(response) - return response.resources.map((_) => ({ ...defaultValues, ...mapReverseId(_) })) + return response.resources.map((_) => fromStored(_ as unknown as Encoded)) }) .pipe( annotateDb({ @@ -492,7 +497,8 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { | undefined, f.order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined, skip, - limit + limit, + json ) ), ns: resolveNamespace @@ -520,7 +526,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { container.items.query<{ f: M }>(q, { partitionKey: nsBasePartitionKey(ns) }).fetchAll() ) yield* annotateFeed(response) - return response.resources.map(({ f }) => ({ ...defaultValues, ...mapReverseId(f as any) }) as any) + return response.resources.map(({ f }) => fromStored(f as Encoded) as any) }) .pipe( annotateDb({ @@ -546,7 +552,7 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) yield* annotateItem(response) return Option.fromNullishOr(response.resource).pipe( - Option.map((_) => ({ ...defaultValues, ...mapReverseId(_) })) + Option.map((_) => fromStored(_)) ) }) .pipe(annotateDb({ @@ -573,12 +579,12 @@ const makeCosmosStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { { onNone: () => container.items.create({ - ...mapId(e), + ...mapId(codec.encode(e)), _partitionKey: nsPartitionValue(ns, e) }), onSome: (eTag) => container.item(e[idKey], nsPartitionValue(ns, e)).replace( - { ...mapId(e), _partitionKey: nsPartitionValue(ns, e) }, + { ...mapId(codec.encode(e)), _partitionKey: nsPartitionValue(ns, e) }, { accessCondition: { type: "IfMatch", diff --git a/packages/infra/src/Store/Cosmos/query.ts b/packages/infra/src/Store/Cosmos/query.ts index ebcc37983..266e0a821 100644 --- a/packages/infra/src/Store/Cosmos/query.ts +++ b/packages/infra/src/Store/Cosmos/query.ts @@ -9,6 +9,7 @@ import type { SupportedValues } from "effect-app/Store" import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.ts" import { isRelationCheck } from "../codeFilter.ts" +import { jsonifyFilter, type JsonLower, toJsonQueryValue } from "../utils.ts" export function logQuery(q: { query: string @@ -60,8 +61,12 @@ export function buildWhereCosmosQuery3( >, order?: NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }>, skip?: number, - limit?: number + limit?: number, + json?: JsonLower ) { + const toJson = json?.toJson ?? toJsonQueryValue + filter = (json?.jsonifyFilter ?? jsonifyFilter)(filter) + defaultValues = toJson(defaultValues) as Record const statement = (x: FilterR, i: number) => { if (x.path === idKey) { x = { ...x, path: "id" } @@ -89,23 +94,56 @@ export function buildWhereCosmosQuery3( return `(NOT ARRAY_CONTAINS(${k}, ${v}))` case "includes-any": - return `ARRAY_CONTAINS_ANY(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") - })` + return `ARRAY_CONTAINS_ANY(${k}, ${(x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ")})` case "notIncludes-any": return `(NOT ARRAY_CONTAINS_ANY(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") + (x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") }))` case "includes-all": - return `ARRAY_CONTAINS_ALL(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") - })` + return `ARRAY_CONTAINS_ALL(${k}, ${(x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ")})` case "notIncludes-all": return `(NOT ARRAY_CONTAINS_ALL(${k}, ${ - (x.value as unknown as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") + (x.value as readonly unknown[]).map((_, i) => `${v}__${i}`).join(", ") }))` + case "hasKey": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = ${v})` + case "notHasKey": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = ${v}))` + case "hasValue": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = ${v})` + case "notHasValue": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = ${v}))` + case "hasKeyValue": + return `ARRAY_CONTAINS(${k}, ${v})` + case "notHasKeyValue": + return `(NOT ARRAY_CONTAINS(${k}, ${v}))` + case "hasKey-any": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[0]))` + case "notHasKey-any": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[0])))` + case "hasKey-all": + return `(NOT EXISTS(SELECT VALUE key FROM key IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = key)))` + case "notHasKey-all": + return `EXISTS(SELECT VALUE key FROM key IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[0] = key))` + case "hasValue-any": + return `EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[1]))` + case "notHasValue-any": + return `(NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE ARRAY_CONTAINS(${v}, p[1])))` + case "hasValue-all": + return `(NOT EXISTS(SELECT VALUE val FROM val IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = val)))` + case "notHasValue-all": + return `EXISTS(SELECT VALUE val FROM val IN ${v} WHERE NOT EXISTS(SELECT VALUE p FROM p IN ${k} WHERE p[1] = val))` + case "hasKeyValue-any": + return `EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE ARRAY_CONTAINS(${k}, pair))` + case "notHasKeyValue-any": + return `(NOT EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE ARRAY_CONTAINS(${k}, pair)))` + case "hasKeyValue-all": + return `(NOT EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE NOT ARRAY_CONTAINS(${k}, pair)))` + case "notHasKeyValue-all": + return `EXISTS(SELECT VALUE pair FROM pair IN ${v} WHERE NOT ARRAY_CONTAINS(${k}, pair))` + case "contains": return `CONTAINS(${k}, ${v}, true)` @@ -166,6 +204,24 @@ export function buildWhereCosmosQuery3( "notIncludes-any": "includes-any", "includes-all": "notIncludes-all", "notIncludes-all": "includes-all", + hasKey: "notHasKey", + notHasKey: "hasKey", + hasValue: "notHasValue", + notHasValue: "hasValue", + hasKeyValue: "notHasKeyValue", + notHasKeyValue: "hasKeyValue", + "hasKey-any": "notHasKey-any", + "notHasKey-any": "hasKey-any", + "hasKey-all": "notHasKey-all", + "notHasKey-all": "hasKey-all", + "hasValue-any": "notHasValue-any", + "notHasValue-any": "hasValue-any", + "hasValue-all": "notHasValue-all", + "notHasValue-all": "hasValue-all", + "hasKeyValue-any": "notHasKeyValue-any", + "notHasKeyValue-any": "hasKeyValue-any", + "hasKeyValue-all": "notHasKeyValue-all", + "notHasKeyValue-all": "hasKeyValue-all", in: "notIn", notIn: "in" } satisfies Record diff --git a/packages/infra/src/Store/Disk.ts b/packages/infra/src/Store/Disk.ts index 0d5bf3f05..1be559dec 100644 --- a/packages/infra/src/Store/Disk.ts +++ b/packages/infra/src/Store/Disk.ts @@ -10,7 +10,9 @@ import * as Console from "effect/Console" import { flow } from "effect/Function" import * as Semaphore from "effect/Semaphore" import { annotateDb } from "../otel.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { makeMemoryStoreInt } from "./Memory.ts" +import { type JsonLower, makeJsonLower } from "./utils.ts" function makeDiskStoreInt( prefix: string, @@ -19,9 +21,12 @@ function makeDiskStoreInt, E, R>, - defaultValues?: Partial + defaultValues?: Partial, + schema?: StoreConfig["schema"], + json?: JsonLower ) { type PM = PersistenceModelType + const codec = makeJsonDocumentCodec(schema) return Effect.gen(function*() { if (namespace !== "primary") { dir = dir + "/" + namespace @@ -44,7 +49,7 @@ function makeDiskStoreInt - Effect.sync(() => JSON.parse(x) as PM[]).pipe( + Effect.sync(() => (JSON.parse(x) as PM[]).map((row) => codec.decode(row))).pipe( annotateDb({ operation: "read.parse", system: "disk", @@ -67,7 +72,7 @@ function makeDiskStoreInt) => Effect - .sync(() => JSON.stringify([...v], undefined, 2)) + .sync(() => JSON.stringify([...v].map((row) => codec.encode(row)), undefined, 2)) .pipe( annotateDb({ operation: "stringify", @@ -117,7 +122,9 @@ function makeDiskStoreInt, E, R>, config?: StoreConfig ) { - const primary = yield* makeDiskStoreInt(prefix, idKey, "primary", dir, name, seed, config?.defaultValues).pipe( - Effect.orDie + const json = makeJsonLower(config) + const primary = yield* makeDiskStoreInt( + prefix, + idKey, + "primary", + dir, + name, + seed, + config?.defaultValues, + config?.schema, + json ) + .pipe( + Effect.orDie + ) const stores = new Map>([["primary", primary]]) const ctx = yield* Effect.context() const semaphores = new Map() @@ -204,7 +223,9 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) { dir, name, seed, - config?.defaultValues + config?.defaultValues, + config?.schema, + json ) .pipe( Effect.orDie, diff --git a/packages/infra/src/Store/Memory.ts b/packages/infra/src/Store/Memory.ts index 059488d41..ffbabef80 100644 --- a/packages/infra/src/Store/Memory.ts +++ b/packages/infra/src/Store/Memory.ts @@ -18,7 +18,8 @@ import * as Struct from "effect/Struct" import { InfraLogger } from "../logger.ts" import { annotateDb } from "../otel.ts" import { codeFilter, codeFilter3_ } from "./codeFilter.ts" -import { get, makeUpdateETag } from "./utils.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" +import { get, jsonifyFilter, type JsonLower, makeJsonLower, makeUpdateETag, toJsonQueryValue } from "./utils.ts" export { get } from "./utils.ts" @@ -332,25 +333,38 @@ export function makeMemoryStoreInt, E, R>, - _defaultValues?: Partial + _defaultValues?: Partial, + schema?: StoreConfig["schema"], + json?: JsonLower ) { type PM = PersistenceModelType return Effect.gen(function*() { const updateETag = makeUpdateETag(modelName) + const codec = makeJsonDocumentCodec(schema) + const encodeDoc = (e: Encoded | PM): PM => codec.encode({ _etag: undefined, ...e }) + const decodeDoc = (e: PM): PM => codec.decode(e) const items_ = yield* seed ?? Effect.sync(() => []) - const defaultValues = _defaultValues ?? {} + const toJson = json?.toJson ?? toJsonQueryValue + const lowerFilter = json?.jsonifyFilter ?? jsonifyFilter + const encodedDefaults = toJson(_defaultValues ?? {}) as Partial - const items = new Map([...items_].map((_) => [_[idKey], { _etag: undefined, ...defaultValues, ..._ }] as const)) + const items = new Map( + [...items_].map((_) => { + const encoded = encodeDoc({ ...encodedDefaults, ..._ }) + return [encoded[idKey], encoded] as const + }) + ) const store = Ref.makeUnsafe>(items) const sem = Semaphore.makeUnsafe(1) const withPermit = sem.withPermits(1) const values = Effect.map(Ref.get(store), (s) => s.values()) - const all = Effect.map(values, Array.fromIterable) + const allStored = Effect.map(values, Array.fromIterable) + const all = Effect.map(allStored, (rows) => rows.map(decodeDoc)) const batchSet = (items: NonEmptyReadonlyArray) => Effect - .forEach(items, (i) => Effect.flatMap(s.find(i[idKey]), (current) => updateETag(i, idKey, current))) + .forEach(items, (i) => Effect.flatMap(s.find(i[idKey]), (current) => updateETag(encodeDoc(i), idKey, current))) .pipe( Effect .tap((items) => @@ -368,7 +382,7 @@ export function makeMemoryStoreInt _), + .map((items) => items.map(decodeDoc) as unknown as NonEmptyReadonlyArray), withPermit ) @@ -414,7 +428,7 @@ export function makeMemoryStoreInt Option.fromNullishOr(_.get(id))), + Effect.map((_) => Option.fromNullishOr(_.get(id)).pipe(Option.map(decodeDoc))), annotateDb({ operation: "find", system: "memory", @@ -424,11 +438,16 @@ export function makeMemoryStoreInt - all + filter: (f: FilterArgs) => + allStored .pipe( - Effect.tap(() => logQuery(f, defaultValues)), - Effect.map(memFilter(f)), + Effect.tap(() => logQuery(f, encodedDefaults)), + Effect.map(memFilter({ ...f, filter: f.filter ? lowerFilter(f.filter) : f.filter })), + Effect.map((rows): (U extends undefined ? Encoded : Pick)[] => + f.select + ? rows as (U extends undefined ? Encoded : Pick)[] + : rows.map(decodeDoc) as (U extends undefined ? Encoded : Pick)[] + ), annotateDb({ operation: "filter", system: "memory", @@ -441,14 +460,15 @@ export function makeMemoryStoreInt updateETag(e, idKey, current)), + Effect.flatMap((current) => updateETag(encodeDoc(e), idKey, current.pipe(Option.map(encodeDoc)))), Effect - .tap((e) => + .tap((stored) => Ref.get(store).pipe( - Effect.map((_) => new Map([..._, [e[idKey], e]])), + Effect.map((_) => new Map([..._, [stored[idKey], stored]])), Effect.flatMap((_) => Ref.set(store, _)) ) ), + Effect.map(decodeDoc), withPermit, annotateDb({ operation: "set", @@ -518,12 +538,15 @@ export const makeMemoryStore = () => ({ seed?: Effect.Effect, E, R>, config?: StoreConfig ) { + const json = makeJsonLower(config) const primary = yield* makeMemoryStoreInt( modelName, idKey, "primary", seed, - config?.defaultValues + config?.defaultValues, + config?.schema, + json ) const ctx = yield* Effect.context() const stores = new Map([["primary", primary]]) @@ -543,7 +566,7 @@ export const makeMemoryStore = () => ({ if (config?.allowNamespace && !config.allowNamespace(namespace)) { throw new Error(`Namespace ${namespace} not allowed!`) } - return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues) + return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues, config?.schema, json) .pipe( Effect.orDie, Effect.provide(ctx), diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index f612a2e50..2ea54f8b2 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -15,8 +15,9 @@ import { SqlClient } from "effect/unstable/sql" import { DatabaseError, OptimisticConcurrencyException } from "../errors.ts" import { InfraLogger } from "../logger.ts" import { annotateDb, type DbSystem } from "../otel.ts" +import { makeJsonDocumentCodec } from "./jsonDocument.ts" import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.ts" -import { makeETag } from "./utils.ts" +import { makeETag, makeJsonLower, toJsonQueryValue } from "./utils.ts" const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e) const sqlIsTransient = (e: unknown) => @@ -46,10 +47,14 @@ export class WithNsTransaction export const parseRow = ( row: { id: string; _etag: string | null; data: string }, idKey: PropertyKey, - defaultValues: Partial + defaultValues: Partial, + decode: (doc: PersistenceModelType) => PersistenceModelType = (doc) => doc ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object - return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + const jsonDefaults = toJsonQueryValue(defaultValues) as Partial + return decode( + { ...jsonDefaults, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + ) } const parseSelectRow = ( @@ -86,7 +91,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} + const json = makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial + const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -112,11 +119,11 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: ) const toRow = (e: PM) => { - const newE = makeETag(e) + const newE = makeETag(codec.encode(e)) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + return { id, _etag: newE._etag!, data, item: codec.decode(newE) } } const exec = (query: string, params?: readonly unknown[]) => @@ -209,7 +216,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = ?` return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, codec.decode)) + ), annotateDb({ operation: "all", system, @@ -231,7 +240,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, codec.decode)) : Option.none() }), annotateDb({ @@ -281,7 +290,8 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: .skip, f .limit, - ns + ns, + json ) }) .pipe( @@ -303,7 +313,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, codec.decode) as any as M + ) }) ) ), @@ -418,7 +430,9 @@ function makeSQLiteStorePerNs( ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} + const json = makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial + const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -430,11 +444,11 @@ function makeSQLiteStorePerNs( })) const toRow = (e: PM) => { - const newE = makeETag(e) + const newE = makeETag(codec.encode(e)) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + return { id, _etag: newE._etag!, data, item: codec.decode(newE) } } const exec = (ns: string, query: string, params?: readonly unknown[]) => @@ -549,7 +563,9 @@ function makeSQLiteStorePerNs( const sqlText = `SELECT id, _etag, data FROM "${tableName}"` return exec(ns, sqlText) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, codec.decode)) + ), annotateDb({ operation: "all", system: "sqlite", @@ -570,7 +586,7 @@ function makeSQLiteStorePerNs( Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, codec.decode)) : Option.none() }), annotateDb({ @@ -619,7 +635,9 @@ function makeSQLiteStorePerNs( f .skip, f - .limit + .limit, + undefined, + json ) ) .pipe( @@ -641,7 +659,9 @@ function makeSQLiteStorePerNs( } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, codec.decode) as any as M + ) }) ) ), diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index 4fffa4e8c..8d7c4cad2 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -12,7 +12,8 @@ import { SqlClient } from "effect/unstable/sql" import { DatabaseError, OptimisticConcurrencyException } from "../../errors.ts" import { InfraLogger } from "../../logger.ts" import { annotateDb } from "../../otel.ts" -import { makeETag } from "../utils.ts" +import { makeJsonDocumentCodec } from "../jsonDocument.ts" +import { makeETag, makeJsonLower, toJsonQueryValue } from "../utils.ts" import { buildWhereSQLQuery, logQuery, pgDialect } from "./query.ts" const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e) @@ -36,10 +37,14 @@ const preserveStoreError = (e: unknown): DatabaseError | OptimisticConcurrencyEx const parseRow = ( row: { id: string; _etag: string | null; data: unknown }, idKey: PropertyKey, - defaultValues: Partial + defaultValues: Partial, + decode: (doc: PersistenceModelType) => PersistenceModelType = (doc) => doc ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object - return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + const jsonDefaults = toJsonQueryValue(defaultValues) as Partial + return decode( + { ...jsonDefaults, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + ) } const parseSelectRow = ( @@ -70,7 +75,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) { type PM = PersistenceModelType const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} + const json = makeJsonLower(config) + const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial + const codec = makeJsonDocumentCodec(config?.schema) const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -96,11 +103,11 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { ) const toRow = (e: PM) => { - const newE = makeETag(e) + const newE = makeETag(codec.encode(e)) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + return { id, _etag: newE._etag!, data, item: codec.decode(newE) } } const exec = (query: string, params?: readonly unknown[]) => @@ -193,7 +200,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = $1` return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, codec.decode)) + ), annotateDb({ operation: "all", system: "postgresql", @@ -215,7 +224,7 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, codec.decode)) : Option.none() }), annotateDb({ @@ -256,7 +265,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { | undefined, f.order, f.skip, - f.limit + f.limit, + ns, + json ) const nsPlaceholder = pgDialect.placeholder(q.params.length + 1) const hasWhere = q.sql.includes("WHERE") @@ -286,7 +297,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, codec.decode) as any as M + ) }) ) ), diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index f6790152f..c41dc6abe 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -6,6 +6,7 @@ import type { AggregateIrExpression, ComputedProjectionIrExpression, ComputedPro import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.ts" import { isRelationCheck } from "../codeFilter.ts" +import { jsonifyFilter, type JsonLower, toJsonQueryValue } from "../utils.ts" export interface SQLDialect { readonly jsonExtract: (path: string) => string @@ -17,6 +18,9 @@ export interface SQLDialect { readonly jsonArrayNotContainsAny: (arrPath: string, valPlaceholders: readonly string[]) => string readonly jsonArrayContainsAll: (arrPath: string, valPlaceholders: readonly string[]) => string readonly jsonArrayNotContainsAll: (arrPath: string, valPlaceholders: readonly string[]) => string + readonly jsonMapHasKey: (arrPath: string, valPlaceholder: string) => string + readonly jsonMapHasValue: (arrPath: string, valPlaceholder: string) => string + readonly jsonMapHasPair: (arrPath: string, valPlaceholder: string) => string readonly caseInsensitiveLike: (expr: string, valPlaceholder: string) => string readonly caseInsensitiveNotLike: (expr: string, valPlaceholder: string) => string readonly jsonColumnType: "JSON" | "JSONB" @@ -45,6 +49,11 @@ export const sqliteDialect: SQLDialect = { `NOT (${ vals.map((v) => `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE value = ${v})`).join(" AND ") })`, + jsonMapHasKey: (arrPath, val) => + `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE json_extract(value, '$[0]') = ${val})`, + jsonMapHasValue: (arrPath, val) => + `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE json_extract(value, '$[1]') = ${val})`, + jsonMapHasPair: (arrPath, val) => `EXISTS(SELECT 1 FROM json_each(data, '$.${arrPath}') WHERE value = ${val})`, caseInsensitiveLike: (expr, val) => `LOWER(${expr}) LIKE LOWER(${val})`, caseInsensitiveNotLike: (expr, val) => `LOWER(${expr}) NOT LIKE LOWER(${val})`, jsonColumnType: "JSON", @@ -112,6 +121,27 @@ export const pgDialect: SQLDialect = { : `data${parts.map((p) => `->'${p}'`).join("")}` return `NOT (${vals.map((v) => `${jsonPath} @> ${v}::jsonb`).join(" AND ")})` }, + jsonMapHasKey: (arrPath, val) => { + const parts = arrPath.split(".") + const jsonPath = parts.length === 1 + ? `data->'${parts[0]}'` + : `data${parts.map((p) => `->'${p}'`).join("")}` + return `EXISTS(SELECT 1 FROM jsonb_array_elements(${jsonPath}) e WHERE e->0 = ${val}::jsonb)` + }, + jsonMapHasValue: (arrPath, val) => { + const parts = arrPath.split(".") + const jsonPath = parts.length === 1 + ? `data->'${parts[0]}'` + : `data${parts.map((p) => `->'${p}'`).join("")}` + return `EXISTS(SELECT 1 FROM jsonb_array_elements(${jsonPath}) e WHERE e->1 = ${val}::jsonb)` + }, + jsonMapHasPair: (arrPath, val) => { + const parts = arrPath.split(".") + const jsonPath = parts.length === 1 + ? `data->'${parts[0]}'` + : `data${parts.map((p) => `->'${p}'`).join("")}` + return `${jsonPath} @> jsonb_build_array(${val}::jsonb)` + }, caseInsensitiveLike: (expr, val) => `${expr} ILIKE ${val}`, caseInsensitiveNotLike: (expr, val) => `${expr} NOT ILIKE ${val}`, jsonColumnType: "JSONB", @@ -149,6 +179,14 @@ const dottedToJsonPath = (path: string) => .filter((p) => p !== "-1") .join(".") +const mapItemParam = (dialect: SQLDialect, value: unknown) => + dialect.jsonColumnType === "JSONB" ? dialect.serializeJsonValue(value) : value + +const mapPairParam = (dialect: SQLDialect, value: unknown) => { + const encoded = dialect.serializeJsonValue(value) + return typeof encoded === "string" ? encoded : JSON.stringify(value) +} + const sqlStringLiteral = (value: string) => `'${value.replaceAll("'", "''")}'` export function buildWhereSQLQuery( @@ -175,8 +213,12 @@ export function buildWhereSQLQuery( order?: NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }>, skip?: number, limit?: number, - namespace?: string + namespace?: string, + json?: JsonLower ) { + const toJson = json?.toJson ?? toJsonQueryValue + filter = (json?.jsonifyFilter ?? jsonifyFilter)(filter) + defaultValues = toJson(defaultValues) as Record const params: unknown[] = [] let paramIndex = 1 @@ -214,7 +256,7 @@ export function buildWhereSQLQuery( switch (x.op) { case "in": { - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const hasNull = vals.some((v) => v == null) const nonNullVals = vals.filter((v) => v != null) const parts: string[] = [] @@ -226,7 +268,7 @@ export function buildWhereSQLQuery( return parts.length > 1 ? `(${parts.join(" OR ")})` : parts[0] ?? "1=0" } case "notIn": { - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const hasNull = vals.some((v) => v == null) const nonNullVals = vals.filter((v) => v != null) const parts: string[] = [] @@ -251,30 +293,133 @@ export function buildWhereSQLQuery( case "includes-any": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayContainsAny(arrPath, placeholders) } case "notIncludes-any": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayNotContainsAny(arrPath, placeholders) } case "includes-all": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayContainsAll(arrPath, placeholders) } case "notIncludes-all": { const arrPath = dottedToJsonPath(resolvedPath) - const vals = x.value as unknown as readonly unknown[] + const vals = x.value as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) return dialect.jsonArrayNotContainsAll(arrPath, placeholders) } + case "hasKey": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return dialect.jsonMapHasKey(arrPath, v) + } + case "notHasKey": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return `NOT (${dialect.jsonMapHasKey(arrPath, v)})` + } + case "hasValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return dialect.jsonMapHasValue(arrPath, v) + } + case "notHasValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapItemParam(dialect, x.value)) + return `NOT (${dialect.jsonMapHasValue(arrPath, v)})` + } + case "hasKeyValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapPairParam(dialect, x.value)) + return dialect.jsonMapHasPair(arrPath, v) + } + case "notHasKeyValue": { + const arrPath = dottedToJsonPath(resolvedPath) + const v = addParam(mapPairParam(dialect, x.value)) + return `NOT (${dialect.jsonMapHasPair(arrPath, v)})` + } + case "hasKey-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))) + return `(${parts.join(" OR ")})` + } + case "notHasKey-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))) + return `NOT (${parts.join(" OR ")})` + } + case "hasKey-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + } + case "notHasKey-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return `NOT (${ + vals.map((val) => dialect.jsonMapHasKey(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + })` + } + case "hasValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))) + return `(${parts.join(" OR ")})` + } + case "notHasValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))) + return `NOT (${parts.join(" OR ")})` + } + case "hasValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + } + case "notHasValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return `NOT (${ + vals.map((val) => dialect.jsonMapHasValue(arrPath, addParam(mapItemParam(dialect, val)))).join(" AND ") + })` + } + case "hasKeyValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))) + return `(${parts.join(" OR ")})` + } + case "notHasKeyValue-any": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + const parts = vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))) + return `NOT (${parts.join(" OR ")})` + } + case "hasKeyValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))).join(" AND ") + } + case "notHasKeyValue-all": { + const arrPath = dottedToJsonPath(resolvedPath) + const vals = x.value as readonly unknown[] + return `NOT (${ + vals.map((val) => dialect.jsonMapHasPair(arrPath, addParam(mapPairParam(dialect, val)))).join(" AND ") + })` + } + case "contains": { const v = addParam(`%${x.value}%`) return dialect.caseInsensitiveLike(k, v) diff --git a/packages/infra/src/Store/codeFilter.ts b/packages/infra/src/Store/codeFilter.ts index 0b270f544..5de521c4b 100644 --- a/packages/infra/src/Store/codeFilter.ts +++ b/packages/infra/src/Store/codeFilter.ts @@ -6,54 +6,102 @@ import type { FieldValues } from "effect-app/Model/filter/types" import * as Option from "effect-app/Option" import type { Filter } from "effect-app/Store" import { assertUnreachable } from "effect-app/utils" -import { compare, get, greaterThan, greaterThanExclusive, lowerThan, lowerThanExclusive } from "./utils.ts" +import { compare, get, greaterThan, greaterThanExclusive, lowerThan, lowerThanExclusive, toJsonQueryValue } from "./utils.ts" -const vAsArr = (v: string) => v as unknown as any[] +const vAsArr = (v: unknown) => toJsonQueryValue(v) as any[] + +const mapEntries = (value: unknown): readonly [unknown, unknown][] => { + const json = toJsonQueryValue(value) + if (!Array.isArray(json)) return [] + return json.filter((entry): entry is [unknown, unknown] => Array.isArray(entry) && entry.length >= 2) +} + +const pairEq = (entry: readonly [unknown, unknown], pair: unknown) => { + const json = toJsonQueryValue(pair) + return Array.isArray(json) && json.length >= 2 && compare(entry[0], json[0]) && compare(entry[1], json[1]) +} const filterStatement = (x: any, p: FilterR) => { - const k = get(x, p.path) + const k = toJsonQueryValue(get(x, p.path)) + const v = toJsonQueryValue(p.value) switch (p.op) { case "in": - return p.value.includes(k) + return (v as unknown[]).includes(k) case "notIn": - return !p.value.includes(k) + return !(v as unknown[]).includes(k) case "lt": - return lowerThan(k, p.value) + return lowerThan(k as any, v as any) case "lte": - return lowerThanExclusive(k, p.value) + return lowerThanExclusive(k as any, v as any) case "gt": - return greaterThan(k, p.value) + return greaterThan(k as any, v as any) case "gte": - return greaterThanExclusive(k, p.value) + return greaterThanExclusive(k as any, v as any) case "includes": - return (k as Array).includes(p.value) + return (k as Array).includes(v) case "notIncludes": - return !(k as Array).includes(p.value) + return !(k as Array).includes(v) case "includes-any": - return (vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) + return (vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) case "notIncludes-any": - return !(vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) + return !(vAsArr(p.value)).some((_) => (k as Array)?.includes(_)) case "includes-all": - return (vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) + return (vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) case "notIncludes-all": - return !(vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) + return !(vAsArr(p.value)).every((_) => (k as Array)?.includes(_)) + case "hasKey": + return mapEntries(k).some(([key]) => compare(key, v)) + case "notHasKey": + return !mapEntries(k).some(([key]) => compare(key, v)) + case "hasValue": + return mapEntries(k).some(([, val]) => compare(val, v)) + case "notHasValue": + return !mapEntries(k).some(([, val]) => compare(val, v)) + case "hasKeyValue": + return mapEntries(k).some((entry) => pairEq(entry, v)) + case "notHasKeyValue": + return !mapEntries(k).some((entry) => pairEq(entry, v)) + case "hasKey-any": + return vAsArr(p.value).some((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "notHasKey-any": + return !vAsArr(p.value).some((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "hasKey-all": + return vAsArr(p.value).every((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "notHasKey-all": + return !vAsArr(p.value).every((key) => mapEntries(k).some(([k0]) => compare(k0, key))) + case "hasValue-any": + return vAsArr(p.value).some((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "notHasValue-any": + return !vAsArr(p.value).some((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "hasValue-all": + return vAsArr(p.value).every((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "notHasValue-all": + return !vAsArr(p.value).every((val) => mapEntries(k).some(([, v0]) => compare(v0, val))) + case "hasKeyValue-any": + return vAsArr(p.value).some((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) + case "notHasKeyValue-any": + return !vAsArr(p.value).some((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) + case "hasKeyValue-all": + return vAsArr(p.value).every((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) + case "notHasKeyValue-all": + return !vAsArr(p.value).every((pair) => mapEntries(k).some((entry) => pairEq(entry, pair))) case "contains": - return (k as string).toLowerCase().includes(p.value.toLowerCase()) + return (k as string).toLowerCase().includes((v as string).toLowerCase()) case "endsWith": - return (k as string).toLowerCase().endsWith(p.value.toLowerCase()) + return (k as string).toLowerCase().endsWith((v as string).toLowerCase()) case "startsWith": - return (k as string).toLowerCase().startsWith(p.value.toLowerCase()) + return (k as string).toLowerCase().startsWith((v as string).toLowerCase()) case "notContains": - return !(k as string).toLowerCase().includes(p.value.toLowerCase()) + return !(k as string).toLowerCase().includes((v as string).toLowerCase()) case "notEndsWith": - return !(k as string).toLowerCase().endsWith(p.value.toLowerCase()) + return !(k as string).toLowerCase().endsWith((v as string).toLowerCase()) case "notStartsWith": - return !(k as string).toLowerCase().startsWith(p.value.toLowerCase()) + return !(k as string).toLowerCase().startsWith((v as string).toLowerCase()) case "neq": - return !compare(k, p.value) + return !compare(k, v) case "eq": case undefined: - return compare(k, p.value) + return compare(k, v) default: { return assertUnreachable(p.op) } diff --git a/packages/infra/src/Store/jsonDocument.ts b/packages/infra/src/Store/jsonDocument.ts new file mode 100644 index 000000000..d0302208b --- /dev/null +++ b/packages/infra/src/Store/jsonDocument.ts @@ -0,0 +1,43 @@ +import type { FieldValues } from "effect-app/Model/filter/types" +import * as S from "effect-app/Schema" +import type { PersistenceModelType } from "effect-app/Store" +import { toJsonQueryValue } from "./utils.ts" + +export interface JsonDocumentCodec { + readonly encode: (doc: PersistenceModelType) => PersistenceModelType + readonly decode: (doc: PersistenceModelType) => PersistenceModelType +} + +const splitEtag = (doc: PersistenceModelType) => { + const { _etag, ...rest } = doc + return { rest: rest as E, _etag } +} + +const joinEtag = ( + rest: E, + _etag: string | undefined +): PersistenceModelType => (_etag === undefined ? rest : { ...rest, _etag }) + +/** + * Encoded document ↔ JSON document. Prefer `Schema.toCodecJson(toEncoded(schema))` + * when the store has a schema; otherwise lower Date/Map/Set structurally. + */ +export const makeJsonDocumentCodec = (schema?: S.Top): JsonDocumentCodec => { + if (schema) { + const codec = S.toCodecJson(S.toEncoded(schema)) as S.Codec + return { + encode: (doc) => { + const { rest, _etag } = splitEtag(doc) + return joinEtag(S.encodeSync(codec)(rest) as E, _etag) + }, + decode: (doc) => { + const { rest, _etag } = splitEtag(doc) + return joinEtag(S.decodeSync(codec)(rest as S.Json), _etag) + } + } + } + return { + encode: (doc) => toJsonQueryValue(doc) as PersistenceModelType, + decode: (doc) => doc + } +} diff --git a/packages/infra/src/Store/utils.ts b/packages/infra/src/Store/utils.ts index f1505adef..8b6ae7da9 100644 --- a/packages/infra/src/Store/utils.ts +++ b/packages/infra/src/Store/utils.ts @@ -1,9 +1,279 @@ import crypto from "crypto" import * as Effect from "effect-app/Effect" +import type { FilterResult, Ops } from "effect-app/Model/filter/filterApi" import * as Option from "effect-app/Option" +import * as S from "effect-app/Schema" +import * as SchemaAST from "effect-app/SchemaAST" import type { PersistenceModelType, SupportedValues2 } from "effect-app/Store" import { OptimisticConcurrencyException } from "../errors.ts" +const dateJson = S.toCodecJson(S.Date) + +const isPlainObject = (value: unknown): value is Record => { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false + if (value instanceof Date || value instanceof Map || value instanceof Set) return false + const proto = Object.getPrototypeOf(value) + return proto === Object.prototype || proto === null +} + +const unwrapAst = (ast: SchemaAST.AST): SchemaAST.AST => SchemaAST.isSuspend(ast) ? unwrapAst(ast.thunk()) : ast + +const unionAst = (hits: readonly SchemaAST.AST[]): SchemaAST.AST | undefined => { + if (hits.length === 0) return undefined + if (hits.length === 1) return hits[0] + return S.Union(hits.map((hit) => S.make(hit)) as [S.Top, S.Top, ...Array]).ast +} + +const encodedObjects = (ast: SchemaAST.AST): SchemaAST.Objects | undefined => { + const current = unwrapAst(ast) + if (SchemaAST.isObjects(current)) return current + if (SchemaAST.isDeclaration(current)) { + const encoded = unwrapAst(SchemaAST.toEncoded(current)) + if (SchemaAST.isObjects(encoded)) return encoded + } + return undefined +} + +const astAtPath = (ast: SchemaAST.AST | undefined, path: readonly string[]): SchemaAST.AST | undefined => { + if (ast === undefined) return undefined + if (path.length === 0) return unwrapAst(ast) + const current = unwrapAst(ast) + const [head, ...tail] = path + if (head === undefined) return current + if (SchemaAST.isUnion(current)) { + return unionAst( + current.types.flatMap((member) => { + const hit = astAtPath(member, path) + return hit === undefined ? [] : [hit] + }) + ) + } + if (head === "-1" || /^\d+$/.test(head)) { + if (SchemaAST.isArrays(current)) { + const element = current.rest[0] ?? current.elements[Number(head)] ?? current.elements[0] + return astAtPath(element, tail) + } + if (SchemaAST.isDeclaration(current) && current.typeParameters.length === 1) { + return astAtPath(current.typeParameters[0], tail) + } + return undefined + } + const objects = encodedObjects(current) + if (objects !== undefined) { + const property = objects.propertySignatures.find((p) => p.name === head) + return property === undefined ? undefined : astAtPath(property.type, tail) + } + return undefined +} + +const elementAst = (ast: SchemaAST.AST | undefined): SchemaAST.AST | undefined => { + if (ast === undefined) return undefined + const current = unwrapAst(ast) + if (SchemaAST.isUnion(current)) { + return unionAst( + current.types.flatMap((member) => { + const hit = elementAst(member) + return hit === undefined ? [] : [hit] + }) + ) + } + if (SchemaAST.isArrays(current)) return unwrapAst(current.rest[0] ?? current.elements[0] ?? current) + if (SchemaAST.isDeclaration(current) && current.typeParameters.length === 1) { + return unwrapAst(current.typeParameters[0]!) + } + return current +} + +const mapKeyAst = (ast: SchemaAST.AST | undefined): SchemaAST.AST | undefined => { + if (ast === undefined) return undefined + const current = unwrapAst(ast) + if (SchemaAST.isUnion(current)) { + return unionAst( + current.types.flatMap((member) => { + const hit = mapKeyAst(member) + return hit === undefined ? [] : [hit] + }) + ) + } + if (SchemaAST.isDeclaration(current) && current.typeParameters.length >= 2) { + return unwrapAst(current.typeParameters[0]!) + } + return current +} + +const mapValueAst = (ast: SchemaAST.AST | undefined): SchemaAST.AST | undefined => { + if (ast === undefined) return undefined + const current = unwrapAst(ast) + if (SchemaAST.isUnion(current)) { + return unionAst( + current.types.flatMap((member) => { + const hit = mapValueAst(member) + return hit === undefined ? [] : [hit] + }) + ) + } + if (SchemaAST.isDeclaration(current) && current.typeParameters.length >= 2) { + return unwrapAst(current.typeParameters[1]!) + } + return current +} + +const asArray = (value: unknown): readonly unknown[] => + Array.isArray(value) ? value : value instanceof Set ? [...value] : [value] + +const encodeJson = (ast: SchemaAST.AST | undefined, value: unknown): unknown => { + if (ast === undefined) return toJsonQueryValue(value) + const current = unwrapAst(ast) + if (isPlainObject(value)) { + const out: Record = {} + for (const [key, child] of Object.entries(value)) { + out[key] = encodeJson(astAtPath(current, [key]), child) + } + return out + } + if (Array.isArray(value)) { + const element = SchemaAST.isArrays(current) || SchemaAST.isUnion(current) + ? elementAst(current) + : current + return value.map((item) => encodeJson(element, item)) + } + if (value instanceof Set) { + return [...value].map((item) => encodeJson(elementAst(current), item)) + } + if (value instanceof Map) { + return [...value.entries()].map(([k, v]) => [ + encodeJson(mapKeyAst(current), k), + encodeJson(mapValueAst(current), v) + ]) + } + return Effect.runSync( + S.encodeUnknownEffect(S.toCodecJson(S.make(current)))(value) as Effect.Effect + ) +} + +const encodeFilterValue = (fieldAst: SchemaAST.AST | undefined, op: Ops, value: unknown): unknown => { + if (fieldAst === undefined) return toJsonQueryValue(value) + if (op === "in" || op === "notIn") { + return asArray(value).map((item) => encodeJson(fieldAst, item)) + } + if ( + op === "includes" + || op === "notIncludes" + || op === "includes-any" + || op === "notIncludes-any" + || op === "includes-all" + || op === "notIncludes-all" + ) { + const element = elementAst(fieldAst) + return op === "includes" || op === "notIncludes" + ? encodeJson(element, value) + : asArray(value).map((item) => encodeJson(element, item)) + } + if ( + op === "hasKeyValue" + || op === "notHasKeyValue" + || op === "hasKeyValue-any" + || op === "notHasKeyValue-any" + || op === "hasKeyValue-all" + || op === "notHasKeyValue-all" + ) { + const keyAst = mapKeyAst(fieldAst) + const valueAst = mapValueAst(fieldAst) + const pair = (item: unknown) => + Array.isArray(item) && item.length >= 2 + ? [encodeJson(keyAst, item[0]), encodeJson(valueAst, item[1])] + : toJsonQueryValue(item) + return op === "hasKeyValue" || op === "notHasKeyValue" ? pair(value) : asArray(value).map(pair) + } + if ( + op === "hasKey" + || op === "notHasKey" + || op === "hasKey-any" + || op === "notHasKey-any" + || op === "hasKey-all" + || op === "notHasKey-all" + ) { + const keyAst = mapKeyAst(fieldAst) + return op === "hasKey" || op === "notHasKey" + ? encodeJson(keyAst, value) + : asArray(value).map((item) => encodeJson(keyAst, item)) + } + if ( + op === "hasValue" + || op === "notHasValue" + || op === "hasValue-any" + || op === "notHasValue-any" + || op === "hasValue-all" + || op === "notHasValue-all" + ) { + const valueAst = mapValueAst(fieldAst) + return op === "hasValue" || op === "notHasValue" + ? encodeJson(valueAst, value) + : asArray(value).map((item) => encodeJson(valueAst, item)) + } + return encodeJson(fieldAst, value) +} + +/** + * Lower Date / Map / Set to JSON when no field schema is available. + * Prefer {@link encodeWithSchema} / {@link jsonifyFilter} with the store schema. + */ +export function toJsonQueryValue(value: unknown): unknown { + if (value instanceof Date) { + return S.encodeSync(dateJson)(value) + } + if (value instanceof Map) { + return [...value.entries()].map(([k, v]) => [toJsonQueryValue(k), toJsonQueryValue(v)]) + } + if (value instanceof Set) { + return [...value].map((v) => toJsonQueryValue(v)) + } + if (Array.isArray(value)) { + return value.map((v) => toJsonQueryValue(v)) + } + if (isPlainObject(value)) { + const out: Record = {} + for (const [k, v] of Object.entries(value)) { + out[k] = toJsonQueryValue(v) + } + return out + } + if (value !== null && typeof value === "object") { + const toJSON = (value as { toJSON?: () => unknown }).toJSON + if (typeof toJSON === "function") { + return toJsonQueryValue(toJSON.call(value)) + } + } + return value +} + +export function encodeWithSchema(schema: S.Top | undefined, value: unknown): unknown { + if (schema === undefined) return toJsonQueryValue(value) + return encodeJson(SchemaAST.toEncoded(schema.ast), value) +} + +export function jsonifyFilter( + filter: readonly FilterResult[], + schema?: S.Top +): FilterResult[] { + const ast = schema === undefined ? undefined : SchemaAST.toEncoded(schema.ast) + return filter.map((r) => + r.t === "and-scope" || r.t === "or-scope" || r.t === "where-scope" + ? { ...r, result: jsonifyFilter(r.result, schema) } + : { ...r, value: encodeFilterValue(astAtPath(ast, r.path.split(".")), r.op, r.value) } + ) +} + +export type JsonLower = { + readonly toJson: (value: unknown) => unknown + readonly jsonifyFilter: (filter: readonly FilterResult[]) => FilterResult[] +} + +export const makeJsonLower = (config?: { readonly schema?: S.Top }): JsonLower => ({ + toJson: (value) => encodeWithSchema(config?.schema, value), + jsonifyFilter: (filter) => jsonifyFilter(filter, config?.schema) +}) + /** Traverse an object by a dot-separated path string, e.g. `"a.b.c"`. */ export function get(obj: any, path: string): any { return path.split(".").reduce((res: any, key: string) => (res != null ? res[key] : res), obj) @@ -55,21 +325,21 @@ export function lowercaseIfString(val: T) { } export function compare(valA: unknown, valB: unknown) { - return valA === valB + return toJsonQueryValue(valA) === toJsonQueryValue(valB) } export function lowerThan(valA: SupportedValues2, valB: SupportedValues2) { - return valA < valB + return (toJsonQueryValue(valA) as SupportedValues2) < (toJsonQueryValue(valB) as SupportedValues2) } export function lowerThanExclusive(valA: SupportedValues2, valB: SupportedValues2) { - return valA <= valB + return (toJsonQueryValue(valA) as SupportedValues2) <= (toJsonQueryValue(valB) as SupportedValues2) } export function greaterThan(valA: SupportedValues2, valB: SupportedValues2) { - return valA > valB + return (toJsonQueryValue(valA) as SupportedValues2) > (toJsonQueryValue(valB) as SupportedValues2) } export function greaterThanExclusive(valA: SupportedValues2, valB: SupportedValues2) { - return valA >= valB + return (toJsonQueryValue(valA) as SupportedValues2) >= (toJsonQueryValue(valB) as SupportedValues2) } diff --git a/packages/infra/test/cosmos-query.test.ts b/packages/infra/test/cosmos-query.test.ts index c272d382d..c3462e351 100644 --- a/packages/infra/test/cosmos-query.test.ts +++ b/packages/infra/test/cosmos-query.test.ts @@ -13,6 +13,101 @@ type OrderEnc = S.Codec.Encoded // Length projection via `relation(...).length()` should emit a scalar // ARRAY_LENGTH expression rather than pulling (or reshaping) the array. +describe("cosmos query filter: native Encoded values", () => { + it("binds Date as ISO string parameters", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "n", op: "eq", value: new Date("2024-01-01T00:00:00.000Z") }], + "Orders", + {} + ) + expect(result.parameters).toEqual( + expect.arrayContaining([{ name: "@v0", value: "2024-01-01T00:00:00.000Z" }]) + ) + }) + + it("binds Map as array of tuples", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "eq", value: new Map([["k", "v"]]) }], + "Orders", + {} + ) + expect(result.parameters).toEqual( + expect.arrayContaining([{ name: "@v0", value: [["k", "v"]] }]) + ) + }) + + it("emits EXISTS over Map JSON tuples for hasKey / hasValue / hasKeyValue", () => { + const byKey = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "n" }], + "Orders", + {} + ) + expect(byKey.query).toContain("EXISTS(SELECT VALUE p FROM p IN f[\"meta\"] WHERE p[0] = @v0)") + expect(byKey.parameters).toEqual(expect.arrayContaining([{ name: "@v0", value: "n" }])) + + const byValue = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasValue", value: 1 }], + "Orders", + {} + ) + expect(byValue.query).toContain("p[1] = @v0") + + const byPair = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "Orders", + {} + ) + expect(byPair.query).toContain("ARRAY_CONTAINS") + expect(byPair.parameters).toEqual(expect.arrayContaining([{ name: "@v0", value: ["n", 1] }])) + + const anyKey = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "meta", op: "hasKey-any", value: ["n", "x"] }], + "Orders", + {} + ) + expect(anyKey.query).toContain("ARRAY_CONTAINS(@v0, p[0])") + }) + + it("binds includes Date as ISO string", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ t: "where", path: "dates", op: "includes", value: new Date("2024-01-01T00:00:00.000Z") }], + "Orders", + {} + ) + expect(result.query).toContain("ARRAY_CONTAINS") + expect(result.parameters).toEqual( + expect.arrayContaining([{ name: "@v0", value: "2024-01-01T00:00:00.000Z" }]) + ) + }) + + it("binds includes-any Date Set as ISO parameters", () => { + const result = buildWhereCosmosQuery3( + "id", + [{ + t: "where", + path: "dates", + op: "includes-any", + value: new Set([new Date("2024-01-01T00:00:00.000Z")]) + }], + "Orders", + {} + ) + expect(result.parameters).toEqual( + expect.arrayContaining([ + { name: "@v0", value: ["2024-01-01T00:00:00.000Z"] }, + { name: "@v0__0", value: "2024-01-01T00:00:00.000Z" } + ]) + ) + }) +}) + describe("cosmos query projection: array length", () => { it("projects packages length via ARRAY_LENGTH", () => { const q = make().pipe( @@ -29,13 +124,13 @@ describe("cosmos query projection: array length", () => { ir.filter ?? [], "Orders", {}, - ir.select as any + ir.select ) expect(result.query).toMatch(/ARRAY_LENGTH\(f(?:\.packages|\["packages"\])\)/) expect(result.query).toContain("AS packageCount") // Must not pull the full array nor reshape via subquery - expect(result.query).not.toMatch(/ARRAY\s*\(\s*SELECT[^)]*FROM\s+t\s+in\s+f[\.\["]/i) + expect(result.query).not.toMatch(/ARRAY\s*\(\s*SELECT[^)]*FROM\s+t\s+in\s+f[.["]/i) expect(result.query).not.toMatch(/SELECT VALUE COUNT/) }) }) @@ -83,7 +178,7 @@ describe("cosmos query projection: union array fields", () => { const packageSelects = select.filter((item) => typeof item === "object" && item !== null && "key" in item && item.key === "packages" ) - const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select as any) + const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select) expect(packageSelects).toHaveLength(1) expect(result.query.match(/\bAS\s+packages\b/g) ?? []).toHaveLength(1) @@ -102,7 +197,7 @@ describe("cosmos query projection: union array fields", () => { const packageSelects = select.filter((item) => typeof item === "object" && item !== null && "key" in item && item.key === "packages" ) - const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select as any) + const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "Orders", {}, ir.select) expect(packageSelects).toHaveLength(1) expect(result.query.match(/\bAS\s+packages\b/g) ?? []).toHaveLength(1) @@ -146,7 +241,7 @@ describe("cosmos query projection: relation-every parameter binding", () => { ) const ir = toFilter(q as any, DN as any) - const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "DN", {}, ir.select as any) + const result = buildWhereCosmosQuery3("id", ir.filter ?? [], "DN", {}, ir.select) // Each filter element binds exactly one parameter: 2 every filters + 2 main filter = 4. expect(result.parameters).toHaveLength(4) diff --git a/packages/infra/test/json-lower.test.ts b/packages/infra/test/json-lower.test.ts new file mode 100644 index 000000000..8ce08f41c --- /dev/null +++ b/packages/infra/test/json-lower.test.ts @@ -0,0 +1,115 @@ +import type { Ops } from "effect-app/Model/filter/filterApi" +import * as S from "effect-app/Schema" +import * as Getter from "effect/SchemaGetter" +import { describe, expect, it } from "vitest" +import { jsonifyFilter } from "../src/Store/utils.js" + +class Day { + readonly ymd: string + constructor(ymd: string) { + this.ymd = ymd + } +} + +const DayFromSelf = S.declare((u): u is Day => u instanceof Day, { + expected: "Day", + toCodecJson: () => + S.link()( + S.String, + { + decode: Getter.transform((s: string) => new Day(s)), + encode: Getter.transform((d: Day) => d.ymd) + } + ) +}) + +const where = (path: string, op: Ops, value: unknown) => ({ + t: "where" as const, + path, + op, + value +}) + +describe("jsonifyFilter Encoded key paths", () => { + it("lowers encodeKeys-renamed native Encoded values via the Encoded field name", () => { + const schema = S + .Struct({ + id: S.String, + day: DayFromSelf + }) + .pipe(S.encodeKeys({ day: "the_day" })) + const day = new Day("2024-06-01") + expect(jsonifyFilter([where("the_day", "eq", day)], schema)).toEqual([ + where("the_day", "eq", "2024-06-01") + ]) + }) + + it("lowers Class.pipe(encodeKeys) using Encoded names, not Type .fields", () => { + class Doc extends S.Class("JsonLowerEncodeKeysDoc")({ + id: S.String, + day: DayFromSelf, + tags: S.NonEmptyArray(S.String) + }) {} + const schema = Doc.pipe(S.encodeKeys({ day: "the_day" })) + const day = new Day("2024-06-01") + expect(jsonifyFilter([where("the_day", "eq", day)], schema)).toEqual([ + where("the_day", "eq", "2024-06-01") + ]) + expect(jsonifyFilter([where("id", "in", ["d1"])], schema)).toEqual([ + where("id", "in", ["d1"]) + ]) + expect(jsonifyFilter([where("tags", "includes-any", ["a"])], schema)).toEqual([ + where("tags", "includes-any", ["a"]) + ]) + }) + + it("does not encode scalar `in` values through an array field renamed onto Encoded `id`", () => { + class Doc extends S.Class("JsonLowerSwappedKeysDoc")({ + id: S.String, + items: S.NonEmptyArray(S.String) + }) {} + const schema = Doc.pipe(S.encodeKeys({ items: "id", id: "item_id" })) + expect(jsonifyFilter([where("item_id", "in", ["d1"])], schema)).toEqual([ + where("item_id", "in", ["d1"]) + ]) + expect(jsonifyFilter([where("id", "includes-any", ["a"])], schema)).toEqual([ + where("id", "includes-any", ["a"]) + ]) + }) + + it("lowers TaggedUnion Opaque `id in`", () => { + const identity = S.Struct({ id: S.String, layout: S.String }) + class Available extends S.Opaque()(S.TaggedStruct("available", { ...identity.fields })) {} + class Reserved extends S.Opaque()( + S.TaggedStruct("reserved", { ...identity.fields, reservation: S.String }) + ) {} + const Cart = S.TaggedUnion([Available, Reserved]) + expect(jsonifyFilter([where("id", "in", ["cart-1"])], Cart)).toEqual([ + where("id", "in", ["cart-1"]) + ]) + }) + + it("lowers TaggedClass union array includes-any through the element schema", () => { + class Picking extends S.TaggedClass()("picking", { + id: S.String, + cartIds: S.NonEmptyArray(S.String), + createdAt: S.Date + }) {} + class Assembling extends S.TaggedClass()("assembling", { + id: S.String, + cartIds: S.NonEmptyArray(S.String), + createdAt: S.Date + }) {} + const Batch = S.TaggedUnion([Picking, Assembling]) + const at = new Date("2024-06-01T00:00:00.000Z") + expect(jsonifyFilter([where("id", "in", ["batch-1"])], Batch)).toEqual([ + where("id", "in", ["batch-1"]) + ]) + expect(jsonifyFilter([where("cartIds", "includes-any", ["cart-1"])], Batch)).toEqual([ + where("cartIds", "includes-any", ["cart-1"]) + ]) + expect(jsonifyFilter([where("createdAt", "eq", at)], Batch)).toEqual([ + where("createdAt", "eq", at.toISOString()) + ]) + }) +}) diff --git a/packages/infra/test/query.test.ts b/packages/infra/test/query.test.ts index 680c7e5f1..1f355b758 100644 --- a/packages/infra/test/query.test.ts +++ b/packages/infra/test/query.test.ts @@ -11,10 +11,16 @@ import * as Option from "effect-app/Option" import * as S from "effect-app/Schema" import { setupRequestContextFromCurrent } from "effect-app/setupRequest" import { flow, pipe } from "effect/Function" +import * as Redacted from "effect/Redacted" +import * as Getter from "effect/SchemaGetter" import * as SchemaTransformation from "effect/SchemaTransformation" import * as Struct from "effect/Struct" +import * as fs from "fs" +import * as os from "os" +import * as path from "path" import { inspect } from "util" import { expect, expectTypeOf, it } from "vitest" +import { DiskStoreLayer } from "../src/Store/Disk.js" import { memFilter, MemoryStoreLive } from "../src/Store/Memory.js" import { SomeService } from "./fixtures.js" @@ -41,7 +47,7 @@ const q = make() where("displayName", "Verona"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z") // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")) ), order("displayName"), page({ take: 10 }), @@ -141,7 +147,7 @@ it("works with repo", () => where("displayName", "Verona"), or( where("displayName", "Riley"), - and("n", "gt", "2021-01-01T00:00:00Z") // TODO: work with To type translation, so Date? + and("n", "gt", new Date("2021-01-01T00:00:00Z")) ), order("displayName"), page({ take: 10 }), @@ -168,6 +174,11 @@ it("works with repo", () => expect(q1).toEqual(items.slice(0, 2).toReversed().map(Struct.pick(["id", "displayName"]))) expect(q2).toEqual(items.slice(0, 2).toReversed().map(Struct.pick(["displayName"]))) + + const byDate = yield* somethingRepo.query( + where("n", new Date("2020-01-01T00:00:00.000Z")) + ) + expect(byDate.map((_) => _.displayName)).toEqual(["Verona", "Riley"]) }) .pipe( Effect.provide(Layer.mergeAll(SomethingRepo.Test, SomeService.Default)), @@ -176,6 +187,150 @@ it("works with repo", () => Effect.runPromise )) +it("memory store round-trips Date/Set/Map via JSON codecs", () => + Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDoc")({ + id: S.String, + at: S.Date, + tags: S.ReadonlySet(S.String), + meta: S.ReadonlyMap({ key: S.String, value: S.Finite }) + }) {} + const at = new Date("2024-06-01T00:00:00.000Z") + const saved = new Doc({ + id: "d1", + at, + tags: new Set(["a", "b"]), + meta: new Map([["n", 1]]) + }) + const repo = yield* makeRepo("JsonCodecDoc", Doc, { makeInitial: Effect.succeed([saved]) }) + const found = yield* repo.find("d1") + expect(Option.isSome(found)).toBe(true) + if (Option.isSome(found)) { + expect(found.value.at.toISOString()).toBe(at.toISOString()) + expect(found.value.tags).toEqual(new Set(["a", "b"])) + expect(found.value.meta).toEqual(new Map([["n", 1]])) + } + const byDate = yield* repo.query(where("at", at)) + expect(byDate.map((_) => _.id)).toEqual(["d1"]) + const byTag = yield* repo.query(where("tags", "includes", "b")) + expect(byTag.map((_) => _.id)).toEqual(["d1"]) + const byKey = yield* repo.query(where("meta", "hasKey", "n")) + expect(byKey.map((_) => _.id)).toEqual(["d1"]) + const byPair = yield* repo.query(where("meta", "hasKeyValue", ["n", 1])) + expect(byPair.map((_) => _.id)).toEqual(["d1"]) + }) + .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) + +class Day { + readonly ymd: string + constructor(ymd: string) { + this.ymd = ymd + } +} + +const DayFromSelf = S.declare((u): u is Day => u instanceof Day, { + expected: "Day", + toCodecJson: () => + S.link()( + S.String, + { + decode: Getter.transform((s: string) => new Day(s)), + encode: Getter.transform((d: Day) => d.ymd) + } + ) +}) + +it("memory store round-trips app native Encoded values via the document schema", () => + Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDayDoc")({ + id: S.String, + day: DayFromSelf + }) {} + const day = new Day("2024-06-01") + const saved = new Doc({ id: "d1", day }) + const repo = yield* makeRepo("JsonCodecDayDoc", Doc, { makeInitial: Effect.succeed([saved]) }) + const found = yield* repo.find("d1") + expect(Option.isSome(found)).toBe(true) + if (Option.isSome(found)) { + expect(found.value.day).toBeInstanceOf(Day) + expect(found.value.day.ymd).toBe("2024-06-01") + } + expect(S.encodeSync(DayFromSelf)(day)).toBe(day) + expect(S.encodeSync(S.toCodecJson(DayFromSelf))(day)).toBe("2024-06-01") + const byDay = yield* repo.query(where("day", day)) + expect(byDay.map((_) => _.id)).toEqual(["d1"]) + }) + .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) + +it("memory store queries encodeKeys-renamed native Encoded fields", () => + Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecEncodeKeysDayDoc")({ + id: S.String, + day: DayFromSelf + }) {} + const schema = Doc.pipe(S.encodeKeys({ day: "the_day" })) + const day = new Day("2024-06-01") + const saved = new Doc({ id: "d1", day }) + const repo = yield* makeRepo("JsonCodecEncodeKeysDayDoc", schema, { makeInitial: Effect.succeed([saved]) }) + const byDay = yield* repo.query(where("the_day", day)) + expect(byDay.map((_) => _.id)).toEqual(["d1"]) + const byId = yield* repo.query(where("id", "in", ["d1"])) + expect(byId.map((_) => _.id)).toEqual(["d1"]) + }) + .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) + +it("disk store round-trips Date/Set/Map via JSON codecs", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "effect-app-disk-json-")) + const diskLive = Layer.merge( + DiskStoreLayer({ url: Redacted.make(`disk://${dir}`), prefix: "", dbName: "test" }, dir), + RepositoryRegistryLive + ) + return Effect + .gen(function*() { + class Doc extends S.Class("JsonCodecDiskDoc")({ + id: S.String, + at: S.Date, + tags: S.ReadonlySet(S.String), + meta: S.ReadonlyMap({ key: S.String, value: S.Finite }) + }) {} + const at = new Date("2024-06-01T00:00:00.000Z") + const saved = new Doc({ + id: "d1", + at, + tags: new Set(["a", "b"]), + meta: new Map([["n", 1]]) + }) + const repo = yield* makeRepo("JsonCodecDiskDoc", Doc, { makeInitial: Effect.succeed([saved]) }) + const found = yield* repo.find("d1") + expect(Option.isSome(found)).toBe(true) + if (Option.isSome(found)) { + expect(found.value.at.toISOString()).toBe(at.toISOString()) + expect(found.value.tags).toEqual(new Set(["a", "b"])) + expect(found.value.meta).toEqual(new Map([["n", 1]])) + } + const jsonFile = fs.readdirSync(dir).find((f) => f.endsWith(".json")) + expect(jsonFile).toBeDefined() + const raw = JSON.parse(fs.readFileSync(path.join(dir, jsonFile!), "utf8")) as Array<{ + at: unknown + tags: unknown + meta: unknown + }> + expect(raw[0]?.at).toBe(at.toISOString()) + expect(raw[0]?.tags).toEqual(["a", "b"]) + expect(raw[0]?.meta).toEqual([["n", 1]]) + }) + .pipe( + Effect.provide(diskLive), + setupRequestContextFromCurrent(), + Effect.scoped, + Effect.runPromise + ) + .finally(() => fs.rmSync(dir, { recursive: true, force: true })) +}) + it("collect", () => Effect .gen(function*() { @@ -196,8 +351,8 @@ it("collect", () => })), S.toType(S.Option(S.String)), (_) => - _.displayName === "Riley" && _.n === "2020-01-01T00:00:00.000Z" - ? Option.some(`${_.displayName}-${_.n}`) + _.displayName === "Riley" && _.n.toISOString() === "2020-01-01T00:00:00.000Z" + ? Option.some(`${_.displayName}-${_.n.toISOString()}`) : Option.none() ), "collect" @@ -215,7 +370,7 @@ it("collect", () => QueryEnd<{ readonly id: string readonly displayName: string - readonly n: string + readonly n: Date readonly union: { readonly _tag: "string" readonly value: string @@ -530,7 +685,7 @@ it( const schema = S.Struct({ id: S.String, createdAt: S.Date.pipe( - S.withDecodingDefault(Effect.sync(() => new Date().toISOString())), + S.withDecodingDefault(Effect.sync(() => new Date())), S.withConstructorDefault(Effect.sync(() => new Date())) ) }) @@ -543,7 +698,7 @@ it( const outputSchema = S.Struct({ id: S.Literal("123"), createdAt: S.Date.pipe( - S.withDecodingDefault(Effect.sync(() => new Date().toISOString())), + S.withDecodingDefault(Effect.sync(() => new Date())), S.withConstructorDefault(Effect.sync(() => new Date())) ) }) @@ -824,7 +979,7 @@ it("ProjectableFromDomain distributes over tagged union Encoded", () => { type GoodCheck = ProjectableFromDomain type BadCheck = ProjectableFromDomain - const _good: GoodCheck = undefined as unknown + const _good: GoodCheck = undefined // @ts-expect-error cancelled branch requires activeRequest not present on domain cancelled const _bad: BadCheck = undefined as unknown void _good @@ -852,7 +1007,7 @@ it("ProjectableFromDomain allows dual same-tag domain variants", () => { type GoodCheck = ProjectableFromDomain type BadFlatCheck = ProjectableFromDomain - const _good: GoodCheck = undefined as unknown + const _good: GoodCheck = undefined // @ts-expect-error packages is not on domain initial; multi-tag flat intersection rejects it const _badFlat: BadFlatCheck = undefined as unknown void _good @@ -1345,6 +1500,49 @@ it("does not allow string queries on arrays", () => expectTypeOf(good2).toEqualTypeOf>() expectTypeOf(good3).toEqualTypeOf>() expectTypeOf(good4).toEqualTypeOf>() + + type Native = { + readonly id: string + readonly dates: Date[] + readonly dateSet: ReadonlySet + readonly tags: ReadonlySet + } + const native = make() + const d = new Date("2020-01-01T00:00:00.000Z") + const n1 = native.pipe(where("dates", "includes", d)) + const n2 = native.pipe(where("dateSet", "includes", d)) + const n3 = native.pipe(where("tags", "includes", "a")) + const n4 = native.pipe(where("dates", "includes-any", [d])) + const n5 = native.pipe(where("dateSet", "includes-any", new Set([d]))) + const n6 = native.pipe(where("id", "in", new Set(["x"]))) + expectTypeOf(n1).toEqualTypeOf>() + expectTypeOf(n2).toEqualTypeOf>() + expectTypeOf(n3).toEqualTypeOf>() + expectTypeOf(n4).toEqualTypeOf>() + expectTypeOf(n5).toEqualTypeOf>() + expectTypeOf(n6).toEqualTypeOf>() + + type WithMap = { + readonly id: string + readonly meta: ReadonlyMap + } + const mapped = make() + const m1 = mapped.pipe(where("meta", "hasKey", "n")) + const m2 = mapped.pipe(where("meta", "hasValue", 1)) + const m3 = mapped.pipe(where("meta", "hasKeyValue", ["n", 1] as const)) + const m4 = mapped.pipe(where("meta", "hasKey-any", ["n", "x"])) + const m5 = mapped.pipe(where("meta", "hasValue-all", new Set([1, 2]))) + const m6 = mapped.pipe(where("meta", "hasKeyValue-any", [["n", 1] as const, ["x", 2] as const])) + expectTypeOf(m1).toEqualTypeOf>() + expectTypeOf(m2).toEqualTypeOf>() + expectTypeOf(m3).toEqualTypeOf>() + expectTypeOf(m4).toEqualTypeOf>() + expectTypeOf(m5).toEqualTypeOf>() + expectTypeOf(m6).toEqualTypeOf>() + // @ts-expect-error cannot hasKey on a string field + mapped.pipe(where("id", "hasKey", "n")) + // @ts-expect-error hasKey value must be the map key type + mapped.pipe(where("meta", "hasKey", 1)) }) .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.scoped, Effect.runPromise)) @@ -2064,6 +2262,57 @@ it("codeFilter: array includes / includes-any / includes-all", () => { expect(runCF(make().pipe(where("tags", "includes-all", ["red", "blue"])))).toEqual(["3"]) }) +it("codeFilter: Date array / Set includes and in", () => { + const d0 = new Date("2020-01-01T00:00:00.000Z") + const d1 = new Date("2021-01-01T00:00:00.000Z") + type DateRow = { + readonly id: string + readonly dates: Date[] + readonly dateSet: ReadonlySet + readonly tag: string + } + const rows: DateRow[] = [ + { id: "1", dates: [d0], dateSet: new Set([d0]), tag: "a" }, + { id: "2", dates: [d1, d0], dateSet: new Set([d1]), tag: "b" } + ] + const run = (q: any) => (memFilter(toFilter(q))(rows) as DateRow[]).map((_) => _.id) + expect(run(make().pipe(where("dates", "includes", d0))).sort()).toEqual(["1", "2"]) + expect(run(make().pipe(where("dates", "includes", d1)))).toEqual(["2"]) + expect(run(make().pipe(where("dateSet", "includes", d0)))).toEqual(["1"]) + expect(run(make().pipe(where("dates", "includes-any", [d1])))).toEqual(["2"]) + expect(run(make().pipe(where("dateSet", "includes-any", new Set([d0, d1])))).sort()).toEqual([ + "1", + "2" + ]) + expect(run(make().pipe(where("tag", "in", new Set(["a"]))))).toEqual(["1"]) +}) + +it("codeFilter: Map hasKey / hasValue / hasKeyValue", () => { + type MapRow = { + readonly id: string + readonly meta: ReadonlyMap + } + const rows: MapRow[] = [ + { id: "1", meta: new Map([["n", 1], ["x", 2]]) }, + { id: "2", meta: new Map([["n", 9]]) }, + { id: "3", meta: new Map([["z", 2]]) } + ] + const run = (q: any) => (memFilter(toFilter(q))(rows) as MapRow[]).map((_) => _.id) + expect(run(make().pipe(where("meta", "hasKey", "n"))).sort()).toEqual(["1", "2"]) + expect(run(make().pipe(where("meta", "notHasKey", "n")))).toEqual(["3"]) + expect(run(make().pipe(where("meta", "hasValue", 2))).sort()).toEqual(["1", "3"]) + expect(run(make().pipe(where("meta", "notHasValue", 2)))).toEqual(["2"]) + expect(run(make().pipe(where("meta", "hasKeyValue", ["n", 1])))).toEqual(["1"]) + expect(run(make().pipe(where("meta", "notHasKeyValue", ["n", 1]))).sort()).toEqual(["2", "3"]) + expect(run(make().pipe(where("meta", "hasKey-any", ["z", "missing"])))).toEqual(["3"]) + expect(run(make().pipe(where("meta", "hasKey-all", ["n", "x"])))).toEqual(["1"]) + expect(run(make().pipe(where("meta", "notHasKey-all", ["n", "x"]))).sort()).toEqual(["2", "3"]) + expect(run(make().pipe(where("meta", "hasValue-any", [9, 99])))).toEqual(["2"]) + expect(run(make().pipe(where("meta", "hasValue-all", [1, 2])))).toEqual(["1"]) + expect(run(make().pipe(where("meta", "hasKeyValue-any", [["n", 9], ["missing", 0]])))).toEqual(["2"]) + expect(run(make().pipe(where("meta", "hasKeyValue-all", [["n", 1], ["x", 2]])))).toEqual(["1"]) +}) + it("codeFilter: in / notIn", () => { expect(runCF(make().pipe(where("tag", "in", ["x", "z"]))).sort()).toEqual(["1", "3"]) expect(runCF(make().pipe(where("tag", "notIn", ["x", "z"]))).sort()).toEqual(["2", "4"]) @@ -2129,7 +2378,7 @@ it("memFilter: agg-count-when groups rows and counts conditionally", () => { }, { key: "total", aggregate: { _tag: "agg-count" } } ] as any - })(rows as any) as any[] + })(rows) as any[] expect(result.length).toBe(2) const nyc = result.find((r: any) => r.city === "NYC")! @@ -2155,7 +2404,7 @@ it("memFilter: agg-sum / agg-min / agg-max aggregate numerics", () => { { key: "min", aggregate: { _tag: "agg-min", field: "salary" } }, { key: "max", aggregate: { _tag: "agg-max", field: "salary" } } ] as any - })(rows as any) as any[] + })(rows) as any[] expect(result.length).toBe(2) const eng = result.find((r: any) => r.dept === "eng")! @@ -2179,7 +2428,7 @@ it("memFilter: aggregate with nested path grouping", () => { { key: "city", path: "address.city" }, { key: "count", aggregate: { _tag: "agg-count" } } ] as any - })(rows as any) as any[] + })(rows) as any[] expect(result.length).toBe(2) expect(result.find((r: any) => r.city === "NYC")!.count).toBe(2) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 433f7373e..0587226fc 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -24,11 +24,157 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("John") }) + it("where eq Date binds ISO string", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "n", op: "eq", value: new Date("2024-01-01T00:00:00.000Z") }], + "users", + {} + ) + expect(result.params).toContain("2024-01-01T00:00:00.000Z") + }) + + it("where eq app native Encoded binds via JsonLower", () => { + class Day { + readonly ymd: string + constructor(ymd: string) { + this.ymd = ymd + } + } + const day = new Day("2024-06-01") + const json = { + toJson: (v: unknown) => v instanceof Day ? v.ymd : v, + jsonifyFilter: (filter: readonly { t: string; path?: string; op?: string; value?: unknown }[]) => + filter.map((r) => r.t === "where" ? { ...r, value: r.value instanceof Day ? r.value.ymd : r.value } : r) + } + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "day", op: "eq", value: day }], + "users", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + json as never + ) + expect(result.params).toContain("2024-06-01") + }) + + it("where in Set binds array values", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "tags", op: "in", value: new Set(["a", "b"]) }], + "users", + {} + ) + expect(result.params).toEqual(expect.arrayContaining(["a", "b"])) + }) + + it("where includes Date binds ISO string", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "dates", op: "includes", value: new Date("2024-01-01T00:00:00.000Z") }], + "users", + {} + ) + expect(result.params).toContain("2024-01-01T00:00:00.000Z") + }) + + it("where hasKey / hasValue / hasKeyValue on Map JSON tuples", () => { + const key = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "n" }], + "users", + {} + ) + expect(key.sql).toContain("json_extract(value, '$[0]')") + expect(key.params).toContain("n") + + const value = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasValue", value: 1 }], + "users", + {} + ) + expect(value.sql).toContain("json_extract(value, '$[1]')") + expect(value.params).toContain(1) + + const pair = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "users", + {} + ) + expect(pair.sql).toContain("json_each") + expect(pair.params).toContain(JSON.stringify(["n", 1])) + + const anyKeys = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey-any", value: ["n", "x"] }], + "users", + {} + ) + expect(anyKeys.sql).toContain(" OR ") + expect(anyKeys.params).toEqual(expect.arrayContaining(["n", "x"])) + }) + + it("pg where hasKey / hasKeyValue uses jsonb tuple elements", () => { + const key = buildWhereSQLQuery( + pgDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "n" }], + "users", + {} + ) + expect(key.sql).toContain("jsonb_array_elements") + expect(key.sql).toContain("e->0") + expect(key.params).toContain(JSON.stringify("n")) + + const pair = buildWhereSQLQuery( + pgDialect, + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "users", + {} + ) + expect(pair.sql).toContain("@>") + expect(pair.sql).toContain("jsonb_build_array") + expect(pair.params).toContain(JSON.stringify(["n", 1])) + }) + + it("where includes-any Date[] binds ISO strings", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ + t: "where", + path: "dates", + op: "includes-any", + value: [new Date("2024-01-01T00:00:00.000Z"), new Date("2024-06-01T00:00:00.000Z")] + }], + "users", + {} + ) + expect(result.params).toEqual( + expect.arrayContaining(["2024-01-01T00:00:00.000Z", "2024-06-01T00:00:00.000Z"]) + ) + }) + it("where eq number", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "eq", value: 25 as any }], + [{ t: "where", path: "age", op: "eq", value: 25 }], "users", {} ) @@ -40,7 +186,7 @@ describe("SQL query builder (SQLite dialect)", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 18 as any }], + [{ t: "where", path: "age", op: "gt", value: 18 }], "users", {} ) @@ -70,7 +216,7 @@ describe("SQL query builder (SQLite dialect)", () => { "id", [ { t: "where", path: "name", op: "eq", value: "Alice" }, - { t: "and", path: "age", op: "gt", value: 18 as any } + { t: "and", path: "age", op: "gt", value: 18 } ], "users", {} @@ -83,7 +229,7 @@ describe("SQL query builder (SQLite dialect)", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "id", op: "in", value: ["a", "b", "c"] as any }], + [{ t: "where", path: "id", op: "in", value: ["a", "b", "c"] }], "users", {} ) @@ -167,7 +313,7 @@ describe("SQL query builder (SQLite dialect)", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "tags", op: "includes-any", value: ["admin", "user"] as any }], + [{ t: "where", path: "tags", op: "includes-any", value: ["admin", "user"] }], "users", {} ) @@ -495,7 +641,7 @@ describe("SQL query builder (PostgreSQL dialect)", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "status", op: "in", value: ["active", "pending"] as any }], + [{ t: "where", path: "status", op: "in", value: ["active", "pending"] }], "users", {} ) @@ -787,7 +933,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r1 = query(db, q1.sql, q1.params) expect(r1.length).toBe(1) - expect((r1[0] as any).id).toBe("1") + expect(r1[0].id).toBe("1") const q2 = buildWhereSQLQuery( sqliteDialect, @@ -798,13 +944,13 @@ describe("SQL Store (SQLite integration)", () => { ) const r2 = query(db, q2.sql, q2.params) expect(r2.length).toBe(1) - expect((r2[0] as any).id).toBe("2") + expect(r2[0].id).toBe("2") // Both queryable by id column const q3 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "id", op: "in", value: ["1", "2"] as any }], + [{ t: "where", path: "id", op: "in", value: ["1", "2"] }], "test_compat", {} ) @@ -830,7 +976,7 @@ describe("SQL Store (SQLite integration)", () => { const q1 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 28 as any }], + [{ t: "where", path: "age", op: "gt", value: 28 }], "test_noid", {} ) @@ -846,8 +992,8 @@ describe("SQL Store (SQLite integration)", () => { ) const r2 = query(db, q2.sql, q2.params) expect(r2.length).toBe(1) - expect((r2[0] as any).id).toBe("2") - expect((JSON.parse((r2[0] as any).data) as any).name).toBe("Bob") + expect(r2[0].id).toBe("2") + expect((JSON.parse(r2[0].data) as any).name).toBe("Bob") // Order + limit still works const q3 = buildWhereSQLQuery( @@ -863,7 +1009,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r3 = query(db, q3.sql, q3.params) expect(r3.length).toBe(2) - expect((JSON.parse((r3[0] as any).data) as any).name).toBe("Bob") // youngest first + expect((JSON.parse(r3[0].data) as any).name).toBe("Bob") // youngest first })) it("query builder generates valid SQL for SQLite", () => @@ -896,13 +1042,13 @@ describe("SQL Store (SQLite integration)", () => { {} ) expect(query(db, q1.sql, q1.params).length).toBe(1) - expect((JSON.parse((query(db, q1.sql, q1.params)[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(query(db, q1.sql, q1.params)[0].data) as any).name).toBe("Alice") // Test gt const q2 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 28 as any }], + [{ t: "where", path: "age", op: "gt", value: 28 }], "test_people", {} ) @@ -927,20 +1073,20 @@ describe("SQL Store (SQLite integration)", () => { "id", [ { t: "where", path: "name", op: "eq", value: "Alice" }, - { t: "and", path: "age", op: "gt", value: 25 as any } + { t: "and", path: "age", op: "gt", value: 25 } ], "test_people", {} ) const r4 = query(db, q4.sql, q4.params) expect(r4.length).toBe(1) - expect((JSON.parse((r4[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(r4[0].data) as any).name).toBe("Alice") // Test IN const q5 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "id", op: "in", value: ["1", "3"] as any }], + [{ t: "where", path: "id", op: "in", value: ["1", "3"] }], "test_people", {} ) @@ -966,7 +1112,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r7 = query(db, q7.sql, q7.params) expect(r7.length).toBe(1) - expect((JSON.parse((r7[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(r7[0].data) as any).name).toBe("Alice") // Test includes (array) const q8 = buildWhereSQLQuery( @@ -987,7 +1133,7 @@ describe("SQL Store (SQLite integration)", () => { { t: "or-scope", result: [ - { t: "where", path: "age", op: "gt", value: 30 as any }, + { t: "where", path: "age", op: "gt", value: 30 }, { t: "and", path: "name", op: "contains", value: "ar" } ], relation: "some" @@ -1012,7 +1158,7 @@ describe("SQL Store (SQLite integration)", () => { ) const r10 = query(db, q10.sql, q10.params) expect(r10.length).toBe(2) - expect((JSON.parse((r10[0] as any).data) as any).name).toBe("Charlie") // oldest first + expect((JSON.parse(r10[0].data) as any).name).toBe("Charlie") // oldest first })) it("computed relation-every / distinct-count / sum / collect run on SQLite", () => @@ -1154,7 +1300,7 @@ describe("SQL Store (SQLite integration)", () => { const results = query(db, nsSql, params) // Should only get Alice and Bob (primary namespace), not Charlie (other namespace) expect(results.length).toBe(2) - const names = results.map((r) => (JSON.parse((r as any).data) as any).name).sort() + const names = results.map((r) => (JSON.parse(r.data) as any).name).sort() expect(names).toEqual(["Alice", "Bob"]) })) @@ -1285,7 +1431,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: true as any }], + [{ t: "where", path: "flag", op: "eq", value: true }], "t", {} ) @@ -1297,7 +1443,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: false as any }], + [{ t: "where", path: "flag", op: "eq", value: false }], "t", {} ) @@ -1308,7 +1454,7 @@ describe("boolean WHERE clauses — query builder", () => { const r1 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "neq", value: true as any }], + [{ t: "where", path: "flag", op: "neq", value: true }], "t", {} ) @@ -1317,7 +1463,7 @@ describe("boolean WHERE clauses — query builder", () => { const r2 = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "in", value: [true, false] as any }], + [{ t: "where", path: "flag", op: "in", value: [true, false] }], "t", {} ) @@ -1328,7 +1474,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: true as any }], + [{ t: "where", path: "flag", op: "eq", value: true }], "t", {} ) @@ -1340,7 +1486,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: false as any }], + [{ t: "where", path: "flag", op: "eq", value: false }], "t", {} ) @@ -1351,7 +1497,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "flag", op: "in", value: [true, false] as any }], + [{ t: "where", path: "flag", op: "in", value: [true, false] }], "t", {} ) @@ -1373,7 +1519,7 @@ describe("boolean WHERE clauses — query builder", () => { const result = buildWhereSQLQuery( pgDialect, "id", - [{ t: "where", path: "age", op: "gt", value: 18 as any }], + [{ t: "where", path: "age", op: "gt", value: 18 }], "t", {} ) @@ -1404,13 +1550,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: true as any }], + [{ t: "where", path: "flag", op: "eq", value: true }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(rows[0].data) as any).name).toBe("Alice") })) it("where flag = false matches only false rows", () => @@ -1426,13 +1572,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "eq", value: false as any }], + [{ t: "where", path: "flag", op: "eq", value: false }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Bob") + expect((JSON.parse(rows[0].data) as any).name).toBe("Bob") })) it("where nested boolean path works", () => @@ -1448,13 +1594,51 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "meta.active", op: "eq", value: true as any }], + [{ t: "where", path: "meta.active", op: "eq", value: true }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Alice") + expect((JSON.parse(rows[0].data) as any).name).toBe("Alice") + })) + + it("where hasKey / hasValue / hasKeyValue match Map tuple JSON", () => + withDb((db) => { + db.exec(`CREATE TABLE "t" (id TEXT PRIMARY KEY, _etag TEXT, data JSON NOT NULL)`) + db + .prepare(`INSERT INTO "t" (id, _etag, data) VALUES (?, ?, ?)`) + .run("1", "e", JSON.stringify({ meta: [["n", 1], ["x", 2]] })) + db + .prepare(`INSERT INTO "t" (id, _etag, data) VALUES (?, ?, ?)`) + .run("2", "e", JSON.stringify({ meta: [["n", 9]] })) + + const byKey = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKey", value: "x" }], + "t", + {} + ) + expect(query(db, byKey.sql, byKey.params).map((r) => r.id)).toEqual(["1"]) + + const byValue = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasValue", value: 9 }], + "t", + {} + ) + expect(query(db, byValue.sql, byValue.params).map((r) => r.id)).toEqual(["2"]) + + const byPair = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta", op: "hasKeyValue", value: ["n", 1] }], + "t", + {} + ) + expect(query(db, byPair.sql, byPair.params).map((r) => r.id)).toEqual(["1"]) })) it("where neq boolean works", () => @@ -1470,13 +1654,13 @@ describe("boolean WHERE clauses — SQLite integration (end-to-end)", () => { const q = buildWhereSQLQuery( sqliteDialect, "id", - [{ t: "where", path: "flag", op: "neq", value: true as any }], + [{ t: "where", path: "flag", op: "neq", value: true }], "t", {} ) const rows = query(db, q.sql, q.params) expect(rows.length).toBe(1) - expect((JSON.parse((rows[0] as any).data) as any).name).toBe("Bob") + expect((JSON.parse(rows[0].data) as any).name).toBe("Bob") })) }) @@ -1573,7 +1757,7 @@ describe("toRow strips _etag and id from data", () => { const toRow = (e: any, idKey: IdKey) => { const newE = makeETag(e) const id = newE[idKey] as string - const { _etag, [idKey]: _id, ...rest } = newE as any + const { _etag, [idKey]: _id, ...rest } = newE const data = JSON.stringify(rest) return { id, _etag: newE._etag!, data, item: newE } } @@ -1708,4 +1892,14 @@ describe("parseRow reconstructs full object from row", () => { expect(reconstructed.tags).toEqual(["admin"]) expect(reconstructed._etag).toBe(newE._etag) }) + + it("lowers Date defaultValues to ISO before merge", () => { + const result: any = parseRow( + { id: "1", _etag: "e1", data: JSON.stringify({ name: "Alice" }) }, + "id", + { at: new Date("2024-06-01T00:00:00.000Z") } + ) + expect(result.at).toBe("2024-06-01T00:00:00.000Z") + expect(result.name).toBe("Alice") + }) }) diff --git a/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts b/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts index 6960e2df7..9d794a84d 100644 --- a/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts +++ b/packages/vue-components/__tests__/OmegaForm/DateValidation.test.ts @@ -41,7 +41,7 @@ describe("Date field validation", () => { setup() { const form = useOmegaForm( S.Struct({ - date: S.Date + date: S.DateFromString }), { onSubmit: async ({ value }) => { diff --git a/packages/vue-components/src/components/OmegaForm/meta/checks.ts b/packages/vue-components/src/components/OmegaForm/meta/checks.ts index eef6a693f..78aca09f4 100644 --- a/packages/vue-components/src/components/OmegaForm/meta/checks.ts +++ b/packages/vue-components/src/components/OmegaForm/meta/checks.ts @@ -93,7 +93,10 @@ export const getFieldMetadataFromAst = (property: S.AST.AST) => { base.type = "boolean" } else if ( S.AST.isDeclaration(property) - && (property.annotations as any)?.typeConstructor?._tag === "Date" + && ( + (property.annotations as any)?.typeConstructor?._tag === "Date" + || (property.annotations as any)?.representation?.id === "effect/schema/Date" + ) ) { base.type = "date" } else { diff --git a/packages/vue-components/stories/OmegaForm/AutoGeneration.vue b/packages/vue-components/stories/OmegaForm/AutoGeneration.vue index 7071b30fb..ea511b1e2 100644 --- a/packages/vue-components/stories/OmegaForm/AutoGeneration.vue +++ b/packages/vue-components/stories/OmegaForm/AutoGeneration.vue @@ -63,7 +63,7 @@ const schema = S.Struct({ boolean: S.Boolean, email: S.Email, url: S.Url, - date: S.Date + date: S.DateFromString }) type Meta = OmegaAutoGenMeta< typeof schema.Encoded, diff --git a/packages/vue-components/stories/OmegaForm/Date.vue b/packages/vue-components/stories/OmegaForm/Date.vue index 3466bf98e..53e7420a5 100644 --- a/packages/vue-components/stories/OmegaForm/Date.vue +++ b/packages/vue-components/stories/OmegaForm/Date.vue @@ -26,7 +26,7 @@ import * as S from "effect-app/Schema" import { useOmegaForm } from "../../src/components/OmegaForm" const schema = S.Struct({ - date: S.NullOr(S.Date) + date: S.NullOr(S.DateFromString) }) const form = useOmegaForm(schema, { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04496f088..428be274a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -342,6 +342,9 @@ importers: "@sendgrid/mail": specifier: ^8.1.6 version: 8.1.6(patch_hash=3d2dd1cd5f52d3eef1772e4be48776258a596f9277c00adf0a471f54fcd3e21a)(debug@4.4.3(supports-color@8.1.1)) + "@sentry/node": + specifier: 10.55.0 + version: 10.55.0(supports-color@8.1.1) effect: specifier: ^4.0.0-beta.107 version: 4.0.0-beta.107 @@ -373,9 +376,6 @@ importers: "@effect/sql-sqlite-node": specifier: 4.0.0-beta.107 version: 4.0.0-beta.107(effect@4.0.0-beta.107) - "@sentry/node": - specifier: 10.55.0 - version: 10.55.0(supports-color@8.1.1) "@sentry/opentelemetry": specifier: 10.55.0 version: 10.55.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/semantic-conventions@1.40.0)