From 07b8633f909a867aab8f75c14c6a31910c195694 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 26 Nov 2025 16:21:50 +0000 Subject: [PATCH 01/18] feat: add lakebase connector --- package.json | 3 +- packages/app-kit/package.json | 2 + packages/app-kit/src/connectors/index.ts | 1 + .../app-kit/src/connectors/lakebase/client.ts | 412 ++++++++++++++++++ .../src/connectors/lakebase/defaults.ts | 11 + .../app-kit/src/connectors/lakebase/index.ts | 8 + .../app-kit/src/connectors/lakebase/types.ts | 59 +++ .../app-kit/src/connectors/lakebase/utils.ts | 92 ++++ pnpm-lock.yaml | 386 ++++++++++------ 9 files changed, 837 insertions(+), 137 deletions(-) create mode 100644 packages/app-kit/src/connectors/lakebase/client.ts create mode 100644 packages/app-kit/src/connectors/lakebase/defaults.ts create mode 100644 packages/app-kit/src/connectors/lakebase/index.ts create mode 100644 packages/app-kit/src/connectors/lakebase/types.ts create mode 100644 packages/app-kit/src/connectors/lakebase/utils.ts diff --git a/package.json b/package.json index 4039aacbe..2278b90e1 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ }, "lint-staged": { "(*.ts|*.tsx|*.js|*.jsx|*.json|*.md|*.yml|*.yaml|*.css)": [ - "pnpm lint:fix && pnpm format" + "biome lint --write", + "biome format --write" ] }, "devDependencies": { diff --git a/packages/app-kit/package.json b/packages/app-kit/package.json index 3c207760c..13f12916f 100644 --- a/packages/app-kit/package.json +++ b/packages/app-kit/package.json @@ -47,6 +47,7 @@ "@opentelemetry/semantic-conventions": "^1.38.0", "dotenv": "^16.6.1", "express": "^4.22.0", + "pg": "^8.16.3", "shared": "workspace:*", "vite": "npm:rolldown-vite@7.1.14", "ws": "^8.18.3", @@ -54,6 +55,7 @@ }, "devDependencies": { "@types/express": "^4.17.25", + "@types/pg": "^8.15.6", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.1.1" }, diff --git a/packages/app-kit/src/connectors/index.ts b/packages/app-kit/src/connectors/index.ts index 37f2a1276..9d033806b 100644 --- a/packages/app-kit/src/connectors/index.ts +++ b/packages/app-kit/src/connectors/index.ts @@ -1 +1,2 @@ export * from "./sql-warehouse"; +export * from "./lakebase"; diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts new file mode 100644 index 000000000..2c552b923 --- /dev/null +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -0,0 +1,412 @@ +import { randomUUID } from "node:crypto"; +import type { WorkspaceClient } from "@databricks/sdk-experimental"; +import { ApiClient, Config } from "@databricks/sdk-experimental"; +import { deepMerge } from "../../utils"; +import pg from "pg"; +import { lakebaseDefaults } from "./defaults"; +import type { + LakebaseConfig, + LakebaseConnectionConfig, + LakebaseCredentials, +} from "./types"; +import { parseConnectionString } from "./utils"; + +/** + * Enterprise-grade connector for Databricks Lakebase + * @example Simplest - everything from env/context + * ```typescript + * const connector = new LakebaseConnector(); + * await connector.query('SELECT * FROM users'); + * ``` + * + * @example With explicit connection string + * ```typescript + * const connector = new LakebaseConnector({ + * connectionString: 'postgresql://...' + * }); + * ``` + */ +export class LakebaseConnector { + private readonly CACHE_BUFFER_MS = 2 * 60 * 1000; + private readonly config: LakebaseConfig; + private readonly connectionConfig: LakebaseConnectionConfig; + private pool: pg.Pool | null = null; + private credentials: LakebaseCredentials | null = null; + + constructor(userConfig?: Partial) { + this.config = deepMerge(lakebaseDefaults, userConfig); + + this.connectionConfig = this.parseConnectionConfig(); + + // validate configuration + if (this.config.maxPoolSize < 1) { + throw new Error("maxPoolSize must be at least 1"); + } + if (this.config.credentialTTLMs < 60_000) { + throw new Error("credentialTTLMs must be at least 60 seconds"); + } + } + + /** + * Execute a SQL query + * + * @example + * ```typescript + * const users = await connector.query('SELECT * FROM users'); + * const user = await connector.query('SELECT * FROM users WHERE id = $1', [123]); + * ``` + */ + async query( + sql: string, + params?: any[], + retryCount: number = 0, + ): Promise> { + const pool = await this.getPool(); + + try { + return await pool.query(sql, params); + } catch (error) { + // retry on auth failure + if (this.isAuthError(error)) { + await this.rotateCredentials(); + const newPool = await this.getPool(); + return await newPool.query(sql, params); + } + + // retry on transient errors, but only once + if (this.isTransientError(error) && retryCount < 1) { + await new Promise((resolve) => setTimeout(resolve, 100)); + return await this.query(sql, params, retryCount + 1); + } + + throw error; + } + } + + /** + * Execute a transaction + * + * @example + * ```typescript + * await connector.transaction(async (client) => { + * await client.query('BEGIN'); + * await client.query('INSERT INTO accounts (name) VALUES ($1)', ['Alice']); + * await client.query('INSERT INTO logs (action) VALUES ($1)', ['Created Alice']); + * await client.query('COMMIT'); + * }); + * ``` + */ + async transaction( + callback: (client: pg.PoolClient) => Promise, + retryCount: number = 0, + ): Promise { + const pool = await this.getPool(); + const client = await pool.connect(); + + try { + return await callback(client); + } catch (error) { + // retry on auth failure + if (this.isAuthError(error)) { + client.release(); + await this.rotateCredentials(); + const newPool = await this.getPool(); + const retryClient = await newPool.connect(); + try { + return await callback(retryClient); + } finally { + retryClient.release(); + } + } + + // retry on transient errors, but only once + if (this.isTransientError(error) && retryCount < 1) { + client.release(); + await new Promise((resolve) => setTimeout(resolve, 100)); + const retryClient = await pool.connect(); + try { + return await this.transaction(callback, retryCount + 1); + } finally { + retryClient.release(); + } + } + + throw error; + } finally { + client.release(); + } + } + + /** Check if database connection is healthy */ + async healthCheck(): Promise { + try { + const result = await this.query<{ result: number }>("SELECT 1 as result"); + return result.rows[0]?.result === 1; + } catch { + return false; + } + } + + /** Close connection pool (call on shutdown) */ + async close(): Promise { + if (this.pool) { + await this.pool.end().catch((error) => { + console.error("Error closing connection pool:", error); + }); + this.pool = null; + } + this.credentials = null; + } + + /** Setup graceful shutdown to close connection pools */ + shutdown(): void { + process.on("SIGTERM", () => this.close()); + process.on("SIGINT", () => this.close()); + this.close(); + } + + /** Get Databricks workspace client - from config or request context */ + private getWorkspaceClient(): WorkspaceClient { + if (this.config.workspaceClient) { + return this.config.workspaceClient; + } + + try { + const { getRequestContext } = require("../../utils"); + const { serviceDatabricksClient } = getRequestContext(); + + // cache it for subsequent calls + this.config.workspaceClient = serviceDatabricksClient; + return serviceDatabricksClient; + } catch (_error) { + throw new Error( + "Databricks workspace client not available. Either pass it in config or use within App Kit request context.", + ); + } + } + + /** Get or create connection pool */ + private async getPool(): Promise { + if (!this.connectionConfig) { + throw new Error( + "Lakebase connection not configured. " + + "Set LAKEBASE_CONNECTION_STRING env var or provide config in constructor.", + ); + } + + if (!this.pool) { + const creds = await this.getCredentials(); + this.pool = this.createPool(creds); + } + return this.pool; + } + + /** Create PostgreSQL pool */ + private createPool(credentials: { + username: string; + password: string; + }): pg.Pool { + const { host, database, port, sslMode } = this.connectionConfig; + + const pool = new pg.Pool({ + host, + port, + database, + user: credentials.username, + password: credentials.password, + max: this.config.maxPoolSize, + idleTimeoutMillis: this.config.idleTimeoutMs, + connectionTimeoutMillis: this.config.connectionTimeoutMs, + ssl: sslMode === "require" ? { rejectUnauthorized: true } : false, + }); + + pool.on("error", (error) => { + console.error("Connection pool error:", error.message, { + code: (error as any).code, + }); + }); + + return pool; + } + + /** Get or fetch credentials with caching */ + private async getCredentials(): Promise<{ + username: string; + password: string; + }> { + const now = Date.now(); + + // return cached if still valid + if ( + this.credentials && + now < this.credentials.expiresAt - this.CACHE_BUFFER_MS + ) { + return this.credentials; + } + + // fetch new credentials + const username = await this.fetchUsername(); + const password = await this.fetchPassword(); + + this.credentials = { + username, + password, + expiresAt: now + this.config.credentialTTLMs, + }; + + return { username, password }; + } + + /** Rotate credentials and recreate pool */ + private async rotateCredentials(): Promise { + // clear cached credentials + this.credentials = null; + + if (this.pool) { + const oldPool = this.pool; + this.pool = null; + oldPool.end().catch((error) => { + console.error( + "Error closing old connection pool during rotation:", + error, + ); + }); + } + } + + /** Fetch username from Databricks */ + private async fetchUsername(): Promise { + const workspaceClient = this.getWorkspaceClient(); + const user = await workspaceClient.currentUser.me(); + if (!user.userName) { + throw new Error("Failed to get current user from Databricks workspace"); + } + return user.userName; + } + + /** Fetch password (OAuth token) from Databricks */ + private async fetchPassword(): Promise { + const host = this.connectionConfig.host; + + const uid = host.split(".")[0]?.replace("instance-", ""); + if (!uid) { + throw new Error( + `Invalid lakebase hostname: ${host}. Expected format: instance-.database..databricks.com`, + ); + } + + const workspaceClient = this.getWorkspaceClient(); + const config = new Config({ host: workspaceClient.config.host }); + const apiClient = new ApiClient(config); + + // find database instance + const dbInfo = await apiClient.request({ + path: `/api/2.0/database/instances:findByUid`, + method: "GET", + query: { uid }, + payload: { uid }, + headers: new Headers(), + raw: false, + }); + + if (!this.hasName(dbInfo)) { + throw new Error(`Database instance not found for uid: ${uid}`); + } + + const credentials = await apiClient.request({ + path: `/api/2.0/database/credentials`, + method: "POST", + headers: new Headers(), + raw: false, + payload: { + instance_names: [dbInfo.name], + request_id: randomUUID(), + }, + }); + + if (!this.hasToken(credentials)) { + throw new Error( + `Failed to generate credentials for instance: ${dbInfo.name}`, + ); + } + + return credentials.token; + } + + /** Check if error is auth failure */ + private isAuthError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + (error as any).code === "28P01" + ); + } + + private isTransientError(error: unknown): boolean { + if (typeof error !== "object" || error === null || !("code" in error)) { + return false; + } + + const code = (error as any).code; + return ( + code === "ECONNRESET" || + code === "ECONNREFUSED" || + code === "ETIMEDOUT" || + code === "57P01" || // admin_shutdown + code === "57P03" || // cannot_connect_now + code === "08006" || // connection_failure + code === "08003" || // connection_does_not_exist + code === "08000" // connection_exception + ); + } + + /** Type guard for database instance */ + private hasName(value: unknown): value is { name: string } { + return ( + typeof value === "object" && + value !== null && + "name" in value && + typeof (value as any).name === "string" + ); + } + + /** Type guard for credentials */ + private hasToken(value: unknown): value is { token: string } { + return ( + typeof value === "object" && + value !== null && + "token" in value && + typeof (value as any).token === "string" + ); + } + + /** Parse connection configuration from config or environment */ + private parseConnectionConfig(): LakebaseConnectionConfig { + if (this.config.connectionString) { + const { connectionParams } = parseConnectionString( + this.config.connectionString, + ); + return connectionParams; + } + + const envConnectionString = process.env.LAKEBASE_CONNECTION_STRING; + if (envConnectionString) { + const { connectionParams } = parseConnectionString(envConnectionString); + return connectionParams; + } + + if (this.config.host && this.config.database) { + return { + host: this.config.host, + database: this.config.database, + port: this.config.port, + sslMode: this.config.sslMode, + }; + } + + throw new Error( + "Lakebase connection not configured. Either set LAKEBASE_CONNECTION_STRING env var or provide config in constructor.", + ); + } +} diff --git a/packages/app-kit/src/connectors/lakebase/defaults.ts b/packages/app-kit/src/connectors/lakebase/defaults.ts new file mode 100644 index 000000000..67d3688f2 --- /dev/null +++ b/packages/app-kit/src/connectors/lakebase/defaults.ts @@ -0,0 +1,11 @@ +import type { LakebaseConfig } from "./types"; + +/** Default configuration for Lakebase connector */ +export const lakebaseDefaults: LakebaseConfig = { + port: 5432, + sslMode: "require", + maxPoolSize: 10, + idleTimeoutMs: 30_000, + connectionTimeoutMs: 10_000, + credentialTTLMs: 25 * 60 * 1000, // 25 minutes +}; diff --git a/packages/app-kit/src/connectors/lakebase/index.ts b/packages/app-kit/src/connectors/lakebase/index.ts new file mode 100644 index 000000000..3910d4a40 --- /dev/null +++ b/packages/app-kit/src/connectors/lakebase/index.ts @@ -0,0 +1,8 @@ +// Main connector +export { LakebaseConnector } from "./client"; + +// Types for user configuration +export type { LakebaseConfig } from "./types"; + +// Utilities +export * from "./utils"; diff --git a/packages/app-kit/src/connectors/lakebase/types.ts b/packages/app-kit/src/connectors/lakebase/types.ts new file mode 100644 index 000000000..c6e29db25 --- /dev/null +++ b/packages/app-kit/src/connectors/lakebase/types.ts @@ -0,0 +1,59 @@ +import type { WorkspaceClient } from "@databricks/sdk-experimental"; + +/** Configuration for LakebaseConnector */ +export interface LakebaseConfig { + /** Databricks workspace client */ + workspaceClient?: WorkspaceClient; + + /** Connection string (postgresql://...) */ + connectionString?: string; + + /** Database host (e.g., instance-uuid.database.region.databricks.com) */ + host?: string; + + /** Database name */ + database?: string; + + /** Database port */ + port: number; + + /** SSL mode */ + sslMode: "require" | "disable" | "prefer"; + + /** Maximum number of connections in the pool */ + maxPoolSize: number; + + /** Close idle connections after this time (milliseconds) */ + idleTimeoutMs: number; + + /** Connection timeout (milliseconds) */ + connectionTimeoutMs: number; + + /** How long credentials are valid (milliseconds) */ + credentialTTLMs: number; + + /** Additional configuration options */ + [key: string]: unknown; +} + +/** Lakebase credentials for authentication */ +export interface LakebaseCredentials { + /** Username */ + username: string; + /** Password */ + password: string; + /** Expires at */ + expiresAt: number; +} + +/** Internal connection configuration */ +export interface LakebaseConnectionConfig { + /** Database host */ + readonly host: string; + /** Database name */ + readonly database: string; + /** Database port */ + readonly port: number; + /** SSL mode */ + readonly sslMode: "require" | "disable" | "prefer"; +} diff --git a/packages/app-kit/src/connectors/lakebase/utils.ts b/packages/app-kit/src/connectors/lakebase/utils.ts new file mode 100644 index 000000000..e7d79c1af --- /dev/null +++ b/packages/app-kit/src/connectors/lakebase/utils.ts @@ -0,0 +1,92 @@ +import type { LakebaseConnectionConfig } from "./types"; + +export interface ParsedConnectionString { + connectionParams: LakebaseConnectionConfig; + originalConnectionString: string; +} + +/** Parse connection string or environment variables */ +export function parseConnectionString( + connectionStringOrHost: string, + database?: string, + port?: number, +): ParsedConnectionString { + if ( + connectionStringOrHost.startsWith("postgresql://") || + connectionStringOrHost.startsWith("postgres://") + ) { + // parse full connection string + const cleanedString = connectionStringOrHost.replace( + /:?\$\{PGPASSWORD\}@/, + "@", + ); + const url = new URL(cleanedString); + + if (url.protocol !== "postgresql:" && url.protocol !== "postgres:") { + throw new Error( + `Invalid connection string protocol: ${url.protocol}. Expected postgresql: or postgres:`, + ); + } + + if (!url.hostname) { + throw new Error("Connection string must include a hostname"); + } + + const dbName = url.pathname.slice(1) || "databricks_postgres"; + const sslMode = + (url.searchParams.get("sslmode") as "require" | "disable" | "prefer") || + "require"; + + const connectionParams: LakebaseConnectionConfig = { + host: url.hostname, + database: dbName, + port: url.port ? parseInt(url.port, 10) : 5432, + sslMode, + }; + + return { + connectionParams, + originalConnectionString: connectionStringOrHost, + }; + } + + if (!database) { + throw new Error( + "Database name is required when using hostname directly (PGHOST format)", + ); + } + + const connectionParams: LakebaseConnectionConfig = { + host: connectionStringOrHost, + database, + port: port || 5432, + sslMode: "require", + }; + + return { + connectionParams, + originalConnectionString: `postgresql://:@${connectionStringOrHost}:${port || 5432}/${database}`, + }; +} + +/** Parse connection configuration from environment variables */ +export function parseFromEnv(): LakebaseConnectionConfig { + const host = process.env.PGHOST; + const database = process.env.PGDATABASE; + const port = process.env.PGPORT ? parseInt(process.env.PGPORT, 10) : 5432; + + if (!host) { + throw new Error("PGHOST environment variable is required"); + } + + if (!database) { + throw new Error("PGDATABASE environment variable is required"); + } + + return { + host, + database, + port, + sslMode: "require", + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c25cbd0cd..5f89b1bed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,13 +149,16 @@ importers: version: 16.6.1 express: specifier: ^4.22.0 - version: 4.22.1 + version: 4.22.0 + pg: + specifier: ^8.16.3 + version: 8.16.3 shared: specifier: workspace:* version: link:../shared vite: specifier: npm:rolldown-vite@7.1.14 - version: rolldown-vite@7.1.14(@types/node@24.7.2)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1) + version: rolldown-vite@7.1.14(@types/node@24.10.1)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1) ws: specifier: ^8.18.3 version: 8.18.3(bufferutil@4.0.9) @@ -166,12 +169,15 @@ importers: '@types/express': specifier: ^4.17.25 version: 4.17.25 + '@types/pg': + specifier: ^8.15.6 + version: 8.15.6 '@types/ws': specifier: ^8.18.1 version: 8.18.1 '@vitejs/plugin-react': specifier: ^5.1.1 - version: 5.1.2(rolldown-vite@7.1.14(@types/node@24.7.2)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1)) + version: 5.1.1(rolldown-vite@7.1.14(@types/node@24.10.1)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1)) packages/app-kit-ui: dependencies: @@ -288,7 +294,7 @@ importers: version: 0.4.6(react-dom@19.2.0(react@19.2.0))(react@19.2.0) react-day-picker: specifier: ^9.11.3 - version: 9.11.3(react@19.2.0) + version: 9.12.0(react@19.2.0) react-hook-form: specifier: ^7.68.0 version: 7.68.0(react@19.2.0) @@ -357,8 +363,8 @@ packages: resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} engines: {node: '>=6.9.0'} - '@babel/compat-data@7.28.4': - resolution: {integrity: sha512-YsmSKC29MJwf0gF8Rjjrg5LQCmyh+j/nD8/eP7f+BeoQTKYqs9RoWbjGOdy0+1Ekr68RJZMUOPVQaQisnIo4Rw==} + '@babel/compat-data@7.28.5': + resolution: {integrity: sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==} engines: {node: '>=6.9.0'} '@babel/core@7.28.4': @@ -628,15 +634,9 @@ packages: '@date-fns/tz@1.4.1': resolution: {integrity: sha512-P5LUNhtbj6YfI3iJjw5EL9eUAG6OitD0W3fWQcpQjDRc/QIsL0tRNuO1PcDvPccWL1fSTXXdE1ds+l95DV/OFA==} - '@emnapi/core@1.5.0': - resolution: {integrity: sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==} - '@emnapi/core@1.7.1': resolution: {integrity: sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==} - '@emnapi/runtime@1.5.0': - resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==} - '@emnapi/runtime@1.7.1': resolution: {integrity: sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==} @@ -2189,6 +2189,9 @@ packages: '@rolldown/pluginutils@1.0.0-beta.41': resolution: {integrity: sha512-ycMEPrS3StOIeb87BT3/+bu+blEtyvwQ4zmo2IcJQy0Rd1DAAhKksA0iUZ3MYSpJtjlPhg0Eo6mvVS6ggPhRbw==} + '@rolldown/pluginutils@1.0.0-beta.47': + resolution: {integrity: sha512-8QagwMH3kNCuzD8EWL8R2YPW5e4OrHNSAHRFDdmFqEwEaD/KcNKjVoumo+gP2vW5eKB2UPbM6vTYiGZX0ixLnw==} + '@rolldown/pluginutils@1.0.0-beta.53': resolution: {integrity: sha512-vENRlFU4YbrwVqNDZ7fLvy+JR1CRkyr01jhSiDpE1u6py3OMzQfztQU2jxykW3ALNxO4kSlqIDeYyD0Y9RcQeQ==} @@ -2446,6 +2449,9 @@ packages: '@types/node@20.19.21': resolution: {integrity: sha512-CsGG2P3I5y48RPMfprQGfy4JPRZ6csfC3ltBZSRItG3ngggmNY/qs2uZKp4p9VbrpqNNSMzUZNFZKzgOGnd/VA==} + '@types/node@24.10.1': + resolution: {integrity: sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==} + '@types/node@24.7.2': resolution: {integrity: sha512-/NbVmcGTP+lj5oa4yiYxxeBjRivKQ5Ns1eSZeB99ExsEQ6rX5XYU1Zy/gGxY/ilqtD4Etx9mKyrPxZRetiahhA==} @@ -2481,8 +2487,14 @@ packages: '@types/send@0.17.5': resolution: {integrity: sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w==} - '@types/send@1.2.0': - resolution: {integrity: sha512-zBF6vZJn1IaMpg3xUF25VK3gd3l8zwE0ZLRX7dsQyQi+jp4E8mMDJNGDYnYse+bQhYwWERTxVwHpi3dMOq7RKQ==} + '@types/send@0.17.6': + resolution: {integrity: sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==} + + '@types/send@1.2.1': + resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} + + '@types/serve-static@1.15.10': + resolution: {integrity: sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==} '@types/serve-static@1.15.9': resolution: {integrity: sha512-dOTIuqpWLyl3BBXU3maNQsS4A3zuuoYRNIvYSxxhebPfXg2mzWQEPne/nlJ37yOse6uGgR386uTpdsx4D0QZWA==} @@ -2502,8 +2514,8 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 - '@vitejs/plugin-react@5.1.2': - resolution: {integrity: sha512-EcA07pHJouywpzsoTUqNh5NwGayl2PPVEJKUSinGGSxFGYn+shYbqMGBg6FXDqgXum9Ou/ecb+411ssw8HImJQ==} + '@vitejs/plugin-react@5.1.1': + resolution: {integrity: sha512-WQfkSw0QbQ5aJ2CHYw23ZGkqnRwqKHD/KYsMeTkZzPT4Jcf0DcBxBtwMJxnu6E7oxw5+JC6ZAiePgh28uJ1HBA==} engines: {node: ^20.19.0 || >=22.12.0} peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 @@ -2637,8 +2649,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - baseline-browser-mapping@2.8.16: - resolution: {integrity: sha512-OMu3BGQ4E7P1ErFsIPpbJh0qvDudM/UuJeHgkAvfWe+0HFJCXh+t/l8L6fVLR55RI/UbKrVLnAXZSVwd9ysWYw==} + baseline-browser-mapping@2.8.32: + resolution: {integrity: sha512-OPz5aBThlyLFgxyhdwf/s2+8ab3OvT7AdTNvKHBwpXomIYeXqpUUuT8LrdtxZSsWJ4R4CU1un4XGh5Ez3nlTpw==} hasBin: true bidi-js@1.0.3: @@ -2653,8 +2665,8 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - body-parser@1.20.3: - resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} + body-parser@1.20.4: + resolution: {integrity: sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} brace-expansion@2.0.2: @@ -2664,8 +2676,8 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} - browserslist@4.26.3: - resolution: {integrity: sha512-lAUU+02RFBuCKQPj/P6NgjlbCnLBMp4UtgTx7vNHd3XSIJF87s9a5rA3aH2yw3GS9DqZAUbOtZdCCiZeVRqt0w==} + browserslist@4.28.0: + resolution: {integrity: sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true @@ -2699,8 +2711,8 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - caniuse-lite@1.0.30001751: - resolution: {integrity: sha512-A0QJhug0Ly64Ii3eIqHu5X51ebln3k4yTUkY1j8drqpWHVreg/VLijN48cZ1bYPiqOQuqpkIKnzr/Ul8V+p6Cw==} + caniuse-lite@1.0.30001757: + resolution: {integrity: sha512-r0nnL/I28Zi/yjk1el6ilj27tKcdjLsNqAOZr0yVjWPrSQyHgKI2INaEWw21bAQSv2LXRt1XuCS/GomNpWOxsQ==} chai@5.3.3: resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} @@ -2813,11 +2825,11 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - cookie-signature@1.0.6: - resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + cookie-signature@1.0.7: + resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} - cookie@0.7.1: - resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} engines: {node: '>= 0.6'} cosmiconfig-typescript-loader@6.2.0: @@ -3013,8 +3025,8 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.237: - resolution: {integrity: sha512-icUt1NvfhGLar5lSWH3tHNzablaA5js3HVHacQimfP8ViEBOQv+L7DKEuHdbTZ0SKCO1ogTJTIL1Gwk9S6Qvcg==} + electron-to-chromium@1.5.262: + resolution: {integrity: sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ==} embla-carousel-react@8.6.0: resolution: {integrity: sha512-0/PjqU7geVmo6F734pmPqpyHqiM99olvyecY7zdweCw+6tKEXnrE90pBiBbMMU8s5tICemzpQ3hi5EpxzGW+JA==} @@ -3117,8 +3129,8 @@ packages: resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} engines: {node: '>=12.0.0'} - express@4.22.1: - resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} + express@4.22.0: + resolution: {integrity: sha512-c2iPh3xp5vvCLgaHK03+mWLFPhox7j1LwyxcZwFVApEv5i0X+IjPpbT50SJJwwLpdBVfp45AkK/v+AFgv/XlfQ==} engines: {node: '>= 0.10.0'} extend@3.0.2: @@ -3151,8 +3163,8 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} - finalhandler@1.3.1: - resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} + finalhandler@1.3.2: + resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} engines: {node: '>= 0.8'} find-up@7.0.0: @@ -3262,6 +3274,10 @@ packages: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + hasBin: true + global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} engines: {node: '>=18'} @@ -3332,6 +3348,10 @@ packages: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} @@ -3878,8 +3898,8 @@ packages: resolution: {integrity: sha512-tn+OxutdqhvoByKJ7p84FZBSUDfUB76bcvj0ugLBvgE9V52LFcnz8cauCDKi6otnctvFCqa9XkrU35pBY5Baig==} engines: {node: '>=18'} - node-releases@2.0.25: - resolution: {integrity: sha512-4auku8B/vw5psvTiiN9j1dAOsXvMoGqJuKJcR+dTdqiXEK20mMTk1UEo3HS16LeGQsVG6+qKTPM9u/qQ2LqATA==} + node-releases@2.0.27: + resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} npm-run-path@5.3.0: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} @@ -3995,10 +4015,21 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pg-cloudflare@1.2.7: + resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==} + + pg-connection-string@2.9.1: + resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==} + pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} + pg-pool@3.10.1: + resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==} + peerDependencies: + pg: '>=8.0' + pg-protocol@1.10.3: resolution: {integrity: sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==} @@ -4006,6 +4037,18 @@ packages: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} + pg@8.16.3: + resolution: {integrity: sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -4071,10 +4114,6 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} - qs@6.13.0: - resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} - engines: {node: '>=0.6'} - qs@6.14.0: resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==} engines: {node: '>=0.6'} @@ -4086,12 +4125,12 @@ packages: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} - raw-body@2.5.2: - resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} engines: {node: '>= 0.8'} - react-day-picker@9.11.3: - resolution: {integrity: sha512-7lD12UvGbkyXqgzbYIGQTbl+x29B9bAf+k0pP5Dcs1evfpKk6zv4EdH/edNc8NxcmCiTNXr2HIYPrSZ3XvmVBg==} + react-day-picker@9.12.0: + resolution: {integrity: sha512-t8OvG/Zrciso5CQJu5b1A7yzEmebvST+S3pOVQJWxwjjVngyG/CA2htN/D15dLI4uTEuLLkbZyS4YYt480FAtA==} engines: {node: '>=18'} peerDependencies: react: '>=16.8.0' @@ -4362,6 +4401,10 @@ packages: resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} + send@0.19.1: + resolution: {integrity: sha512-p4rRk4f23ynFEfcD9LA0xRYngj+IyGiEYyqqOak8kaN0TvNmuxC2dcVeBn62GpCeR2CpWqyHCNScTP91QbAVFg==} + engines: {node: '>= 0.8.0'} + serve-static@1.16.2: resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} engines: {node: '>= 0.8.0'} @@ -4436,6 +4479,10 @@ packages: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} @@ -4670,6 +4717,9 @@ packages: undici-types@7.14.0: resolution: {integrity: sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==} + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -4678,8 +4728,8 @@ packages: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} - update-browserslist-db@1.1.3: - resolution: {integrity: sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==} + update-browserslist-db@1.1.4: + resolution: {integrity: sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==} hasBin: true peerDependencies: browserslist: '>= 4.21.0' @@ -4970,7 +5020,7 @@ snapshots: js-tokens: 4.0.0 picocolors: 1.1.1 - '@babel/compat-data@7.28.4': {} + '@babel/compat-data@7.28.5': {} '@babel/core@7.28.4': dependencies: @@ -5030,9 +5080,9 @@ snapshots: '@babel/helper-compilation-targets@7.27.2': dependencies: - '@babel/compat-data': 7.28.4 + '@babel/compat-data': 7.28.5 '@babel/helper-validator-option': 7.27.1 - browserslist: 4.26.3 + browserslist: 4.28.0 lru-cache: 5.1.1 semver: 6.3.1 @@ -5040,8 +5090,8 @@ snapshots: '@babel/helper-module-imports@7.27.1': dependencies: - '@babel/traverse': 7.28.4 - '@babel/types': 7.28.4 + '@babel/traverse': 7.28.5 + '@babel/types': 7.28.5 transitivePeerDependencies: - supports-color @@ -5049,8 +5099,8 @@ snapshots: dependencies: '@babel/core': 7.28.4 '@babel/helper-module-imports': 7.27.1 - '@babel/helper-validator-identifier': 7.27.1 - '@babel/traverse': 7.28.4 + '@babel/helper-validator-identifier': 7.28.5 + '@babel/traverse': 7.28.5 transitivePeerDependencies: - supports-color @@ -5058,8 +5108,8 @@ snapshots: dependencies: '@babel/core': 7.28.5 '@babel/helper-module-imports': 7.27.1 - '@babel/helper-validator-identifier': 7.27.1 - '@babel/traverse': 7.28.4 + '@babel/helper-validator-identifier': 7.28.5 + '@babel/traverse': 7.28.5 transitivePeerDependencies: - supports-color @@ -5076,7 +5126,7 @@ snapshots: '@babel/helpers@7.28.4': dependencies: '@babel/template': 7.27.2 - '@babel/types': 7.28.4 + '@babel/types': 7.28.5 '@babel/parser@7.28.4': dependencies: @@ -5111,8 +5161,8 @@ snapshots: '@babel/template@7.27.2': dependencies: '@babel/code-frame': 7.27.1 - '@babel/parser': 7.28.4 - '@babel/types': 7.28.4 + '@babel/parser': 7.28.5 + '@babel/types': 7.28.5 '@babel/traverse@7.28.4': dependencies: @@ -5328,23 +5378,12 @@ snapshots: '@date-fns/tz@1.4.1': {} - '@emnapi/core@1.5.0': - dependencies: - '@emnapi/wasi-threads': 1.1.0 - tslib: 2.8.1 - optional: true - '@emnapi/core@1.7.1': dependencies: '@emnapi/wasi-threads': 1.1.0 tslib: 2.8.1 optional: true - '@emnapi/runtime@1.5.0': - dependencies: - tslib: 2.8.1 - optional: true - '@emnapi/runtime@1.7.1': dependencies: tslib: 2.8.1 @@ -5510,8 +5549,8 @@ snapshots: '@napi-rs/wasm-runtime@1.0.7': dependencies: - '@emnapi/core': 1.5.0 - '@emnapi/runtime': 1.5.0 + '@emnapi/core': 1.7.1 + '@emnapi/runtime': 1.7.1 '@tybys/wasm-util': 0.10.1 optional: true @@ -6992,6 +7031,8 @@ snapshots: '@rolldown/pluginutils@1.0.0-beta.41': {} + '@rolldown/pluginutils@1.0.0-beta.47': {} + '@rolldown/pluginutils@1.0.0-beta.53': {} '@rollup/rollup-android-arm-eabi@4.52.4': @@ -7110,33 +7151,33 @@ snapshots: '@types/babel__core@7.20.5': dependencies: - '@babel/parser': 7.28.4 - '@babel/types': 7.28.4 + '@babel/parser': 7.28.5 + '@babel/types': 7.28.5 '@types/babel__generator': 7.27.0 '@types/babel__template': 7.4.4 '@types/babel__traverse': 7.28.0 '@types/babel__generator@7.27.0': dependencies: - '@babel/types': 7.28.4 + '@babel/types': 7.28.5 '@types/babel__template@7.4.4': dependencies: - '@babel/parser': 7.28.4 - '@babel/types': 7.28.4 + '@babel/parser': 7.28.5 + '@babel/types': 7.28.5 '@types/babel__traverse@7.28.0': dependencies: - '@babel/types': 7.28.4 + '@babel/types': 7.28.5 '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/bunyan@1.8.11': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/chai@5.2.2': dependencies: @@ -7144,7 +7185,7 @@ snapshots: '@types/connect@3.4.38': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/conventional-commits-parser@5.0.1': dependencies: @@ -7180,10 +7221,10 @@ snapshots: '@types/express-serve-static-core@4.19.7': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/qs': 6.14.0 '@types/range-parser': 1.2.7 - '@types/send': 1.2.0 + '@types/send': 1.2.1 '@types/express@4.17.23': dependencies: @@ -7197,7 +7238,7 @@ snapshots: '@types/body-parser': 1.19.6 '@types/express-serve-static-core': 4.19.7 '@types/qs': 6.14.0 - '@types/serve-static': 1.15.9 + '@types/serve-static': 1.15.10 '@types/fined@1.1.5': {} @@ -7215,25 +7256,29 @@ snapshots: '@types/memcached@2.2.10': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/mime@1.3.5': {} '@types/mysql@2.15.27': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/node@20.19.21': dependencies: undici-types: 6.21.0 + '@types/node@24.10.1': + dependencies: + undici-types: 7.16.0 + '@types/node@24.7.2': dependencies: undici-types: 7.14.0 '@types/oracledb@6.5.2': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/pg-pool@2.0.6': dependencies: @@ -7241,7 +7286,7 @@ snapshots: '@types/pg@8.15.6': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 pg-protocol: 1.10.3 pg-types: 2.2.0 @@ -7266,21 +7311,32 @@ snapshots: '@types/send@0.17.5': dependencies: '@types/mime': 1.3.5 - '@types/node': 24.7.2 + '@types/node': 24.10.1 - '@types/send@1.2.0': + '@types/send@0.17.6': dependencies: - '@types/node': 24.7.2 + '@types/mime': 1.3.5 + '@types/node': 24.10.1 + + '@types/send@1.2.1': + dependencies: + '@types/node': 24.10.1 + + '@types/serve-static@1.15.10': + dependencies: + '@types/http-errors': 2.0.5 + '@types/node': 24.10.1 + '@types/send': 0.17.6 '@types/serve-static@1.15.9': dependencies: '@types/http-errors': 2.0.5 - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/send': 0.17.5 '@types/tedious@4.0.14': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@types/through@0.0.33': dependencies: @@ -7288,7 +7344,7 @@ snapshots: '@types/ws@8.18.1': dependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 '@vitejs/plugin-react@5.0.4(rolldown-vite@7.1.14(@types/node@20.19.21)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1))': dependencies: @@ -7314,15 +7370,15 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitejs/plugin-react@5.1.2(rolldown-vite@7.1.14(@types/node@24.7.2)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1))': + '@vitejs/plugin-react@5.1.1(rolldown-vite@7.1.14(@types/node@24.10.1)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1))': dependencies: '@babel/core': 7.28.5 '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.28.5) '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.28.5) - '@rolldown/pluginutils': 1.0.0-beta.53 + '@rolldown/pluginutils': 1.0.0-beta.47 '@types/babel__core': 7.20.5 react-refresh: 0.18.0 - vite: rolldown-vite@7.1.14(@types/node@24.7.2)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1) + vite: rolldown-vite@7.1.14(@types/node@24.10.1)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1) transitivePeerDependencies: - supports-color @@ -7460,7 +7516,7 @@ snapshots: base64-js@1.5.1: {} - baseline-browser-mapping@2.8.16: {} + baseline-browser-mapping@2.8.32: {} bidi-js@1.0.3: dependencies: @@ -7476,18 +7532,18 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - body-parser@1.20.3: + body-parser@1.20.4: dependencies: bytes: 3.1.2 content-type: 1.0.5 debug: 2.6.9 depd: 2.0.0 destroy: 1.2.0 - http-errors: 2.0.0 + http-errors: 2.0.1 iconv-lite: 0.4.24 on-finished: 2.4.1 - qs: 6.13.0 - raw-body: 2.5.2 + qs: 6.14.0 + raw-body: 2.5.3 type-is: 1.6.18 unpipe: 1.0.0 transitivePeerDependencies: @@ -7501,13 +7557,13 @@ snapshots: dependencies: fill-range: 7.1.1 - browserslist@4.26.3: + browserslist@4.28.0: dependencies: - baseline-browser-mapping: 2.8.16 - caniuse-lite: 1.0.30001751 - electron-to-chromium: 1.5.237 - node-releases: 2.0.25 - update-browserslist-db: 1.1.3(browserslist@4.26.3) + baseline-browser-mapping: 2.8.32 + caniuse-lite: 1.0.30001757 + electron-to-chromium: 1.5.262 + node-releases: 2.0.27 + update-browserslist-db: 1.1.4(browserslist@4.28.0) buffer-equal-constant-time@1.0.1: {} @@ -7537,7 +7593,7 @@ snapshots: callsites@3.1.0: {} - caniuse-lite@1.0.30001751: {} + caniuse-lite@1.0.30001757: {} chai@5.3.3: dependencies: @@ -7647,9 +7703,9 @@ snapshots: convert-source-map@2.0.0: {} - cookie-signature@1.0.6: {} + cookie-signature@1.0.7: {} - cookie@0.7.1: {} + cookie@0.7.2: {} cosmiconfig-typescript-loader@6.2.0(@types/node@24.7.2)(cosmiconfig@9.0.0(typescript@5.9.3))(typescript@5.9.3): dependencies: @@ -7804,7 +7860,7 @@ snapshots: ee-first@1.1.1: {} - electron-to-chromium@1.5.237: {} + electron-to-chromium@1.5.262: {} embla-carousel-react@8.6.0(react@19.2.0): dependencies: @@ -7911,23 +7967,23 @@ snapshots: expect-type@1.2.2: {} - express@4.22.1: + express@4.22.0: dependencies: accepts: 1.3.8 array-flatten: 1.1.1 - body-parser: 1.20.3 + body-parser: 1.20.4 content-disposition: 0.5.4 content-type: 1.0.5 - cookie: 0.7.1 - cookie-signature: 1.0.6 + cookie: 0.7.2 + cookie-signature: 1.0.7 debug: 2.6.9 depd: 2.0.0 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 1.3.1 + finalhandler: 1.3.2 fresh: 0.5.2 - http-errors: 2.0.0 + http-errors: 2.0.1 merge-descriptors: 1.0.3 methods: 1.1.2 on-finished: 2.4.1 @@ -7937,10 +7993,10 @@ snapshots: qs: 6.14.0 range-parser: 1.2.1 safe-buffer: 5.2.1 - send: 0.19.0 + send: 0.19.1 serve-static: 1.16.2 setprototypeof: 1.2.0 - statuses: 2.0.1 + statuses: 2.0.2 type-is: 1.6.18 utils-merge: 1.0.1 vary: 1.1.2 @@ -7968,14 +8024,14 @@ snapshots: dependencies: to-regex-range: 5.0.1 - finalhandler@1.3.1: + finalhandler@1.3.2: dependencies: debug: 2.6.9 encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 parseurl: 1.3.3 - statuses: 2.0.1 + statuses: 2.0.2 unpipe: 1.0.0 transitivePeerDependencies: - supports-color @@ -8113,6 +8169,15 @@ snapshots: package-json-from-dist: 1.0.1 path-scurry: 1.11.1 + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + global-directory@4.0.1: dependencies: ini: 4.1.1 @@ -8195,6 +8260,14 @@ snapshots: statuses: 2.0.1 toidentifier: 1.0.1 + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 @@ -8342,7 +8415,7 @@ snapshots: istanbul-lib-instrument@6.0.3: dependencies: - '@babel/core': 7.28.4 + '@babel/core': 7.28.5 '@babel/parser': 7.28.4 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 @@ -8696,7 +8769,7 @@ snapshots: transitivePeerDependencies: - '@types/node' - node-releases@2.0.25: {} + node-releases@2.0.27: {} npm-run-path@5.3.0: dependencies: @@ -8807,8 +8880,17 @@ snapshots: pathval@2.0.1: {} + pg-cloudflare@1.2.7: + optional: true + + pg-connection-string@2.9.1: {} + pg-int8@1.0.1: {} + pg-pool@3.10.1(pg@8.16.3): + dependencies: + pg: 8.16.3 + pg-protocol@1.10.3: {} pg-types@2.2.0: @@ -8819,6 +8901,20 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 + pg@8.16.3: + dependencies: + pg-connection-string: 2.9.1 + pg-pool: 3.10.1(pg@8.16.3) + pg-protocol: 1.10.3 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.2.7 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} picomatch@2.3.1: {} @@ -8880,7 +8976,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 24.7.2 + '@types/node': 24.10.1 long: 5.3.2 proxy-addr@2.0.7: @@ -8897,10 +8993,6 @@ snapshots: punycode@2.3.1: {} - qs@6.13.0: - dependencies: - side-channel: 1.1.0 - qs@6.14.0: dependencies: side-channel: 1.1.0 @@ -8909,14 +9001,14 @@ snapshots: range-parser@1.2.1: {} - raw-body@2.5.2: + raw-body@2.5.3: dependencies: bytes: 3.1.2 - http-errors: 2.0.0 + http-errors: 2.0.1 iconv-lite: 0.4.24 unpipe: 1.0.0 - react-day-picker@9.11.3(react@19.2.0): + react-day-picker@9.12.0(react@19.2.0): dependencies: '@date-fns/tz': 1.4.1 date-fns: 4.1.0 @@ -9066,7 +9158,7 @@ snapshots: rimraf@5.0.10: dependencies: - glob: 10.4.5 + glob: 10.5.0 rolldown-plugin-dts@0.16.11(rolldown@1.0.0-beta.53)(typescript@5.9.3): dependencies: @@ -9103,7 +9195,7 @@ snapshots: tsx: 4.20.6 yaml: 2.8.1 - rolldown-vite@7.1.14(@types/node@24.7.2)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1): + rolldown-vite@7.1.14(@types/node@24.10.1)(esbuild@0.25.10)(jiti@2.6.1)(tsx@4.20.6)(yaml@2.8.1): dependencies: '@oxc-project/runtime': 0.92.0 fdir: 6.5.0(picomatch@4.0.3) @@ -9113,7 +9205,7 @@ snapshots: rolldown: 1.0.0-beta.41 tinyglobby: 0.2.15 optionalDependencies: - '@types/node': 24.7.2 + '@types/node': 24.10.1 esbuild: 0.25.10 fsevents: 2.3.3 jiti: 2.6.1 @@ -9232,6 +9324,24 @@ snapshots: transitivePeerDependencies: - supports-color + send@0.19.1: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + serve-static@1.16.2: dependencies: encodeurl: 2.0.0 @@ -9308,6 +9418,8 @@ snapshots: statuses@2.0.1: {} + statuses@2.0.2: {} + std-env@3.10.0: {} string-argv@0.3.2: {} @@ -9508,13 +9620,15 @@ snapshots: undici-types@7.14.0: {} + undici-types@7.16.0: {} + unicorn-magic@0.1.0: {} unpipe@1.0.0: {} - update-browserslist-db@1.1.3(browserslist@4.26.3): + update-browserslist-db@1.1.4(browserslist@4.28.0): dependencies: - browserslist: 4.26.3 + browserslist: 4.28.0 escalade: 3.2.0 picocolors: 1.1.1 From ac329b358f7133a3b5a083b72533a2d27a709fc1 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 26 Nov 2025 22:49:51 +0000 Subject: [PATCH 02/18] feat: cache manager with persistent and in-memory storage --- .cursor/rules/test-strategy.en.mdc | 91 +++++++ .cursor/rules/v5.en.mdc | 169 +++++++++++++ .../src/analytics/tests/analytics.test.ts | 41 +++ packages/app-kit/src/cache/defaults.ts | 12 + .../app-kit/src/cache/storage/defaults.ts | 15 ++ packages/app-kit/src/cache/storage/index.ts | 3 + .../app-kit/src/cache/storage/lakebase.ts | 235 ++++++++++++++++++ packages/app-kit/src/cache/storage/memory.ts | 107 ++++++++ packages/app-kit/src/cache/storage/types.ts | 27 ++ packages/app-kit/src/connectors/index.ts | 2 +- .../app-kit/src/connectors/lakebase/client.ts | 31 +-- .../app-kit/src/connectors/lakebase/index.ts | 6 - .../app-kit/src/connectors/lakebase/types.ts | 3 - packages/app-kit/src/core/app-kit.ts | 2 + .../app-kit/src/core/tests/databricks.test.ts | 19 ++ packages/app-kit/src/plugin/plugin.ts | 2 +- .../app-kit/src/plugin/tests/cache.test.ts | 121 +++++++-- .../app-kit/src/plugin/tests/plugin.test.ts | 10 +- packages/shared/src/cache.ts | 15 +- principles.md | 23 ++ 20 files changed, 880 insertions(+), 54 deletions(-) create mode 100644 .cursor/rules/test-strategy.en.mdc create mode 100644 .cursor/rules/v5.en.mdc create mode 100644 packages/app-kit/src/cache/defaults.ts create mode 100644 packages/app-kit/src/cache/storage/defaults.ts create mode 100644 packages/app-kit/src/cache/storage/index.ts create mode 100644 packages/app-kit/src/cache/storage/lakebase.ts create mode 100644 packages/app-kit/src/cache/storage/memory.ts create mode 100644 packages/app-kit/src/cache/storage/types.ts create mode 100644 principles.md diff --git a/.cursor/rules/test-strategy.en.mdc b/.cursor/rules/test-strategy.en.mdc new file mode 100644 index 000000000..8783e722c --- /dev/null +++ b/.cursor/rules/test-strategy.en.mdc @@ -0,0 +1,91 @@ +--- +alwaysApply: true +--- + +## Test strategy rules + +These rules define the test process that **must** be followed whenever you implement or modify test code. A test task is **not** considered complete unless all of the steps below are satisfied. + +--- + +## 1. Test perspective table (equivalence partitioning / boundary values) + +1. Before starting any test work, you **must** first present a “test perspectives table” in Markdown table format. +2. The table must include at least the following columns: `Case ID`, `Input / Precondition`, `Perspective (Equivalence / Boundary)`, `Expected Result`, `Notes`. +3. Rows must comprehensively cover normal, abnormal, and boundary cases. For boundary values, you must include `0 / minimum / maximum / ±1 / empty / NULL` at a minimum. + Among the boundary candidates (0 / minimum / maximum / ±1 / empty / NULL), you may omit those that are not meaningful for the given specification, as long as you record in `Notes` why they are out of scope. +4. If you later discover missing perspectives, update the table after self‑review and add the necessary cases. +5. Note: For minor adjustments to existing tests (such as tweaking messages or slightly updating expectations) that do not introduce new branches or constraints, creating or updating a test perspectives table is optional. + +### Template example + +| Case ID | Input / Precondition | Perspective (Equivalence / Boundary) | Expected Result | Notes | +|--------|----------------------|---------------------------------------|----------------------------------------------|-------| +| TC-N-01 | Valid input A | Equivalence – normal | Processing succeeds and returns expected value | - | +| TC-A-01 | NULL | Boundary – NULL | Validation error (required field) | - | +| ... | ... | ... | ... | ... | + +--- + +## 2. Test code implementation policy + +1. Implement **all** cases listed in the table above as automated tests. +2. Ensure you include **at least as many failure cases as success cases** (validation errors, exceptions, external dependency failures, etc.). +3. Your tests must cover the following perspectives: + - Normal paths (main scenarios) + - Abnormal paths (validation errors, exception paths) + - Boundary values (0, minimum, maximum, ±1, empty, NULL) + - Inputs with invalid types or formats + - Failures of external dependencies (e.g. API / DB / messaging, when applicable) + - Exception types and error messages +4. Additionally, aim for 100% branch coverage, and design extra cases yourself as needed to achieve it. + Treat 100% branch coverage as a target. When it is not reasonably achievable, at minimum cover all high‑impact business branches and primary error paths. + If any branches remain uncovered, explicitly document the reasons and impact in `Notes` or the PR description. + +--- + +## 3. Given / When / Then comments + +Each test case must include the following comment format: + +```text +// Given: Preconditions +// When: Operation to execute +// Then: Expected result / assertions +``` + +Place these comments directly above the test code or within the steps so that readers can easily follow the scenario. + +--- + +## 4. Exception and error verification + +1. For cases where exceptions occur, explicitly verify both the **exception type** and the **message**. +2. For validation‑related abnormal cases, also verify error codes and field information, if available. +3. When simulating failures of external dependencies, use stubs/mocks and confirm that the expected exceptions, retries, and fallbacks are invoked. + +--- + +## 5. Execution commands and coverage + +1. At the end of test implementation, always document the **execution command** and **coverage collection method** at the end of the documentation or PR description. + - Examples: `npm run test`, `pnpm vitest run --coverage`, `pytest --cov=...` +2. Check branch coverage and statement coverage, aiming for 100% branch coverage as a target (when it is not reasonably achievable, prioritize covering high‑impact business branches and primary error paths). +3. Attach coverage report results (screenshots or summaries) where reasonably possible. + +--- + +## 6. Operational notes + +1. Any test changes that do not comply with these rules should be sent back during review. +2. Even when there are no external dependencies, you must still include failure cases by **using mocks to simulate failures**. +3. When new branches or constraints are added to the target specification, update both the test perspective table and the test code at the same time. +4. If there are cases that are difficult to automate, clearly document the reasons and alternative measures, and obtain agreement with the reviewer. + The alternative measures must at least describe the affected functionality and risks, the manual verification steps, the expected results, and how logs or screenshots will be recorded. +5. In principle, any PR that includes a meaningful change to production code (such as new features, bug fixes, or refactors that may affect behavior) must also include corresponding additions or updates to automated tests. +6. If adding or updating tests is reasonably difficult, clearly document the reasons and the alternative verification steps (such as manual test procedures) in the PR description and obtain agreement from the reviewer. +7. Even for refactors that are not intended to change behavior, confirm that the changed areas are sufficiently covered by existing tests, and add tests when coverage is insufficient. + +--- + +Always adhere to these rules and continuously self‑check for missing perspectives when designing and implementing tests. diff --git a/.cursor/rules/v5.en.mdc b/.cursor/rules/v5.en.mdc new file mode 100644 index 000000000..072b01aa9 --- /dev/null +++ b/.cursor/rules/v5.en.mdc @@ -0,0 +1,169 @@ +--- +alwaysApply: true +--- + +# v5: Coding support rules + +You are a highly capable AI assistant. This file defines only the behaviour required to achieve maximum productivity and safety for **code‑centric tasks**. +This file provides the foundational rules for carrying out coding‑related tasks. + +--- + +## 0. Common assumptions + +- **Target tasks**: Coding assistance, refactoring, debugging, and authoring development‑related documentation +- **Language**: Follow the language used in the user’s instructions and input (if not explicitly specified, reply in the language the user is using). +- **Rule precedence**: System > Workspace‑common rules > This file (v5) +- **Completion policy**: Do not stop halfway. Keep working persistently until the user’s request is satisfied. If constraints prevent completion, clearly state current progress and remaining tasks. +- **Priority and conflicts between instructions**: Follow the user’s instructions based on system and workspace‑common rules. If instructions conflict or are ambiguous, do not arbitrarily interpret them for convenience; ask a brief clarification before proceeding. +- **User‑specified preferences take precedence**: When the user specifies an output format (bullet list, code only, etc.) or length, treat that preference as higher priority than the defaults in this file. +- **Response style**: + - Avoid excessive preambles; state conclusions and changes first. + - Keep explanations to what is necessary and sufficient, and be especially brief for lightweight tasks. + - Limit example code to only what is needed (avoid huge code blocks). + - Only share deep reasoning processes or long thought logs when the user explicitly asks; otherwise stick to conclusions and key rationales. + +--- + +## 1. Task classification and reasoning depth + +Task classification (🟢/🟡/🔴) and approval conditions follow the workspace‑common rules. +This section only defines **differences in reasoning depth and procedure for coding assistance**. +If the user explicitly requests a different way of working (e.g. “design only first”), prioritize that instruction. + +### 🟢 Lightweight tasks (e.g. small fixes / simple investigation) + +- Examples: A few‑line change in a single file, quick root‑cause check for a bug, checking configuration values. +- Design consultations without code changes, refactor strategy discussions, and general Q&A should also, in principle, be handled as 🟢 tasks with concise answers. +- **Reasoning policy**: + - Avoid deep brainstorming; aim for the shortest path to a solution. + - Do not perform large‑scale design discussions or present a Plan. +- **Execution flow**: + 1. Summarize the task in one line. + 2. Read only the necessary files with `read_file` / `grep`, then immediately apply the fix with `apply_patch`. + 3. Report the result in 1–2 sentences (do not use checklists or detailed templates). + +### 🟡 Standard tasks (e.g. feature additions / small refactors) + +- Examples: Changes spanning multiple files, implementing a single API endpoint, creating a component. +- **Reasoning policy**: + - Present a brief analysis and a "todo list" before implementation. + - Leverage adaptive reasoning while avoiding unnecessarily long thought logs. +- **Execution flow**: + 1. Present 3–7 key subtasks in a checklist. + 2. Read relevant files and apply staged changes with `apply_patch`. + 3. When possible, check for basic errors with `read_lints`. + 4. Finally, summarize in a few sentences **what you changed, in which files, and to what extent**. + +### 🔴 Critical tasks (e.g. architecture/security/cost‑impacting work) + +- Examples: Authentication/authorization changes, DB schema changes, infrastructure changes, modifications likely to affect production. +- **Reasoning policy**: + - First carefully analyze impact and risk, then present a Plan and wait for approval. + - Consider rollback steps and security/cost impact. +- **Execution flow**: + - Always use `create_plan`, and only start work after the user explicitly approves (following the common rules). + +--- + +## 2. Tool usage policy for coding + +### 2.1 Core tools + +- **`read_file`**: Always read relevant files before making changes. For large files, focus on only the necessary ranges. +- **`apply_patch`**: Primary method for code changes. + - When the user asks you to “implement” something, **do not stop at a proposal—actually apply patches** unless there is a blocker. + - Keep each patch to a semantically coherent unit of change. +- **`grep` / `codebase_search`**: + - Use `grep` to locate strings and symbols. + - Use `codebase_search` when searching by meaning or behavior. + +### 2.2 Parallel execution and long‑running operations + +- **`multi_tool_use.parallel`**: + - For read‑only tools like `read_file` / `grep` / `codebase_search` / `web_search`, actively execute them in parallel when there are no dependencies. + - Do not run them in parallel with `apply_patch` or other state‑changing commands. +- **`run_terminal_cmd`**: + - Use only when the user explicitly requests it or when builds/tests are clearly necessary. + - Add non‑interactive flags (e.g. `--yes`) for commands that would otherwise require input. + - For commands that run for a long time, use `is_background: true`. + +### 2.3 Web and browser‑related tools + +- **`web_search`** usage: + - Actively search even without user instruction in cases such as: + - **External services** (models, AI services, clouds) where latest specs/pricing matter + - **Version‑dependent behavior or breaking changes** in libraries/frameworks + - Specific error messages or compatibility issues where built‑in knowledge may be risky + - Only when you actually search, briefly (1–2 sentences) share **what you searched for**. +- **`mcp_cursor-ide-browser_browser_script`** (hereafter `browser_script`): + - Use for checking web app behavior or doing E2E‑like verification. + - Do not start local servers on your own; only do so when instructed by the user. + +### 2.4 Static analysis tools + +- **`read_lints`**: + - For files where you made non‑trivial code changes, check for lint errors when feasible and fix those you can quickly resolve. + +--- + +## 3. Standard flow for coding tasks + +- For any task type, do not leave the flow half‑finished; if constraints prevent completion, clearly indicate “what is done so far and what remains”. + +### 3.1 Lightweight tasks (🟢) + +1. Summarize the task in one line. +2. Check 1–2 related files with `read_file` / `grep`. +3. Immediately fix using `apply_patch`. +4. Perform minimal verification as needed (e.g. visually confirm there are no type errors). +5. Communicate the result in 1–2 sentences. + +### 3.2 Standard tasks (🟡) + +1. Organize the goal, constraints, and expected impact in 2–3 sentences. +2. Present a checklist with about 3–7 items. +3. Read related files and apply changes in multiple passes using `apply_patch`. +4. Use `read_lints` to check for basic errors and fix them on the spot when possible. +5. Finally, concisely summarize what you changed (which files, how they changed, and any known limitations). + +### 3.3 Critical tasks (🔴) + +- Follow the existing rule: `create_plan` → approval → phased execution. +- Break code changes into **small, safe steps**, and check state at each step. +- In `create_plan`, include at least: purpose, expected impact, major risks, and rollback approach (how to revert). + +--- + +## 4. Errors, types, security, and cost + +- **Lint/type errors**: + - Resolve errors you introduced as much as possible on the spot. + - If the root cause is complex and cannot be fixed immediately, explicitly state that, and either revert to a safe state or limit the impact. +- **No `any` / no degradation**: + - Do not add `any` or intentionally degrade features just to “hide” errors. + - Even when a temporary workaround is necessary, briefly explain the rationale and risks. +- **Security / production / cost**: + - Treat changes involving authentication/authorization, network boundaries, data retention, or pricing as “critical tasks”. + - In such cases, present a Plan and obtain user approval before implementation. + +--- + +## 5. Output style and explanation granularity + +- **Lightweight tasks**: + - 1–2 sentence result reports are sufficient. Do not use detailed templates or long text. +- **Standard tasks and above**: + - Use headings (`##` / `###`) and bullet lists to organize changes, impact, and caveats. + - When quoting code, show only the necessary surrounding lines. +- **Code block usage**: + - When quoting existing code, include the file path so it is clear where it comes from. + - For new proposal code, show only the smallest copyable unit. +- **User‑specified preferences take precedence**: + - If the user requests “short”, “longer”, “bullet list”, or “code only”, prioritize that over the defaults here. +- **Disclosure of reasoning process**: + - Only provide deep reasoning logs or long thought processes when the user explicitly asks; by default, stick to conclusions and the main rationale. + +--- + +By following these rules and leveraging adaptive reasoning and the toolset, autonomously execute coding tasks **safely and efficiently**. diff --git a/packages/app-kit/src/analytics/tests/analytics.test.ts b/packages/app-kit/src/analytics/tests/analytics.test.ts index 3eb105a78..9e7b2fbd6 100644 --- a/packages/app-kit/src/analytics/tests/analytics.test.ts +++ b/packages/app-kit/src/analytics/tests/analytics.test.ts @@ -10,12 +10,53 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { AnalyticsPlugin, analytics } from "../analytics"; import type { IAnalyticsConfig } from "../types"; +// Mock CacheManager singleton with actual caching behavior +const { mockCacheStore, mockCacheInstance } = vi.hoisted(() => { + const store = new Map(); + + const generateKey = (parts: unknown[], userKey: string): string => { + const { createHash } = require("crypto"); + const allParts = [userKey, ...parts]; + const serialized = JSON.stringify(allParts); + return createHash("sha256").update(serialized).digest("hex"); + }; + + const instance = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + getOrExecute: vi.fn( + async (key: unknown[], fn: () => Promise, userKey: string) => { + const cacheKey = generateKey(key, userKey); + if (store.has(cacheKey)) { + return store.get(cacheKey); + } + const result = await fn(); + store.set(cacheKey, result); + return result; + }, + ), + generateKey: vi.fn((parts: unknown[], userKey: string) => + generateKey(parts, userKey), + ), + }; + + return { mockCacheStore: store, mockCacheInstance: instance }; +}); + +vi.mock("@databricks-apps/cache", () => ({ + CacheManager: { + getInstanceSync: vi.fn(() => mockCacheInstance), + }, +})); + describe("Analytics Plugin", () => { let config: IAnalyticsConfig; beforeEach(() => { config = { timeout: 5000 }; setupDatabricksEnv(); + mockCacheStore.clear(); }); test("Analytics plugin data should have correct name", () => { diff --git a/packages/app-kit/src/cache/defaults.ts b/packages/app-kit/src/cache/defaults.ts new file mode 100644 index 000000000..492755c7e --- /dev/null +++ b/packages/app-kit/src/cache/defaults.ts @@ -0,0 +1,12 @@ +import type { CacheConfig } from "shared"; + +/** Default configuration for cache */ +export const cacheDefaults: CacheConfig = { + enabled: true, + ttl: 3600, // 1 hour + maxSize: 1000, // 1000 entries + cacheKey: [], // no cache key by default + persistentCache: true, // use lakebase as persistent cache by default + cleanupProbability: 0.01, // 1% probability of triggering cleanup on each get operation + strictPersistence: false, // if false, use in-memory storage if lakebase is unavailable +}; diff --git a/packages/app-kit/src/cache/storage/defaults.ts b/packages/app-kit/src/cache/storage/defaults.ts new file mode 100644 index 000000000..1d345cfe4 --- /dev/null +++ b/packages/app-kit/src/cache/storage/defaults.ts @@ -0,0 +1,15 @@ +/** Default configuration for in-memory storage */ +export const inMemoryStorageDefaults = { + /** Maximum number of entries in the cache */ + maxSize: 1000, +}; + +/** Default configuration for Lakebase storage */ +export const lakebaseStorageDefaults = { + /** Table name for the cache */ + tableName: "appkit_cache_entries", + /** Maximum number of entries in the cache */ + maxSize: 5000, + /** Number of entries to evict when cache is full */ + evictionBatchSize: 100, +}; diff --git a/packages/app-kit/src/cache/storage/index.ts b/packages/app-kit/src/cache/storage/index.ts new file mode 100644 index 000000000..b943af03f --- /dev/null +++ b/packages/app-kit/src/cache/storage/index.ts @@ -0,0 +1,3 @@ +export { LakebaseStorage } from "./lakebase"; +export { InMemoryStorage } from "./memory"; +export type { CacheEntry, CacheStorage } from "./types"; diff --git a/packages/app-kit/src/cache/storage/lakebase.ts b/packages/app-kit/src/cache/storage/lakebase.ts new file mode 100644 index 000000000..fd31cebed --- /dev/null +++ b/packages/app-kit/src/cache/storage/lakebase.ts @@ -0,0 +1,235 @@ +import type { LakebaseConnector } from "../../connectors"; +import type { CacheConfig } from "shared"; +import { lakebaseStorageDefaults } from "./defaults"; +import type { CacheEntry, CacheStorage } from "./types"; + +/** + * Lakebase cache storage implementation. Uses a least recently used (LRU) eviction policy + * to manage memory usage and ensure efficient cache operations. + * + * @example + * const lakebaseStorage = new LakebaseStorage(config, connector); + * await lakebaseStorage.initialize(); + * await lakebaseStorage.get("my-key"); + * await lakebaseStorage.set("my-key", "my-value"); + * await lakebaseStorage.delete("my-key"); + * await lakebaseStorage.clear(); + * await lakebaseStorage.has("my-key"); + * + */ +export class LakebaseStorage implements CacheStorage { + private readonly connector: LakebaseConnector; + private readonly tableName: string; + private readonly maxSize: number; + private readonly evictionBatchSize: number; + private initialized: boolean; + + constructor(config: CacheConfig, connector: LakebaseConnector) { + this.connector = connector; + this.maxSize = config.maxSize ?? lakebaseStorageDefaults.maxSize; + this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize; + this.tableName = lakebaseStorageDefaults.tableName; + this.initialized = false; + } + + /** Initialize the Lakebase storage and run migrations if necessary */ + async initialize(): Promise { + if (this.initialized) return; + + try { + await this.runMigrations(); + this.initialized = true; + } catch (error) { + console.error("Error in for Lakebase storage initialization:", error); + throw error; + } + } + + /** + * Get a cached value from the Lakebase storage + * @param key - Cache key + * @returns Promise of the cached value or null if not found + */ + async get(key: string): Promise | null> { + await this.ensureInitialized(); + + const result = await this.connector.query<{ value: T; expiry: string }>( + `SELECT value, expiry FROM ${this.tableName} WHERE cache_key = $1`, + [key], + ); + + if (result.rows.length === 0) return null; + + const entry = result.rows[0]; + + // fire-and-forget update + this.connector + .query( + `UPDATE ${this.tableName} SET last_accessed = $1 WHERE cache_key = $2`, + [Date.now(), key], + ) + .catch(() => { + console.debug("Error updating last_accessed time for key:", key); + }); + + return { + value: entry.value as T, + expiry: Number(entry.expiry), + }; + } + + /** + * Set a value in the Lakebase storage + * @param key - Cache key + * @param entry - Cache entry + * @returns Promise of the result + */ + async set(key: string, entry: CacheEntry): Promise { + await this.ensureInitialized(); + + const exists = await this.has(key); + if (!exists) { + const currentSize = await this.size(); + if (currentSize >= this.maxSize) { + await this.evictLRU(); + } + } + + await this.connector.query( + `INSERT INTO ${this.tableName} (cache_key, value, expiry, last_accessed) + VALUES ($1, $2, $3, $4) + ON CONFLICT (cache_key) + DO UPDATE SET value = $2, expiry = $3, last_accessed = $4 + `, + [key, JSON.stringify(entry.value), entry.expiry, Date.now()], + ); + } + + /** + * Delete a value from the Lakebase storage + * @param key - Cache key + * @returns Promise of the result + */ + async delete(key: string): Promise { + await this.ensureInitialized(); + await this.connector.query( + `DELETE FROM ${this.tableName} WHERE cache_key = $1`, + [key], + ); + } + + /** Clear the Lakebase storage */ + async clear(): Promise { + await this.ensureInitialized(); + await this.connector.query(`TRUNCATE TABLE ${this.tableName}`); + } + + /** + * Check if a value exists in the Lakebase storage + * @param key - Cache key + * @returns Promise of true if the value exists, false otherwise + */ + async has(key: string): Promise { + await this.ensureInitialized(); + + const result = await this.connector.query<{ exists: boolean }>( + `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE cache_key = $1) as exists`, + [key], + ); + + return result.rows[0]?.exists ?? false; + } + + /** + * Get the size of the Lakebase storage + * @returns Promise of the size of the storage + */ + async size(): Promise { + await this.ensureInitialized(); + + const result = await this.connector.query<{ count: string }>( + `SELECT COUNT(*) as count FROM ${this.tableName}`, + ); + return parseInt(result.rows[0]?.count ?? "0", 10); + } + + /** + * Check if the Lakebase storage is persistent + * @returns true if the storage is persistent, false otherwise + */ + isPersistent(): boolean { + return true; + } + + /** + * Check if the Lakebase storage is healthy + * @returns Promise of true if the storage is healthy, false otherwise + */ + async healthCheck(): Promise { + try { + return await this.connector.healthCheck(); + } catch { + return false; + } + } + + /** Close the Lakebase storage */ + async close(): Promise { + await this.connector.close(); + } + + /** + * Cleanup expired entries from the Lakebase storage + * @returns Promise of the number of expired entries + */ + async cleanupExpired(): Promise { + await this.ensureInitialized(); + const result = await this.connector.query<{ count: string }>( + `WITH deleted as (DELETE FROM ${this.tableName} WHERE expiry < $1 RETURNING *) SELECT COUNT(*) as count FROM deleted`, + [Date.now()], + ); + return parseInt(result.rows[0]?.count ?? "0", 10); + } + + /** Evict the least recently used entries from the Lakebase storage (batched) */ + private async evictLRU(): Promise { + await this.connector.query( + `DELETE FROM ${this.tableName} WHERE cache_key IN ( + SELECT cache_key FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1 + )`, + [this.evictionBatchSize], + ); + } + + /** Ensure the Lakebase storage is initialized */ + private async ensureInitialized(): Promise { + if (!this.initialized) { + await this.initialize(); + } + } + + /** Run migrations for the Lakebase storage */ + private async runMigrations(): Promise { + try { + await this.connector.query(` + CREATE TABLE IF NOT EXISTS ${this.tableName} ( + cache_key VARCHAR(255) PRIMARY KEY, + value JSONB NOT NULL, + expiry BIGINT NOT NULL, + last_accessed BIGINT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + `); + + await this.connector.query(` + CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); + `); + await this.connector.query(` + CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); + `); + } catch (error) { + console.error("Error in running migrations for Lakebase storage:", error); + throw error; + } + } +} diff --git a/packages/app-kit/src/cache/storage/memory.ts b/packages/app-kit/src/cache/storage/memory.ts new file mode 100644 index 000000000..22bf3e883 --- /dev/null +++ b/packages/app-kit/src/cache/storage/memory.ts @@ -0,0 +1,107 @@ +import type { CacheConfig } from "shared"; +import { inMemoryStorageDefaults } from "./defaults"; +import type { CacheEntry, CacheStorage } from "./types"; + +/** + * In-memory cache storage implementation. Uses a least recently used (LRU) eviction policy + * to manage memory usage and ensure efficient cache operations. + */ +export class InMemoryStorage implements CacheStorage { + private cache: Map = new Map(); + private accessOrder: Map = new Map(); + private accessCounter: number; + private maxSize: number; + + constructor(config: CacheConfig) { + this.cache = new Map(); + this.accessOrder = new Map(); + this.maxSize = config.maxSize ?? inMemoryStorageDefaults.maxSize; + this.accessCounter = 0; + } + + /** Get an entry from the cache */ + async get(key: string): Promise | null> { + const entry = this.cache.get(key); + if (!entry) return null; + + this.accessOrder.set(key, ++this.accessCounter); + return entry as CacheEntry; + } + + /** Set an entry in the cache */ + async set(key: string, entry: CacheEntry): Promise { + if (this.cache.size >= this.maxSize && !this.cache.has(key)) { + this.evictLRU(); + } + + this.cache.set(key, entry); + this.accessOrder.set(key, ++this.accessCounter); + } + + /** Delete an entry from the cache */ + async delete(key: string): Promise { + this.cache.delete(key); + this.accessOrder.delete(key); + } + + /** Clean in-memory cache */ + async clear(): Promise { + this.cache.clear(); + this.accessOrder.clear(); + this.accessCounter = 0; + } + + /** Check if the cache has an entry */ + async has(key: string): Promise { + const entry = this.cache.get(key); + if (!entry) return false; + + if (Date.now() > entry.expiry) { + this.cache.delete(key); + this.accessOrder.delete(key); + return false; + } + + return true; + } + + /** Get the size of the cache */ + async size(): Promise { + return this.cache.size; + } + + /** Check if the cache is persistent */ + isPersistent(): boolean { + return false; + } + + /** Check the health of the cache */ + async healthCheck(): Promise { + return true; + } + + /** Close the cache */ + async close(): Promise { + this.cache.clear(); + this.accessOrder.clear(); + this.accessCounter = 0; + } + + /** Evict the least recently used entry (LRU) */ + private evictLRU(): void { + let oldestKey: string | null = null; + let oldestAccess = Infinity; + + for (const [key, accessTime] of this.accessOrder) { + if (accessTime < oldestAccess) { + oldestAccess = accessTime; + oldestKey = key; + } + } + + if (oldestKey) { + this.cache.delete(oldestKey); + this.accessOrder.delete(oldestKey); + } + } +} diff --git a/packages/app-kit/src/cache/storage/types.ts b/packages/app-kit/src/cache/storage/types.ts new file mode 100644 index 000000000..45997b923 --- /dev/null +++ b/packages/app-kit/src/cache/storage/types.ts @@ -0,0 +1,27 @@ +/** Cache entry interface */ +export interface CacheEntry { + value: T; + expiry: number; +} + +/** Cache storage interface */ +export interface CacheStorage { + /** Get a cached value from the storage */ + get(key: string): Promise | null>; + /** Set a value in the storage */ + set(key: string, entry: CacheEntry): Promise; + /** Delete a value from the storage */ + delete(key: string): Promise; + /** Clear the storage */ + clear(): Promise; + /** Check if a value exists in the storage */ + has(key: string): Promise; + /** Get the size of the storage */ + size(): Promise; + /** Check if the storage is persistent */ + isPersistent(): boolean; + /** Check if the storage is healthy */ + healthCheck(): Promise; + /** Close the storage */ + close(): Promise; +} diff --git a/packages/app-kit/src/connectors/index.ts b/packages/app-kit/src/connectors/index.ts index 9d033806b..70702b4b2 100644 --- a/packages/app-kit/src/connectors/index.ts +++ b/packages/app-kit/src/connectors/index.ts @@ -1,2 +1,2 @@ -export * from "./sql-warehouse"; export * from "./lakebase"; +export * from "./sql-warehouse"; diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index 2c552b923..50cb3a9df 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -9,7 +9,6 @@ import type { LakebaseConnectionConfig, LakebaseCredentials, } from "./types"; -import { parseConnectionString } from "./utils"; /** * Enterprise-grade connector for Databricks Lakebase @@ -383,19 +382,7 @@ export class LakebaseConnector { /** Parse connection configuration from config or environment */ private parseConnectionConfig(): LakebaseConnectionConfig { - if (this.config.connectionString) { - const { connectionParams } = parseConnectionString( - this.config.connectionString, - ); - return connectionParams; - } - - const envConnectionString = process.env.LAKEBASE_CONNECTION_STRING; - if (envConnectionString) { - const { connectionParams } = parseConnectionString(envConnectionString); - return connectionParams; - } - + // get connection from config if (this.config.host && this.config.database) { return { host: this.config.host, @@ -405,8 +392,22 @@ export class LakebaseConnector { }; } + // get connection from environment variables + const pgHost = process.env.PGHOST; + const pgDatabase = process.env.PGDATABASE; + const pgPort = process.env.PGPORT; + const pgSslMode = process.env.PGSSLMODE; + if (pgHost && pgDatabase && pgPort && pgSslMode) { + return { + host: pgHost, + database: pgDatabase, + port: Number(pgPort) ?? 5432, + sslMode: (pgSslMode as "require" | "disable" | "prefer") ?? "require", + }; + } + throw new Error( - "Lakebase connection not configured. Either set LAKEBASE_CONNECTION_STRING env var or provide config in constructor.", + "Lakebase connection not configured. Set PGHOST/PGDATABASE env vars or provide host/database in config.", ); } } diff --git a/packages/app-kit/src/connectors/lakebase/index.ts b/packages/app-kit/src/connectors/lakebase/index.ts index 3910d4a40..fe028c409 100644 --- a/packages/app-kit/src/connectors/lakebase/index.ts +++ b/packages/app-kit/src/connectors/lakebase/index.ts @@ -1,8 +1,2 @@ -// Main connector export { LakebaseConnector } from "./client"; - -// Types for user configuration export type { LakebaseConfig } from "./types"; - -// Utilities -export * from "./utils"; diff --git a/packages/app-kit/src/connectors/lakebase/types.ts b/packages/app-kit/src/connectors/lakebase/types.ts index c6e29db25..230b9cf52 100644 --- a/packages/app-kit/src/connectors/lakebase/types.ts +++ b/packages/app-kit/src/connectors/lakebase/types.ts @@ -5,9 +5,6 @@ export interface LakebaseConfig { /** Databricks workspace client */ workspaceClient?: WorkspaceClient; - /** Connection string (postgresql://...) */ - connectionString?: string; - /** Database host (e.g., instance-uuid.database.region.databricks.com) */ host?: string; diff --git a/packages/app-kit/src/core/app-kit.ts b/packages/app-kit/src/core/app-kit.ts index afb2c32ea..ed82ac13f 100644 --- a/packages/app-kit/src/core/app-kit.ts +++ b/packages/app-kit/src/core/app-kit.ts @@ -6,6 +6,7 @@ import type { PluginData, PluginMap, } from "shared"; +import { CacheManager } from "../cache"; import type { TelemetryConfig } from "../telemetry"; import { TelemetryManager } from "../telemetry"; @@ -86,6 +87,7 @@ export class AppKit { config: { plugins?: T; telemetry?: TelemetryConfig } = {}, ): Promise> { TelemetryManager.initialize(config.telemetry); + await CacheManager.getInstance(); const rawPlugins = config.plugins as T; const preparedPlugins = AppKit.preparePlugins(rawPlugins); diff --git a/packages/app-kit/src/core/tests/databricks.test.ts b/packages/app-kit/src/core/tests/databricks.test.ts index 1396a8efe..09d77726e 100644 --- a/packages/app-kit/src/core/tests/databricks.test.ts +++ b/packages/app-kit/src/core/tests/databricks.test.ts @@ -6,6 +6,25 @@ import { createApp, AppKit } from "../app-kit"; // Mock environment validation vi.mock("../utils", () => ({ validateEnv: vi.fn(), + deepMerge: vi.fn((a, b) => ({ ...a, ...b })), +})); + +// Mock CacheManager +vi.mock("@databricks-apps/cache", () => ({ + CacheManager: { + getInstance: vi.fn().mockResolvedValue({ + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + getOrExecute: vi.fn(), + }), + getInstanceSync: vi.fn().mockReturnValue({ + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + getOrExecute: vi.fn(), + }), + }, })); // Test plugin classes for different phases diff --git a/packages/app-kit/src/plugin/plugin.ts b/packages/app-kit/src/plugin/plugin.ts index bd98202f9..ac21317bd 100644 --- a/packages/app-kit/src/plugin/plugin.ts +++ b/packages/app-kit/src/plugin/plugin.ts @@ -51,7 +51,7 @@ export abstract class Plugin< this.name = config.name ?? "plugin"; this.telemetry = TelemetryManager.getProvider(this.name, config.telemetry); this.streamManager = new StreamManager(); - this.cache = new CacheManager(undefined, this.telemetry); + this.cache = CacheManager.getInstanceSync(); this.app = new AppManager(); this.devFileReader = DevFileReader.getInstance(); diff --git a/packages/app-kit/src/plugin/tests/cache.test.ts b/packages/app-kit/src/plugin/tests/cache.test.ts index 8e3b145a9..9c0ec7569 100644 --- a/packages/app-kit/src/plugin/tests/cache.test.ts +++ b/packages/app-kit/src/plugin/tests/cache.test.ts @@ -16,21 +16,75 @@ vi.mock("../../telemetry", () => ({ }, })); +/** Mock CacheManager for testing */ +class MockCacheManager { + private cache = new Map(); + private inFlightRequests = new Map>(); + + async getOrExecute( + key: (string | number | object)[], + fn: () => Promise, + userKey: string, + options?: { ttl?: number }, + ): Promise { + const cacheKey = this.generateKey(key, userKey); + const cached = await this.get(cacheKey); + if (cached !== null) { + return cached; + } + + const inFlight = this.inFlightRequests.get(cacheKey); + if (inFlight) { + return inFlight as Promise; + } + + const promise = fn() + .then(async (result) => { + await this.set(cacheKey, result, options); + return result; + }) + .finally(() => { + this.inFlightRequests.delete(cacheKey); + }); + + this.inFlightRequests.set(cacheKey, promise); + return promise; + } + + async get(key: string): Promise { + const entry = this.cache.get(key); + if (!entry) return null; + + if (Date.now() > entry.expiry) { + this.cache.delete(key); + return null; + } + return entry.value as T; + } + + async set( + key: string, + value: T, + options?: { ttl?: number }, + ): Promise { + const expiryTime = Date.now() + (options?.ttl ?? 3600) * 1000; + this.cache.set(key, { value, expiry: expiryTime }); + } + + generateKey(parts: (string | number | object)[], userKey: string): string { + const { createHash } = require("crypto"); + const allParts = [userKey, ...parts]; + const serialized = JSON.stringify(allParts); + return createHash("sha256").update(serialized).digest("hex"); + } +} + describe("CacheInterceptor", () => { - let cacheManager: CacheManager; + let cacheManager: MockCacheManager; let context: ExecutionContext; beforeEach(() => { - const mockTelemetry = createMockTelemetry(); - vi.mocked(TelemetryManager.getProvider).mockReturnValue( - mockTelemetry as TelemetryProvider, - ); - const telemetry = TelemetryManager.getProvider("cache-test", { - traces: false, - metrics: false, - logs: false, - }); - cacheManager = new CacheManager({}, telemetry); + cacheManager = new MockCacheManager(); context = { metadata: new Map(), userKey: "service", @@ -42,7 +96,10 @@ describe("CacheInterceptor", () => { enabled: false, cacheKey: ["test"], }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); const fn = vi.fn().mockResolvedValue("result"); const result = await interceptor.intercept(fn, context); @@ -56,7 +113,10 @@ describe("CacheInterceptor", () => { enabled: true, cacheKey: [], }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); const fn = vi.fn().mockResolvedValue("result"); const result = await interceptor.intercept(fn, context); @@ -70,11 +130,14 @@ describe("CacheInterceptor", () => { enabled: true, cacheKey: ["test", "key"], }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); // Pre-populate cache const cacheKey = cacheManager.generateKey(["test", "key"], "service"); - cacheManager.set(cacheKey, "cached-result"); + await cacheManager.set(cacheKey, "cached-result"); const fn = vi.fn().mockResolvedValue("new-result"); @@ -90,7 +153,10 @@ describe("CacheInterceptor", () => { cacheKey: ["test", "key"], ttl: 3600, }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); const fn = vi.fn().mockResolvedValue("fresh-result"); const result = await interceptor.intercept(fn, context); @@ -100,7 +166,7 @@ describe("CacheInterceptor", () => { // Verify result was cached const cacheKey = cacheManager.generateKey(["test", "key"], "service"); - const cached = cacheManager.get(cacheKey); + const cached = await cacheManager.get(cacheKey); expect(cached).toBe("fresh-result"); }); @@ -113,14 +179,17 @@ describe("CacheInterceptor", () => { metadata: new Map(), userKey: "user1", }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); const fn = vi.fn().mockResolvedValue("user-result"); await interceptor.intercept(fn, contextWithToken); // Cache key should include userKey const cacheKey = cacheManager.generateKey(["query", "sales"], "user1"); - const cached = cacheManager.get(cacheKey); + const cached = await cacheManager.get(cacheKey); expect(cached).toBe("user-result"); }); @@ -129,7 +198,10 @@ describe("CacheInterceptor", () => { enabled: true, cacheKey: ["query", "profile"], }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); // Service account context const context1: ExecutionContext = { @@ -154,8 +226,8 @@ describe("CacheInterceptor", () => { // Verify separate cache entries const key1 = cacheManager.generateKey(["query", "profile"], "service"); const key2 = cacheManager.generateKey(["query", "profile"], "user1"); - expect(cacheManager.get(key1)).toBe("service-account-data"); - expect(cacheManager.get(key2)).toBe("user-data"); + expect(await cacheManager.get(key1)).toBe("service-account-data"); + expect(await cacheManager.get(key2)).toBe("user-data"); }); test("should respect TTL setting", async () => { @@ -164,7 +236,10 @@ describe("CacheInterceptor", () => { cacheKey: ["test"], ttl: 1, // 1 second }; - const interceptor = new CacheInterceptor(cacheManager, config); + const interceptor = new CacheInterceptor( + cacheManager as unknown as Parameters[0], + config, + ); const fn = vi.fn().mockResolvedValue("result"); await interceptor.intercept(fn, context); diff --git a/packages/app-kit/src/plugin/tests/plugin.test.ts b/packages/app-kit/src/plugin/tests/plugin.test.ts index b4c665d73..b431b209d 100644 --- a/packages/app-kit/src/plugin/tests/plugin.test.ts +++ b/packages/app-kit/src/plugin/tests/plugin.test.ts @@ -17,7 +17,11 @@ import { Plugin } from "../plugin"; // Mock all dependencies vi.mock("../../app"); -vi.mock("../../cache"); +vi.mock("../../cache", () => ({ + CacheManager: { + getInstanceSync: vi.fn(), + }, +})); vi.mock("../../stream"); vi.mock("../../utils", () => ({ validateEnv: vi.fn(), @@ -152,7 +156,7 @@ describe("Plugin", () => { }; // Setup constructor mocks - vi.mocked(CacheManager).mockImplementation(() => mockCache); + vi.mocked(CacheManager.getInstanceSync).mockReturnValue(mockCache); vi.mocked(AppManager).mockImplementation(() => mockApp); vi.mocked(StreamManager).mockImplementation(() => mockStreamManager); vi.mocked(TelemetryManager.getProvider).mockReturnValue( @@ -187,7 +191,7 @@ describe("Plugin", () => { test("should initialize managers", () => { new TestPlugin(config); - expect(CacheManager).toHaveBeenCalledTimes(1); + expect(CacheManager.getInstanceSync).toHaveBeenCalledTimes(1); expect(AppManager).toHaveBeenCalledTimes(1); expect(StreamManager).toHaveBeenCalledTimes(1); }); diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index 0df05c9af..3ece4a5ef 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -1,6 +1,17 @@ +/** Configuration for caching */ export interface CacheConfig { + /** Whether caching is enabled */ enabled?: boolean; - ttl?: number; // time to live in seconds - maxSize?: number; // maximum number of entries in the cache + /** Time to live in seconds */ + ttl?: number; + /** Maximum number of entries in the cache */ + maxSize?: number; + /** Cache key */ cacheKey?: (string | number | object)[]; + /** Whether to use persistent cache */ + persistentCache?: boolean; + /** Whether to enforce strict persistence */ + strictPersistence?: boolean; + + [key: string]: unknown; } diff --git a/principles.md b/principles.md new file mode 100644 index 000000000..34e88452a --- /dev/null +++ b/principles.md @@ -0,0 +1,23 @@ +SDK Core Principles +1. Highly Opinionated + The SDK must provide a clear path with best practices for building Databricks + applications. We provide strong defaults, with advanced customization when needed. +2. Built for Application Use Cases + This SDK is for application development, not infrastructure management. + Databricks' internal implementation details must be abstracted. We're building an + application SDK, not a service wrapper. +3. Delightful Developer Experience + Every interface, doc, example, tool, and implementation must provide developer joy. Combined with the Highly Opinionated principle, this creates a true plug-and-play experience. +4. Zero-Trust Security + Minimize exposed surface area, fail safely by default, and validate all inputs. + The SDK must always have a zero-trust mindset. +5. Optimized for Humans and AI + Developers and LLMs both use this SDK. Every API must be discoverable, + self-documenting, and inferable by both types of users. Test with both. +6. Production-Ready from Day One + Even the smallest feature can be used by enterprise users, so everything + shipped must be production-ready. Observability, reliability, and scalability + since day one. +7. Layered Extensibility +The SDK provides high-level plugins, low-level primitives, and extension points for custom plugins. It integrates into any application architecture and never blocks your path forward. + From 4152de0bb6009674ecb632cca3a048ec862c5cff Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 26 Nov 2025 23:12:48 +0000 Subject: [PATCH 03/18] refactor: lakebase storage to persistent storage --- packages/app-kit/src/cache/storage/index.ts | 2 +- .../storage/{lakebase.ts => persistent.ts} | 53 ++++++++++--------- 2 files changed, 29 insertions(+), 26 deletions(-) rename packages/app-kit/src/cache/storage/{lakebase.ts => persistent.ts} (80%) diff --git a/packages/app-kit/src/cache/storage/index.ts b/packages/app-kit/src/cache/storage/index.ts index b943af03f..2b593d952 100644 --- a/packages/app-kit/src/cache/storage/index.ts +++ b/packages/app-kit/src/cache/storage/index.ts @@ -1,3 +1,3 @@ -export { LakebaseStorage } from "./lakebase"; export { InMemoryStorage } from "./memory"; +export { PersistentStorage } from "./persistent"; export type { CacheEntry, CacheStorage } from "./types"; diff --git a/packages/app-kit/src/cache/storage/lakebase.ts b/packages/app-kit/src/cache/storage/persistent.ts similarity index 80% rename from packages/app-kit/src/cache/storage/lakebase.ts rename to packages/app-kit/src/cache/storage/persistent.ts index fd31cebed..47e5ae6f6 100644 --- a/packages/app-kit/src/cache/storage/lakebase.ts +++ b/packages/app-kit/src/cache/storage/persistent.ts @@ -4,20 +4,20 @@ import { lakebaseStorageDefaults } from "./defaults"; import type { CacheEntry, CacheStorage } from "./types"; /** - * Lakebase cache storage implementation. Uses a least recently used (LRU) eviction policy + * Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy * to manage memory usage and ensure efficient cache operations. * * @example - * const lakebaseStorage = new LakebaseStorage(config, connector); - * await lakebaseStorage.initialize(); - * await lakebaseStorage.get("my-key"); - * await lakebaseStorage.set("my-key", "my-value"); - * await lakebaseStorage.delete("my-key"); - * await lakebaseStorage.clear(); - * await lakebaseStorage.has("my-key"); + * const persistentStorage = new PersistentStorage(config, connector); + * await persistentStorage.initialize(); + * await persistentStorage.get("my-key"); + * await persistentStorage.set("my-key", "my-value"); + * await persistentStorage.delete("my-key"); + * await persistentStorage.clear(); + * await persistentStorage.has("my-key"); * */ -export class LakebaseStorage implements CacheStorage { +export class PersistentStorage implements CacheStorage { private readonly connector: LakebaseConnector; private readonly tableName: string; private readonly maxSize: number; @@ -32,7 +32,7 @@ export class LakebaseStorage implements CacheStorage { this.initialized = false; } - /** Initialize the Lakebase storage and run migrations if necessary */ + /** Initialize the persistent storage and run migrations if necessary */ async initialize(): Promise { if (this.initialized) return; @@ -40,13 +40,13 @@ export class LakebaseStorage implements CacheStorage { await this.runMigrations(); this.initialized = true; } catch (error) { - console.error("Error in for Lakebase storage initialization:", error); + console.error("Error in for persistent storage initialization:", error); throw error; } } /** - * Get a cached value from the Lakebase storage + * Get a cached value from the persistent storage * @param key - Cache key * @returns Promise of the cached value or null if not found */ @@ -79,7 +79,7 @@ export class LakebaseStorage implements CacheStorage { } /** - * Set a value in the Lakebase storage + * Set a value in the persistent storage * @param key - Cache key * @param entry - Cache entry * @returns Promise of the result @@ -106,7 +106,7 @@ export class LakebaseStorage implements CacheStorage { } /** - * Delete a value from the Lakebase storage + * Delete a value from the persistent storage * @param key - Cache key * @returns Promise of the result */ @@ -118,14 +118,14 @@ export class LakebaseStorage implements CacheStorage { ); } - /** Clear the Lakebase storage */ + /** Clear the persistent storage */ async clear(): Promise { await this.ensureInitialized(); await this.connector.query(`TRUNCATE TABLE ${this.tableName}`); } /** - * Check if a value exists in the Lakebase storage + * Check if a value exists in the persistent storage * @param key - Cache key * @returns Promise of true if the value exists, false otherwise */ @@ -141,7 +141,7 @@ export class LakebaseStorage implements CacheStorage { } /** - * Get the size of the Lakebase storage + * Get the size of the persistent storage * @returns Promise of the size of the storage */ async size(): Promise { @@ -154,7 +154,7 @@ export class LakebaseStorage implements CacheStorage { } /** - * Check if the Lakebase storage is persistent + * Check if the persistent storage is persistent * @returns true if the storage is persistent, false otherwise */ isPersistent(): boolean { @@ -162,7 +162,7 @@ export class LakebaseStorage implements CacheStorage { } /** - * Check if the Lakebase storage is healthy + * Check if the persistent storage is healthy * @returns Promise of true if the storage is healthy, false otherwise */ async healthCheck(): Promise { @@ -173,13 +173,13 @@ export class LakebaseStorage implements CacheStorage { } } - /** Close the Lakebase storage */ + /** Close the persistent storage */ async close(): Promise { await this.connector.close(); } /** - * Cleanup expired entries from the Lakebase storage + * Cleanup expired entries from the persistent storage * @returns Promise of the number of expired entries */ async cleanupExpired(): Promise { @@ -191,7 +191,7 @@ export class LakebaseStorage implements CacheStorage { return parseInt(result.rows[0]?.count ?? "0", 10); } - /** Evict the least recently used entries from the Lakebase storage (batched) */ + /** Evict the least recently used entries from the persistent storage (batched) */ private async evictLRU(): Promise { await this.connector.query( `DELETE FROM ${this.tableName} WHERE cache_key IN ( @@ -201,14 +201,14 @@ export class LakebaseStorage implements CacheStorage { ); } - /** Ensure the Lakebase storage is initialized */ + /** Ensure the persistent storage is initialized */ private async ensureInitialized(): Promise { if (!this.initialized) { await this.initialize(); } } - /** Run migrations for the Lakebase storage */ + /** Run migrations for the persistent storage */ private async runMigrations(): Promise { try { await this.connector.query(` @@ -228,7 +228,10 @@ export class LakebaseStorage implements CacheStorage { CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `); } catch (error) { - console.error("Error in running migrations for Lakebase storage:", error); + console.error( + "Error in running migrations for persistent storage:", + error, + ); throw error; } } From 3986e0e0cd8bd20ea75bcd2a0d2dbf9fec02c33f Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 26 Nov 2025 23:48:32 +0000 Subject: [PATCH 04/18] refactor: get lakebase app name connector from env var --- .../app-kit/src/connectors/lakebase/client.ts | 77 ++++++++----------- .../app-kit/src/connectors/lakebase/types.ts | 5 ++ 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index 50cb3a9df..0ebbc1144 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -285,31 +285,12 @@ export class LakebaseConnector { /** Fetch password (OAuth token) from Databricks */ private async fetchPassword(): Promise { - const host = this.connectionConfig.host; - - const uid = host.split(".")[0]?.replace("instance-", ""); - if (!uid) { - throw new Error( - `Invalid lakebase hostname: ${host}. Expected format: instance-.database..databricks.com`, - ); - } - const workspaceClient = this.getWorkspaceClient(); const config = new Config({ host: workspaceClient.config.host }); const apiClient = new ApiClient(config); - // find database instance - const dbInfo = await apiClient.request({ - path: `/api/2.0/database/instances:findByUid`, - method: "GET", - query: { uid }, - payload: { uid }, - headers: new Headers(), - raw: false, - }); - - if (!this.hasName(dbInfo)) { - throw new Error(`Database instance not found for uid: ${uid}`); + if (!this.connectionConfig.appName) { + throw new Error(`Database app name not found in connection config`); } const credentials = await apiClient.request({ @@ -318,14 +299,14 @@ export class LakebaseConnector { headers: new Headers(), raw: false, payload: { - instance_names: [dbInfo.name], + instance_names: [this.connectionConfig.appName], request_id: randomUUID(), }, }); if (!this.hasToken(credentials)) { throw new Error( - `Failed to generate credentials for instance: ${dbInfo.name}`, + `Failed to generate credentials for instance: ${this.connectionConfig.appName}`, ); } @@ -360,16 +341,6 @@ export class LakebaseConnector { ); } - /** Type guard for database instance */ - private hasName(value: unknown): value is { name: string } { - return ( - typeof value === "object" && - value !== null && - "name" in value && - typeof (value as any).name === "string" - ); - } - /** Type guard for credentials */ private hasToken(value: unknown): value is { token: string } { return ( @@ -383,31 +354,43 @@ export class LakebaseConnector { /** Parse connection configuration from config or environment */ private parseConnectionConfig(): LakebaseConnectionConfig { // get connection from config - if (this.config.host && this.config.database) { + if (this.config.host && this.config.database && this.config.appName) { return { host: this.config.host, database: this.config.database, - port: this.config.port, - sslMode: this.config.sslMode, + port: this.config.port ?? 5432, + sslMode: this.config.sslMode ?? "require", + appName: this.config.appName, }; } // get connection from environment variables const pgHost = process.env.PGHOST; const pgDatabase = process.env.PGDATABASE; + const pgAppName = process.env.PGAPPNAME; + if (!pgHost || !pgDatabase || !pgAppName) { + throw new Error( + "Lakebase connection not configured. Required env vars: PGHOST, PGDATABASE, PGAPPNAME. " + + "Optional: PGPORT (default: 5432), PGSSLMODE (default: require).", + ); + } const pgPort = process.env.PGPORT; - const pgSslMode = process.env.PGSSLMODE; - if (pgHost && pgDatabase && pgPort && pgSslMode) { - return { - host: pgHost, - database: pgDatabase, - port: Number(pgPort) ?? 5432, - sslMode: (pgSslMode as "require" | "disable" | "prefer") ?? "require", - }; + const port = pgPort ? parseInt(pgPort, 10) : 5432; + + if (Number.isNaN(port)) { + throw new Error(`Invalid port: ${pgPort}. Must be a number.`); } - throw new Error( - "Lakebase connection not configured. Set PGHOST/PGDATABASE env vars or provide host/database in config.", - ); + const pgSSLMode = process.env.PGSSLMODE; + const sslMode = + (pgSSLMode as "require" | "disable" | "prefer") || "require"; + + return { + host: pgHost, + database: pgDatabase, + port, + sslMode, + appName: pgAppName, + }; } } diff --git a/packages/app-kit/src/connectors/lakebase/types.ts b/packages/app-kit/src/connectors/lakebase/types.ts index 230b9cf52..8b945a452 100644 --- a/packages/app-kit/src/connectors/lakebase/types.ts +++ b/packages/app-kit/src/connectors/lakebase/types.ts @@ -14,6 +14,9 @@ export interface LakebaseConfig { /** Database port */ port: number; + /** App name */ + appName?: string; + /** SSL mode */ sslMode: "require" | "disable" | "prefer"; @@ -53,4 +56,6 @@ export interface LakebaseConnectionConfig { readonly port: number; /** SSL mode */ readonly sslMode: "require" | "disable" | "prefer"; + /** App name */ + readonly appName?: string; } From bc596d1d03ce1a92c95daccc4ac31f897cbccce9 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Thu, 27 Nov 2025 11:27:48 +0000 Subject: [PATCH 05/18] test: add cache storage tests --- .../src/analytics/tests/analytics.test.ts | 10 +- .../app-kit/src/plugin/tests/cache.test.ts | 30 +- .../backend/cache/tests/cache-manager.test.ts | 348 +++++++++++++ packages/backend/cache/tests/memory.test.ts | 233 +++++++++ .../backend/cache/tests/persistent.test.ts | 413 +++++++++++++++ .../backend/connectors/tests/lakebase.test.ts | 489 ++++++++++++++++++ 6 files changed, 1510 insertions(+), 13 deletions(-) create mode 100644 packages/backend/cache/tests/cache-manager.test.ts create mode 100644 packages/backend/cache/tests/memory.test.ts create mode 100644 packages/backend/cache/tests/persistent.test.ts create mode 100644 packages/backend/connectors/tests/lakebase.test.ts diff --git a/packages/app-kit/src/analytics/tests/analytics.test.ts b/packages/app-kit/src/analytics/tests/analytics.test.ts index 9e7b2fbd6..10a3b4e69 100644 --- a/packages/app-kit/src/analytics/tests/analytics.test.ts +++ b/packages/app-kit/src/analytics/tests/analytics.test.ts @@ -15,7 +15,7 @@ const { mockCacheStore, mockCacheInstance } = vi.hoisted(() => { const store = new Map(); const generateKey = (parts: unknown[], userKey: string): string => { - const { createHash } = require("crypto"); + const { createHash } = require("node:crypto"); const allParts = [userKey, ...parts]; const serialized = JSON.stringify(allParts); return createHash("sha256").update(serialized).digest("hex"); @@ -221,7 +221,7 @@ describe("Analytics Plugin", () => { }, { userDatabricksClient: mockUserClient as any, - userName: "user-token-123", + userId: "user-token-123", }, ); @@ -318,7 +318,7 @@ describe("Analytics Plugin", () => { async () => { await handler(mockReq1, mockRes1); }, - { userName: "user-token-1" }, + { userId: "user-token-1" }, ); const mockReq2 = createMockRequest({ @@ -331,7 +331,7 @@ describe("Analytics Plugin", () => { async () => { await handler(mockReq2, mockRes2); }, - { userName: "user-token-2" }, + { userId: "user-token-2" }, ); const mockReq1Again = createMockRequest({ @@ -344,7 +344,7 @@ describe("Analytics Plugin", () => { async () => { await handler(mockReq1Again, mockRes1Again); }, - { userName: "user-token-1" }, + { userId: "user-token-1" }, ); expect(executeMock).toHaveBeenCalledTimes(2); diff --git a/packages/app-kit/src/plugin/tests/cache.test.ts b/packages/app-kit/src/plugin/tests/cache.test.ts index 9c0ec7569..daee2c208 100644 --- a/packages/app-kit/src/plugin/tests/cache.test.ts +++ b/packages/app-kit/src/plugin/tests/cache.test.ts @@ -72,7 +72,7 @@ class MockCacheManager { } generateKey(parts: (string | number | object)[], userKey: string): string { - const { createHash } = require("crypto"); + const { createHash } = require("node:crypto"); const allParts = [userKey, ...parts]; const serialized = JSON.stringify(allParts); return createHash("sha256").update(serialized).digest("hex"); @@ -97,7 +97,9 @@ describe("CacheInterceptor", () => { cacheKey: ["test"], }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); const fn = vi.fn().mockResolvedValue("result"); @@ -114,7 +116,9 @@ describe("CacheInterceptor", () => { cacheKey: [], }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); const fn = vi.fn().mockResolvedValue("result"); @@ -131,7 +135,9 @@ describe("CacheInterceptor", () => { cacheKey: ["test", "key"], }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); @@ -154,7 +160,9 @@ describe("CacheInterceptor", () => { ttl: 3600, }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); const fn = vi.fn().mockResolvedValue("fresh-result"); @@ -180,7 +188,9 @@ describe("CacheInterceptor", () => { userKey: "user1", }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); const fn = vi.fn().mockResolvedValue("user-result"); @@ -199,7 +209,9 @@ describe("CacheInterceptor", () => { cacheKey: ["query", "profile"], }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); @@ -237,7 +249,9 @@ describe("CacheInterceptor", () => { ttl: 1, // 1 second }; const interceptor = new CacheInterceptor( - cacheManager as unknown as Parameters[0], + cacheManager as unknown as ConstructorParameters< + typeof CacheInterceptor + >[0], config, ); const fn = vi.fn().mockResolvedValue("result"); diff --git a/packages/backend/cache/tests/cache-manager.test.ts b/packages/backend/cache/tests/cache-manager.test.ts new file mode 100644 index 000000000..b2ba13aef --- /dev/null +++ b/packages/backend/cache/tests/cache-manager.test.ts @@ -0,0 +1,348 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { CacheManager } from "../src/index"; +import type { CacheStorage } from "../src/storage/types"; + +// Mock the storage modules +vi.mock("../src/storage/memory", () => ({ + InMemoryStorage: vi.fn().mockImplementation(() => createMockStorage()), +})); + +vi.mock("../src/storage/persistent", () => ({ + PersistentStorage: vi.fn().mockImplementation(() => { + const storage = createMockStorage(); + storage.isPersistent = vi.fn().mockReturnValue(true); + return storage; + }), +})); + +// Mock LakebaseConnector +vi.mock("@databricks-apps/connectors", () => ({ + LakebaseConnector: vi.fn().mockImplementation(() => ({ + healthCheck: vi.fn().mockResolvedValue(true), + close: vi.fn().mockResolvedValue(undefined), + })), +})); + +// Mock WorkspaceClient +vi.mock("@databricks/sdk-experimental", () => ({ + WorkspaceClient: vi.fn().mockImplementation(() => ({})), +})); + +/** Create a mock storage for testing */ +function createMockStorage(): CacheStorage { + const cache = new Map(); + + return { + get: vi.fn().mockImplementation(async (key: string) => { + return cache.get(key) || null; + }), + set: vi.fn().mockImplementation(async (key: string, entry: any) => { + cache.set(key, entry); + }), + delete: vi.fn().mockImplementation(async (key: string) => { + cache.delete(key); + }), + clear: vi.fn().mockImplementation(async () => { + cache.clear(); + }), + has: vi.fn().mockImplementation(async (key: string) => { + return cache.has(key); + }), + size: vi.fn().mockImplementation(async () => { + return cache.size; + }), + isPersistent: vi.fn().mockReturnValue(false), + healthCheck: vi.fn().mockResolvedValue(true), + close: vi.fn().mockResolvedValue(undefined), + }; +} + +describe("CacheManager", () => { + // Reset singleton between tests + beforeEach(() => { + // Access private static fields to reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe("singleton pattern", () => { + test("getInstanceSync should throw when not initialized", () => { + expect(() => CacheManager.getInstanceSync()).toThrow( + "CacheManager not initialized", + ); + }); + + test("getInstance should create singleton", async () => { + const instance1 = await CacheManager.getInstance({ + persistentCache: false, + }); + const instance2 = await CacheManager.getInstance(); + + expect(instance1).toBe(instance2); + }); + + test("getInstanceSync should return instance after initialization", async () => { + await CacheManager.getInstance({ persistentCache: false }); + + const instance = CacheManager.getInstanceSync(); + + expect(instance).toBeInstanceOf(CacheManager); + }); + }); + + describe("generateKey", () => { + test("should generate consistent hash for same inputs", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + const key1 = cache.generateKey(["users", 123], "user1"); + const key2 = cache.generateKey(["users", 123], "user1"); + + expect(key1).toBe(key2); + }); + + test("should generate different hash for different inputs", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + const key1 = cache.generateKey(["users", 123], "user1"); + const key2 = cache.generateKey(["users", 456], "user1"); + const key3 = cache.generateKey(["users", 123], "user2"); + + expect(key1).not.toBe(key2); + expect(key1).not.toBe(key3); + expect(key2).not.toBe(key3); + }); + + test("should handle objects in key parts", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + const key1 = cache.generateKey([{ filter: "active" }], "user1"); + const key2 = cache.generateKey([{ filter: "active" }], "user1"); + const key3 = cache.generateKey([{ filter: "inactive" }], "user1"); + + expect(key1).toBe(key2); + expect(key1).not.toBe(key3); + }); + }); + + describe("get/set operations", () => { + test("should return null for non-existent key", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + const result = await cache.get("non-existent"); + + expect(result).toBeNull(); + }); + + test("should set and get value", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await cache.set("test-key", { data: "test-value" }); + const result = await cache.get("test-key"); + + expect(result).toEqual({ data: "test-value" }); + }); + + test("should respect TTL expiry", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + // Set with very short TTL + await cache.set("test-key", "value", { ttl: 0.001 }); // 1ms + + // Wait for expiry + await new Promise((resolve) => setTimeout(resolve, 10)); + + const result = await cache.get("test-key"); + + expect(result).toBeNull(); + }); + }); + + describe("delete operation", () => { + test("should delete existing key", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await cache.set("test-key", "value"); + await cache.delete("test-key"); + + const result = await cache.get("test-key"); + expect(result).toBeNull(); + }); + }); + + describe("has operation", () => { + test("should return true for existing key", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await cache.set("test-key", "value"); + + const exists = await cache.has("test-key"); + expect(exists).toBe(true); + }); + + test("should return false for non-existent key", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + const exists = await cache.has("non-existent"); + expect(exists).toBe(false); + }); + + test("should return false for expired key", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await cache.set("test-key", "value", { ttl: 0.001 }); + await new Promise((resolve) => setTimeout(resolve, 10)); + + const exists = await cache.has("test-key"); + expect(exists).toBe(false); + }); + }); + + describe("clear operation", () => { + test("should clear all entries", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await cache.set("key1", "value1"); + await cache.set("key2", "value2"); + + await cache.clear(); + + expect(await cache.get("key1")).toBeNull(); + expect(await cache.get("key2")).toBeNull(); + }); + }); + + describe("getOrExecute", () => { + test("should execute function on cache miss", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + const fn = vi.fn().mockResolvedValue("result"); + + const result = await cache.getOrExecute(["key"], fn, "user1"); + + expect(result).toBe("result"); + expect(fn).toHaveBeenCalledTimes(1); + }); + + test("should return cached value on cache hit", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + const fn = vi.fn().mockResolvedValue("new-result"); + + // First call - populates cache + await cache.getOrExecute(["key"], async () => "cached-result", "user1"); + + // Second call - should use cache + const result = await cache.getOrExecute(["key"], fn, "user1"); + + expect(result).toBe("cached-result"); + expect(fn).not.toHaveBeenCalled(); + }); + + test("should deduplicate concurrent requests", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + let callCount = 0; + const fn = vi.fn().mockImplementation(async () => { + callCount++; + await new Promise((resolve) => setTimeout(resolve, 50)); + return `result-${callCount}`; + }); + + // Fire multiple concurrent requests + const promises = [ + cache.getOrExecute(["key"], fn, "user1"), + cache.getOrExecute(["key"], fn, "user1"), + cache.getOrExecute(["key"], fn, "user1"), + ]; + + const results = await Promise.all(promises); + + // All should return same result + expect(results[0]).toBe(results[1]); + expect(results[1]).toBe(results[2]); + // Function should only be called once + expect(fn).toHaveBeenCalledTimes(1); + }); + + test("should use different cache keys for different users", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await cache.getOrExecute(["key"], async () => "user1-data", "user1"); + await cache.getOrExecute(["key"], async () => "user2-data", "user2"); + + const result1 = await cache.getOrExecute( + ["key"], + async () => "should-not-be-called", + "user1", + ); + const result2 = await cache.getOrExecute( + ["key"], + async () => "should-not-be-called", + "user2", + ); + + expect(result1).toBe("user1-data"); + expect(result2).toBe("user2-data"); + }); + }); + + describe("disabled cache", () => { + test("should bypass cache when disabled", async () => { + const cache = await CacheManager.getInstance({ + enabled: false, + persistentCache: false, + }); + const fn = vi.fn().mockResolvedValue("result"); + + const result1 = await cache.getOrExecute(["key"], fn, "user1"); + const result2 = await cache.getOrExecute(["key"], fn, "user1"); + + expect(result1).toBe("result"); + expect(result2).toBe("result"); + expect(fn).toHaveBeenCalledTimes(2); // Called twice because cache is disabled + }); + + test("should return null for get when disabled", async () => { + const cache = await CacheManager.getInstance({ + enabled: false, + persistentCache: false, + }); + + await cache.set("test-key", "value"); + const result = await cache.get("test-key"); + + expect(result).toBeNull(); + }); + + test("should return false for has when disabled", async () => { + const cache = await CacheManager.getInstance({ + enabled: false, + persistentCache: false, + }); + + await cache.set("test-key", "value"); + const exists = await cache.has("test-key"); + + expect(exists).toBe(false); + }); + }); + + describe("storage health", () => { + test("should check storage health", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + const isHealthy = await cache.isStorageHealthy(); + + expect(isHealthy).toBe(true); + }); + }); + + describe("close", () => { + test("should close storage", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + + await expect(cache.close()).resolves.not.toThrow(); + }); + }); +}); diff --git a/packages/backend/cache/tests/memory.test.ts b/packages/backend/cache/tests/memory.test.ts new file mode 100644 index 000000000..6d80f0013 --- /dev/null +++ b/packages/backend/cache/tests/memory.test.ts @@ -0,0 +1,233 @@ +import { beforeEach, describe, expect, test } from "vitest"; +import { InMemoryStorage } from "../src/storage/memory"; + +describe("InMemoryStorage", () => { + let storage: InMemoryStorage; + + beforeEach(() => { + storage = new InMemoryStorage({ maxSize: 5 }); + }); + + describe("basic operations", () => { + test("should set and get a value", async () => { + const entry = { value: "test-value", expiry: Date.now() + 10000 }; + await storage.set("key1", entry); + + const result = await storage.get("key1"); + + expect(result).toEqual(entry); + }); + + test("should return null for non-existent key", async () => { + const result = await storage.get("non-existent"); + + expect(result).toBeNull(); + }); + + test("should delete a value", async () => { + const entry = { value: "test-value", expiry: Date.now() + 10000 }; + await storage.set("key1", entry); + + await storage.delete("key1"); + + const result = await storage.get("key1"); + expect(result).toBeNull(); + }); + + test("should check if key exists", async () => { + const entry = { value: "test-value", expiry: Date.now() + 10000 }; + await storage.set("key1", entry); + + expect(await storage.has("key1")).toBe(true); + expect(await storage.has("non-existent")).toBe(false); + }); + + test("should return correct size", async () => { + expect(await storage.size()).toBe(0); + + await storage.set("key1", { value: "v1", expiry: Date.now() + 10000 }); + expect(await storage.size()).toBe(1); + + await storage.set("key2", { value: "v2", expiry: Date.now() + 10000 }); + expect(await storage.size()).toBe(2); + }); + + test("should clear all entries", async () => { + await storage.set("key1", { value: "v1", expiry: Date.now() + 10000 }); + await storage.set("key2", { value: "v2", expiry: Date.now() + 10000 }); + + await storage.clear(); + + expect(await storage.size()).toBe(0); + expect(await storage.get("key1")).toBeNull(); + expect(await storage.get("key2")).toBeNull(); + }); + }); + + describe("expiry handling", () => { + test("should return entry if not expired", async () => { + const entry = { value: "test-value", expiry: Date.now() + 10000 }; + await storage.set("key1", entry); + + const result = await storage.get("key1"); + + expect(result).toEqual(entry); + }); + + test("should return expired entry on get (expiry check is done by CacheManager)", async () => { + const entry = { value: "test-value", expiry: Date.now() - 1000 }; + await storage.set("key1", entry); + + // InMemoryStorage.get() returns the entry even if expired + // The expiry check is done by CacheManager + const result = await storage.get("key1"); + expect(result).toEqual(entry); + }); + + test("should return false for expired key on has()", async () => { + const entry = { value: "test-value", expiry: Date.now() - 1000 }; + await storage.set("key1", entry); + + const exists = await storage.has("key1"); + + expect(exists).toBe(false); + }); + + test("should delete expired entry when checking has()", async () => { + const entry = { value: "test-value", expiry: Date.now() - 1000 }; + await storage.set("key1", entry); + + await storage.has("key1"); + + // Entry should be deleted after has() check + expect(await storage.size()).toBe(0); + }); + }); + + describe("LRU eviction", () => { + test("should evict least recently used entry when full", async () => { + // Fill storage to capacity (maxSize = 5) + for (let i = 1; i <= 5; i++) { + await storage.set(`key${i}`, { + value: `value${i}`, + expiry: Date.now() + 10000, + }); + } + + expect(await storage.size()).toBe(5); + + // Add one more entry, should evict key1 (least recently used) + await storage.set("key6", { + value: "value6", + expiry: Date.now() + 10000, + }); + + expect(await storage.size()).toBe(5); + expect(await storage.get("key1")).toBeNull(); // evicted + expect(await storage.get("key6")).not.toBeNull(); // new entry exists + }); + + test("should update access order on get", async () => { + // Fill storage + for (let i = 1; i <= 5; i++) { + await storage.set(`key${i}`, { + value: `value${i}`, + expiry: Date.now() + 10000, + }); + } + + // Access key1 to make it recently used + await storage.get("key1"); + + // Add new entry, should evict key2 (now least recently used) + await storage.set("key6", { + value: "value6", + expiry: Date.now() + 10000, + }); + + expect(await storage.get("key1")).not.toBeNull(); // still exists (was accessed) + expect(await storage.get("key2")).toBeNull(); // evicted + }); + + test("should update access order on set (existing key)", async () => { + // Fill storage + for (let i = 1; i <= 5; i++) { + await storage.set(`key${i}`, { + value: `value${i}`, + expiry: Date.now() + 10000, + }); + } + + // Update key1 to make it recently used + await storage.set("key1", { + value: "updated-value1", + expiry: Date.now() + 10000, + }); + + // Add new entry, should evict key2 (now least recently used) + await storage.set("key6", { + value: "value6", + expiry: Date.now() + 10000, + }); + + expect(await storage.get("key1")).not.toBeNull(); // still exists (was updated) + expect(await storage.get("key2")).toBeNull(); // evicted + }); + + test("should not evict when updating existing key", async () => { + // Fill storage + for (let i = 1; i <= 5; i++) { + await storage.set(`key${i}`, { + value: `value${i}`, + expiry: Date.now() + 10000, + }); + } + + // Update existing key should not trigger eviction + await storage.set("key3", { + value: "updated-value3", + expiry: Date.now() + 10000, + }); + + expect(await storage.size()).toBe(5); + // All keys should still exist + for (let i = 1; i <= 5; i++) { + expect(await storage.get(`key${i}`)).not.toBeNull(); + } + }); + }); + + describe("storage properties", () => { + test("should report as non-persistent", () => { + expect(storage.isPersistent()).toBe(false); + }); + + test("should always return true for healthCheck", async () => { + expect(await storage.healthCheck()).toBe(true); + }); + + test("should clear storage on close", async () => { + await storage.set("key1", { value: "v1", expiry: Date.now() + 10000 }); + + await storage.close(); + + expect(await storage.size()).toBe(0); + }); + }); + + describe("default maxSize", () => { + test("should use default maxSize when not provided", async () => { + const defaultStorage = new InMemoryStorage({}); + + // Default is 1000, just verify it accepts many entries + for (let i = 1; i <= 100; i++) { + await defaultStorage.set(`key${i}`, { + value: `value${i}`, + expiry: Date.now() + 10000, + }); + } + + expect(await defaultStorage.size()).toBe(100); + }); + }); +}); diff --git a/packages/backend/cache/tests/persistent.test.ts b/packages/backend/cache/tests/persistent.test.ts new file mode 100644 index 000000000..86eea2b0e --- /dev/null +++ b/packages/backend/cache/tests/persistent.test.ts @@ -0,0 +1,413 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { PersistentStorage } from "../src/storage/persistent"; + +/** Mock LakebaseConnector for testing */ +const createMockConnector = () => ({ + query: vi.fn(), + healthCheck: vi.fn().mockResolvedValue(true), + close: vi.fn().mockResolvedValue(undefined), +}); + +describe("PersistentStorage", () => { + let storage: PersistentStorage; + let mockConnector: ReturnType; + + beforeEach(() => { + mockConnector = createMockConnector(); + + // Default: migrations succeed + mockConnector.query.mockResolvedValue({ rows: [] }); + + storage = new PersistentStorage({ maxSize: 100 }, mockConnector as any); + }); + + describe("initialization", () => { + test("should run migrations on initialize", async () => { + await storage.initialize(); + + // Should create table + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("CREATE TABLE IF NOT EXISTS"), + ); + + // Should create indexes + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("CREATE INDEX IF NOT EXISTS"), + ); + }); + + test("should only initialize once", async () => { + await storage.initialize(); + await storage.initialize(); + + // CREATE TABLE should only be called once (first initialization) + const createTableCalls = mockConnector.query.mock.calls.filter((call) => + call[0].includes("CREATE TABLE"), + ); + expect(createTableCalls.length).toBe(1); + }); + + test("should throw on migration error", async () => { + const consoleErrorSpy = vi + .spyOn(console, "error") + .mockImplementation(() => {}); + + mockConnector.query.mockRejectedValue(new Error("migration failed")); + + await expect(storage.initialize()).rejects.toThrow("migration failed"); + + consoleErrorSpy.mockRestore(); + }); + }); + + describe("get", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should return cached entry", async () => { + const expiry = Date.now() + 10000; + mockConnector.query.mockResolvedValueOnce({ + rows: [{ value: { data: "test" }, expiry: String(expiry) }], + }); + + const result = await storage.get("test-key"); + + expect(result).toEqual({ + value: { data: "test" }, + expiry, + }); + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("SELECT value, expiry"), + ["test-key"], + ); + }); + + test("should return null for non-existent key", async () => { + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + const result = await storage.get("non-existent"); + + expect(result).toBeNull(); + }); + + test("should update last_accessed on get (fire-and-forget)", async () => { + const expiry = Date.now() + 10000; + mockConnector.query + .mockResolvedValueOnce({ + rows: [{ value: { data: "test" }, expiry: String(expiry) }], + }) + .mockResolvedValue({ rows: [] }); + + await storage.get("test-key"); + + // Wait for fire-and-forget update + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("UPDATE"), + expect.arrayContaining([expect.any(Number), "test-key"]), + ); + }); + }); + + describe("set", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should insert new entry", async () => { + // has() returns false + mockConnector.query.mockResolvedValueOnce({ + rows: [{ exists: false }], + }); + // size() returns 0 + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "0" }], + }); + // INSERT succeeds + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + await storage.set("test-key", { + value: { data: "test" }, + expiry: Date.now() + 10000, + }); + + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("INSERT INTO"), + expect.arrayContaining(["test-key", expect.any(String)]), + ); + }); + + test("should update existing entry", async () => { + // has() returns true + mockConnector.query.mockResolvedValueOnce({ + rows: [{ exists: true }], + }); + // INSERT/UPDATE succeeds + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + await storage.set("test-key", { + value: { data: "updated" }, + expiry: Date.now() + 10000, + }); + + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("ON CONFLICT"), + expect.any(Array), + ); + }); + + test("should evict LRU when full", async () => { + // has() returns false + mockConnector.query.mockResolvedValueOnce({ + rows: [{ exists: false }], + }); + // size() returns maxSize (100) + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "100" }], + }); + // DELETE (eviction) succeeds + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + // INSERT succeeds + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + await storage.set("new-key", { + value: { data: "new" }, + expiry: Date.now() + 10000, + }); + + // Should have called DELETE for LRU eviction + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("DELETE FROM"), + expect.any(Array), + ); + }); + + test("should serialize value as JSON", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ exists: false }], + }); + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "0" }], + }); + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + const value = { nested: { array: [1, 2, 3] } }; + await storage.set("test-key", { + value, + expiry: Date.now() + 10000, + }); + + const insertCall = mockConnector.query.mock.calls.find((call) => + call[0].includes("INSERT"), + ); + + expect(insertCall?.[1]?.[1]).toBe(JSON.stringify(value)); + }); + }); + + describe("delete", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should delete entry", async () => { + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + await storage.delete("test-key"); + + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("DELETE FROM"), + ["test-key"], + ); + }); + }); + + describe("clear", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should truncate table", async () => { + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + await storage.clear(); + + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("TRUNCATE TABLE"), + ); + }); + }); + + describe("has", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should return true when key exists", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ exists: true }], + }); + + const result = await storage.has("test-key"); + + expect(result).toBe(true); + }); + + test("should return false when key does not exist", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ exists: false }], + }); + + const result = await storage.has("non-existent"); + + expect(result).toBe(false); + }); + + test("should return false when query returns no rows", async () => { + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + const result = await storage.has("test-key"); + + expect(result).toBe(false); + }); + }); + + describe("size", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should return count of entries", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "42" }], + }); + + const result = await storage.size(); + + expect(result).toBe(42); + }); + + test("should return 0 when empty", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "0" }], + }); + + const result = await storage.size(); + + expect(result).toBe(0); + }); + + test("should return 0 when no rows", async () => { + mockConnector.query.mockResolvedValueOnce({ rows: [] }); + + const result = await storage.size(); + + expect(result).toBe(0); + }); + }); + + describe("cleanupExpired", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should delete expired entries", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "5" }], + }); + + const deleted = await storage.cleanupExpired(); + + expect(deleted).toBe(5); + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("DELETE FROM"), + expect.arrayContaining([expect.any(Number)]), + ); + }); + + test("should return 0 when no expired entries", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ count: "0" }], + }); + + const deleted = await storage.cleanupExpired(); + + expect(deleted).toBe(0); + }); + }); + + describe("storage properties", () => { + test("should report as persistent", () => { + expect(storage.isPersistent()).toBe(true); + }); + + test("should delegate healthCheck to connector", async () => { + mockConnector.healthCheck.mockResolvedValueOnce(true); + + const result = await storage.healthCheck(); + + expect(result).toBe(true); + expect(mockConnector.healthCheck).toHaveBeenCalled(); + }); + + test("should return false on healthCheck error", async () => { + mockConnector.healthCheck.mockRejectedValueOnce(new Error("failed")); + + const result = await storage.healthCheck(); + + expect(result).toBe(false); + }); + + test("should close connector on close", async () => { + await storage.close(); + + expect(mockConnector.close).toHaveBeenCalled(); + }); + }); + + describe("auto-initialization", () => { + test("should auto-initialize on get if not initialized", async () => { + const uninitializedStorage = new PersistentStorage( + { maxSize: 100 }, + mockConnector as any, + ); + + mockConnector.query.mockResolvedValue({ rows: [] }); + + await uninitializedStorage.get("test-key"); + + // Should have run migrations + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("CREATE TABLE"), + ); + }); + + test("should auto-initialize on set if not initialized", async () => { + const uninitializedStorage = new PersistentStorage( + { maxSize: 100 }, + mockConnector as any, + ); + + mockConnector.query.mockResolvedValue({ rows: [] }); + + await uninitializedStorage.set("test-key", { + value: "test", + expiry: Date.now() + 10000, + }); + + // Should have run migrations + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("CREATE TABLE"), + ); + }); + }); +}); diff --git a/packages/backend/connectors/tests/lakebase.test.ts b/packages/backend/connectors/tests/lakebase.test.ts new file mode 100644 index 000000000..969d2b4cc --- /dev/null +++ b/packages/backend/connectors/tests/lakebase.test.ts @@ -0,0 +1,489 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { LakebaseConnector } from "../src/lakebase/client"; + +// Mock pg module +vi.mock("pg", () => { + const mockQuery = vi.fn(); + const mockConnect = vi.fn(); + const mockEnd = vi.fn().mockResolvedValue(undefined); + const mockRelease = vi.fn(); + const mockOn = vi.fn(); + + const MockPool = vi.fn(() => ({ + query: mockQuery, + connect: mockConnect, + end: mockEnd, + on: mockOn, + })); + + return { + default: { Pool: MockPool }, + Pool: MockPool, + __mockQuery: mockQuery, + __mockConnect: mockConnect, + __mockEnd: mockEnd, + __mockRelease: mockRelease, + __mockOn: mockOn, + __MockPool: MockPool, + }; +}); + +// Mock Databricks SDK +vi.mock("@databricks/sdk-experimental", () => { + const mockMe = vi.fn(); + const mockRequest = vi.fn(); + + const MockWorkspaceClient = vi.fn(() => ({ + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + })); + + const MockApiClient = vi.fn(() => ({ + request: mockRequest, + })); + + const MockConfig = vi.fn(() => ({})); + + return { + WorkspaceClient: MockWorkspaceClient, + ApiClient: MockApiClient, + Config: MockConfig, + __mockMe: mockMe, + __mockRequest: mockRequest, + __MockWorkspaceClient: MockWorkspaceClient, + __MockApiClient: MockApiClient, + }; +}); + +describe("LakebaseConnector", () => { + beforeEach(() => { + vi.clearAllMocks(); + // Set required env vars + process.env.PGHOST = "test-host.databricks.com"; + process.env.PGDATABASE = "test-db"; + process.env.PGAPPNAME = "test-app"; + }); + + describe("configuration", () => { + test("should throw error when maxPoolSize is less than 1", () => { + expect( + () => + new LakebaseConnector({ + maxPoolSize: 0, + workspaceClient: {} as any, + }), + ).toThrow("maxPoolSize must be at least 1"); + }); + + test("should throw error when credentialTTLMs is less than 60 seconds", () => { + expect( + () => + new LakebaseConnector({ + credentialTTLMs: 30_000, + workspaceClient: {} as any, + }), + ).toThrow("credentialTTLMs must be at least 60 seconds"); + }); + + test("should create connector with valid config", () => { + const connector = new LakebaseConnector({ + workspaceClient: {} as any, + }); + + expect(connector).toBeInstanceOf(LakebaseConnector); + }); + + test("should throw when env vars are missing", () => { + delete process.env.PGHOST; + delete process.env.PGDATABASE; + delete process.env.PGAPPNAME; + + expect(() => new LakebaseConnector()).toThrow( + "Lakebase connection not configured", + ); + }); + + test("should throw when PGPORT is invalid", () => { + process.env.PGPORT = "invalid"; + + expect(() => new LakebaseConnector()).toThrow("Invalid port"); + }); + + test("should parse env vars correctly", () => { + process.env.PGPORT = "5433"; + process.env.PGSSLMODE = "disable"; + + const connector = new LakebaseConnector(); + + expect(connector).toBeInstanceOf(LakebaseConnector); + }); + + test("should use explicit config over env vars", () => { + const connector = new LakebaseConnector({ + host: "explicit-host.databricks.com", + database: "explicit-db", + appName: "explicit-app", + port: 5434, + sslMode: "prefer", + workspaceClient: {} as any, + }); + + expect(connector).toBeInstanceOf(LakebaseConnector); + }); + }); + + describe("query", () => { + let connector: LakebaseConnector; + let mockQuery: ReturnType; + let mockMe: ReturnType; + let mockRequest: ReturnType; + + beforeEach(async () => { + const pg = await import("pg"); + const sdk = await import("@databricks/sdk-experimental"); + + mockQuery = (pg as any).__mockQuery; + mockMe = (sdk as any).__mockMe; + mockRequest = (sdk as any).__mockRequest; + + // Setup default mocks + mockMe.mockResolvedValue({ userName: "test-user@example.com" }); + mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockQuery.mockResolvedValue({ rows: [{ result: 1 }] }); + + connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + }); + + test("should execute query successfully", async () => { + const result = await connector.query("SELECT 1 as result"); + + expect(result.rows).toEqual([{ result: 1 }]); + expect(mockQuery).toHaveBeenCalledWith("SELECT 1 as result", undefined); + }); + + test("should execute query with parameters", async () => { + mockQuery.mockResolvedValue({ rows: [{ id: 1, name: "test" }] }); + + const result = await connector.query( + "SELECT * FROM users WHERE id = $1", + [1], + ); + + expect(result.rows).toEqual([{ id: 1, name: "test" }]); + expect(mockQuery).toHaveBeenCalledWith( + "SELECT * FROM users WHERE id = $1", + [1], + ); + }); + + test("should retry on auth error (28P01)", async () => { + const authError = new Error("auth failed") as any; + authError.code = "28P01"; + + mockQuery + .mockRejectedValueOnce(authError) + .mockResolvedValue({ rows: [{ result: 1 }] }); + + const result = await connector.query("SELECT 1"); + + expect(result.rows).toEqual([{ result: 1 }]); + expect(mockQuery).toHaveBeenCalledTimes(2); + }); + + test("should retry once on transient error", async () => { + const transientError = new Error("connection reset") as any; + transientError.code = "ECONNRESET"; + + mockQuery + .mockRejectedValueOnce(transientError) + .mockResolvedValue({ rows: [{ result: 1 }] }); + + const result = await connector.query("SELECT 1"); + + expect(result.rows).toEqual([{ result: 1 }]); + expect(mockQuery).toHaveBeenCalledTimes(2); + }); + + test("should not retry transient error more than once", async () => { + const transientError = new Error("connection reset") as any; + transientError.code = "ECONNRESET"; + + mockQuery.mockRejectedValue(transientError); + + await expect(connector.query("SELECT 1")).rejects.toThrow( + "connection reset", + ); + expect(mockQuery).toHaveBeenCalledTimes(2); + }); + + test("should throw non-retriable errors immediately", async () => { + const syntaxError = new Error("syntax error") as any; + syntaxError.code = "42601"; + + mockQuery.mockRejectedValue(syntaxError); + + await expect(connector.query("SELEC 1")).rejects.toThrow("syntax error"); + expect(mockQuery).toHaveBeenCalledTimes(1); + }); + }); + + describe("transaction", () => { + let connector: LakebaseConnector; + let mockConnect: ReturnType; + let mockMe: ReturnType; + let mockRequest: ReturnType; + + beforeEach(async () => { + const pg = await import("pg"); + const sdk = await import("@databricks/sdk-experimental"); + + mockConnect = (pg as any).__mockConnect; + mockMe = (sdk as any).__mockMe; + mockRequest = (sdk as any).__mockRequest; + + mockMe.mockResolvedValue({ userName: "test-user@example.com" }); + mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + + const mockClient = { + query: vi.fn().mockResolvedValue({ rows: [] }), + release: vi.fn(), + }; + mockConnect.mockResolvedValue(mockClient); + + connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + }); + + test("should execute transaction successfully", async () => { + const result = await connector.transaction(async (client) => { + await client.query("BEGIN"); + await client.query("INSERT INTO test VALUES (1)"); + await client.query("COMMIT"); + return "success"; + }); + + expect(result).toBe("success"); + }); + + test("should release client after transaction", async () => { + const mockClient = { + query: vi.fn().mockResolvedValue({ rows: [] }), + release: vi.fn(), + }; + mockConnect.mockResolvedValue(mockClient); + + await connector.transaction(async (client) => { + await client.query("SELECT 1"); + return "done"; + }); + + expect(mockClient.release).toHaveBeenCalled(); + }); + }); + + describe("healthCheck", () => { + let connector: LakebaseConnector; + let mockQuery: ReturnType; + let mockMe: ReturnType; + let mockRequest: ReturnType; + + beforeEach(async () => { + const pg = await import("pg"); + const sdk = await import("@databricks/sdk-experimental"); + + mockQuery = (pg as any).__mockQuery; + mockMe = (sdk as any).__mockMe; + mockRequest = (sdk as any).__mockRequest; + + mockMe.mockResolvedValue({ userName: "test-user@example.com" }); + mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + + connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + }); + + test("should return true when database is healthy", async () => { + mockQuery.mockResolvedValue({ rows: [{ result: 1 }] }); + + const isHealthy = await connector.healthCheck(); + + expect(isHealthy).toBe(true); + }); + + test("should return false when database is unhealthy", async () => { + mockQuery.mockRejectedValue(new Error("connection failed")); + + const isHealthy = await connector.healthCheck(); + + expect(isHealthy).toBe(false); + }); + + test("should return false when result is unexpected", async () => { + mockQuery.mockResolvedValue({ rows: [{ result: 0 }] }); + + const isHealthy = await connector.healthCheck(); + + expect(isHealthy).toBe(false); + }); + }); + + describe("close", () => { + let connector: LakebaseConnector; + let mockEnd: ReturnType; + let mockQuery: ReturnType; + let mockMe: ReturnType; + let mockRequest: ReturnType; + + beforeEach(async () => { + const pg = await import("pg"); + const sdk = await import("@databricks/sdk-experimental"); + + mockEnd = (pg as any).__mockEnd; + mockQuery = (pg as any).__mockQuery; + mockMe = (sdk as any).__mockMe; + mockRequest = (sdk as any).__mockRequest; + + mockMe.mockResolvedValue({ userName: "test-user@example.com" }); + mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockQuery.mockResolvedValue({ rows: [{ result: 1 }] }); + mockEnd.mockResolvedValue(undefined); + + connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + }); + + test("should close connection pool", async () => { + // Initialize pool by making a query + await connector.query("SELECT 1"); + + await connector.close(); + + expect(mockEnd).toHaveBeenCalled(); + }); + + test("should handle close when pool not initialized", async () => { + // Don't make any queries, pool is not initialized + await expect(connector.close()).resolves.not.toThrow(); + }); + }); + + describe("credentials", () => { + let mockMe: ReturnType; + let mockRequest: ReturnType; + let mockQuery: ReturnType; + + beforeEach(async () => { + const pg = await import("pg"); + const sdk = await import("@databricks/sdk-experimental"); + + mockQuery = (pg as any).__mockQuery; + mockMe = (sdk as any).__mockMe; + mockRequest = (sdk as any).__mockRequest; + + mockQuery.mockResolvedValue({ rows: [{ result: 1 }] }); + }); + + test("should throw when username cannot be fetched", async () => { + mockMe.mockResolvedValue({ userName: null }); + mockRequest.mockResolvedValue({ token: "test-token" }); + + const connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + + await expect(connector.query("SELECT 1")).rejects.toThrow( + "Failed to get current user", + ); + }); + + test("should throw when token cannot be fetched", async () => { + mockMe.mockResolvedValue({ userName: "test-user@example.com" }); + mockRequest.mockResolvedValue({ error: "unauthorized" }); + + const connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + + await expect(connector.query("SELECT 1")).rejects.toThrow( + "Failed to generate credentials", + ); + }); + }); + + describe("transient error codes", () => { + let connector: LakebaseConnector; + let mockQuery: ReturnType; + let mockMe: ReturnType; + let mockRequest: ReturnType; + + beforeEach(async () => { + const pg = await import("pg"); + const sdk = await import("@databricks/sdk-experimental"); + + mockQuery = (pg as any).__mockQuery; + mockMe = (sdk as any).__mockMe; + mockRequest = (sdk as any).__mockRequest; + + mockMe.mockResolvedValue({ userName: "test-user@example.com" }); + mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + + connector = new LakebaseConnector({ + workspaceClient: { + currentUser: { me: mockMe }, + config: { host: "https://test.databricks.com" }, + } as any, + }); + }); + + const transientCodes = [ + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "57P01", // admin_shutdown + "57P03", // cannot_connect_now + "08006", // connection_failure + "08003", // connection_does_not_exist + "08000", // connection_exception + ]; + + test.each(transientCodes)( + "should retry on transient error code: %s", + async (code) => { + const error = new Error(`transient error ${code}`) as any; + error.code = code; + + mockQuery + .mockRejectedValueOnce(error) + .mockResolvedValue({ rows: [{ result: 1 }] }); + + const result = await connector.query("SELECT 1"); + + expect(result.rows).toEqual([{ result: 1 }]); + expect(mockQuery).toHaveBeenCalledTimes(2); + }, + ); + }); +}); From 725718a04c07b2af2e2f1259c4fa2a2690d95cc5 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Thu, 27 Nov 2025 12:14:20 +0000 Subject: [PATCH 06/18] refactor: optimized persistent cache schema --- package.json | 1 + .../app-kit/src/cache/storage/defaults.ts | 4 +- .../app-kit/src/cache/storage/persistent.ts | 137 +++++++++++++----- .../backend/cache/tests/persistent.test.ts | 125 +++++++++------- packages/shared/src/cache.ts | 4 +- pnpm-lock.yaml | 3 + 6 files changed, 182 insertions(+), 92 deletions(-) diff --git a/package.json b/package.json index 2278b90e1..23d8532d2 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "jsdom": "^27.0.0", "lint-staged": "^15.5.1", "plop": "^4.0.4", + "pg": "^8.16.3", "publint": "^0.3.15", "tsdown": "^0.15.7", "tsx": "^4.20.6", diff --git a/packages/app-kit/src/cache/storage/defaults.ts b/packages/app-kit/src/cache/storage/defaults.ts index 1d345cfe4..0268beff3 100644 --- a/packages/app-kit/src/cache/storage/defaults.ts +++ b/packages/app-kit/src/cache/storage/defaults.ts @@ -8,8 +8,8 @@ export const inMemoryStorageDefaults = { export const lakebaseStorageDefaults = { /** Table name for the cache */ tableName: "appkit_cache_entries", - /** Maximum number of entries in the cache */ - maxSize: 5000, + /** Maximum number of bytes in the cache */ + maxBytes: 256 * 1024 * 1024, // 256MB /** Number of entries to evict when cache is full */ evictionBatchSize: 100, }; diff --git a/packages/app-kit/src/cache/storage/persistent.ts b/packages/app-kit/src/cache/storage/persistent.ts index 47e5ae6f6..e04891310 100644 --- a/packages/app-kit/src/cache/storage/persistent.ts +++ b/packages/app-kit/src/cache/storage/persistent.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import type { LakebaseConnector } from "../../connectors"; import type { CacheConfig } from "shared"; import { lakebaseStorageDefaults } from "./defaults"; @@ -20,13 +21,13 @@ import type { CacheEntry, CacheStorage } from "./types"; export class PersistentStorage implements CacheStorage { private readonly connector: LakebaseConnector; private readonly tableName: string; - private readonly maxSize: number; + private readonly maxBytes: number; private readonly evictionBatchSize: number; private initialized: boolean; constructor(config: CacheConfig, connector: LakebaseConnector) { this.connector = connector; - this.maxSize = config.maxSize ?? lakebaseStorageDefaults.maxSize; + this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes; this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize; this.tableName = lakebaseStorageDefaults.tableName; this.initialized = false; @@ -53,10 +54,14 @@ export class PersistentStorage implements CacheStorage { async get(key: string): Promise | null> { await this.ensureInitialized(); - const result = await this.connector.query<{ value: T; expiry: string }>( - `SELECT value, expiry FROM ${this.tableName} WHERE cache_key = $1`, - [key], - ); + const keyHash = this.hashKey(key); + + const result = await this.connector.query<{ + value: Buffer; + expiry: string; + }>(`SELECT value, expiry FROM ${this.tableName} WHERE key_hash = $1`, [ + keyHash, + ]); if (result.rows.length === 0) return null; @@ -65,15 +70,15 @@ export class PersistentStorage implements CacheStorage { // fire-and-forget update this.connector .query( - `UPDATE ${this.tableName} SET last_accessed = $1 WHERE cache_key = $2`, - [Date.now(), key], + `UPDATE ${this.tableName} SET last_accessed = NOW() WHERE key_hash = $1`, + [keyHash], ) .catch(() => { console.debug("Error updating last_accessed time for key:", key); }); return { - value: entry.value as T, + value: this.deserializeValue(entry.value), expiry: Number(entry.expiry), }; } @@ -87,21 +92,23 @@ export class PersistentStorage implements CacheStorage { async set(key: string, entry: CacheEntry): Promise { await this.ensureInitialized(); - const exists = await this.has(key); - if (!exists) { - const currentSize = await this.size(); - if (currentSize >= this.maxSize) { - await this.evictLRU(); - } + const keyHash = this.hashKey(key); + const keyBytes = Buffer.from(key, "utf-8"); + const valueBytes = this.serializeValue(entry.value); + const byteSize = keyBytes.length + valueBytes.length; + + const totalBytes = await this.totalBytes(); + if (totalBytes + byteSize > this.maxBytes) { + await this.evictBySize(byteSize); } await this.connector.query( - `INSERT INTO ${this.tableName} (cache_key, value, expiry, last_accessed) - VALUES ($1, $2, $3, $4) - ON CONFLICT (cache_key) - DO UPDATE SET value = $2, expiry = $3, last_accessed = $4 + `INSERT INTO ${this.tableName} (key_hash, key, value, byte_size, expiry, created_at, last_accessed) + VALUES ($1, $2, $3, $4, $5, NOW(), NOW()) + ON CONFLICT (key_hash) + DO UPDATE SET value = $3, byte_size = $4, expiry = $5, last_accessed = NOW() `, - [key, JSON.stringify(entry.value), entry.expiry, Date.now()], + [keyHash, keyBytes, valueBytes, byteSize, entry.expiry], ); } @@ -112,9 +119,10 @@ export class PersistentStorage implements CacheStorage { */ async delete(key: string): Promise { await this.ensureInitialized(); + const keyHash = this.hashKey(key); await this.connector.query( - `DELETE FROM ${this.tableName} WHERE cache_key = $1`, - [key], + `DELETE FROM ${this.tableName} WHERE key_hash = $1`, + [keyHash], ); } @@ -131,10 +139,11 @@ export class PersistentStorage implements CacheStorage { */ async has(key: string): Promise { await this.ensureInitialized(); + const keyHash = this.hashKey(key); const result = await this.connector.query<{ exists: boolean }>( - `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE cache_key = $1) as exists`, - [key], + `SELECT EXISTS(SELECT 1 FROM ${this.tableName} WHERE key_hash = $1) as exists`, + [keyHash], ); return result.rows[0]?.exists ?? false; @@ -153,6 +162,16 @@ export class PersistentStorage implements CacheStorage { return parseInt(result.rows[0]?.count ?? "0", 10); } + /** Get the total number of bytes in the persistent storage */ + async totalBytes(): Promise { + await this.ensureInitialized(); + + const result = await this.connector.query<{ total: string }>( + `SELECT COALESCE(SUM(byte_size), 0) as total FROM ${this.tableName}`, + ); + return parseInt(result.rows[0]?.total ?? "0", 10); + } + /** * Check if the persistent storage is persistent * @returns true if the storage is persistent, false otherwise @@ -191,12 +210,19 @@ export class PersistentStorage implements CacheStorage { return parseInt(result.rows[0]?.count ?? "0", 10); } - /** Evict the least recently used entries from the persistent storage (batched) */ - private async evictLRU(): Promise { + /** Evict entries from the persistent storage by size */ + private async evictBySize(requiredBytes: number): Promise { + const freedByExpiry = await this.cleanupExpired(); + if (freedByExpiry > 0) { + const currentBytes = await this.totalBytes(); + if (currentBytes + requiredBytes <= this.maxBytes) { + return; + } + } + await this.connector.query( - `DELETE FROM ${this.tableName} WHERE cache_key IN ( - SELECT cache_key FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1 - )`, + `DELETE FROM ${this.tableName} WHERE key_hash IN + (SELECT key_hash FROM ${this.tableName} ORDER BY last_accessed ASC LIMIT $1)`, [this.evictionBatchSize], ); } @@ -208,25 +234,58 @@ export class PersistentStorage implements CacheStorage { } } + /** Generate a 64-bit hash for the cache key using SHA256 */ + private hashKey(key: string): bigint { + if (!key) throw new Error("Cache key cannot be empty"); + const hash = createHash("sha256").update(key).digest(); + return hash.readBigInt64BE(0); + } + + /** Serialize a value to a buffer */ + private serializeValue(value: T): Buffer { + return Buffer.from(JSON.stringify(value), "utf-8"); + } + + /** Deserialize a value from a buffer */ + private deserializeValue(buffer: Buffer): T { + return JSON.parse(buffer.toString("utf-8")) as T; + } + /** Run migrations for the persistent storage */ private async runMigrations(): Promise { try { await this.connector.query(` CREATE TABLE IF NOT EXISTS ${this.tableName} ( - cache_key VARCHAR(255) PRIMARY KEY, - value JSONB NOT NULL, + id BIGSERIAL PRIMARY KEY, + key_hash BIGINT NOT NULL, + key BYTEA NOT NULL, + value BYTEA NOT NULL, + byte_size INTEGER NOT NULL, expiry BIGINT NOT NULL, - last_accessed BIGINT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + last_accessed TIMESTAMP NOT NULL DEFAULT NOW() ) `); - await this.connector.query(` - CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); - `); - await this.connector.query(` - CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); - `); + // unique index on key_hash for fast lookups + await this.connector.query( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.tableName}_key_hash ON ${this.tableName} (key_hash);`, + ); + + // index on expiry for cleanup queries + await this.connector.query( + `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_expiry ON ${this.tableName} (expiry); `, + ); + + // index on last_accessed for LRU eviction + await this.connector.query( + `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_last_accessed ON ${this.tableName} (last_accessed); `, + ); + + // index on byte_size for monitoring + await this.connector.query( + `CREATE INDEX IF NOT EXISTS idx_${this.tableName}_byte_size ON ${this.tableName} (byte_size); `, + ); } catch (error) { console.error( "Error in running migrations for persistent storage:", diff --git a/packages/backend/cache/tests/persistent.test.ts b/packages/backend/cache/tests/persistent.test.ts index 86eea2b0e..0d3ae957c 100644 --- a/packages/backend/cache/tests/persistent.test.ts +++ b/packages/backend/cache/tests/persistent.test.ts @@ -18,7 +18,10 @@ describe("PersistentStorage", () => { // Default: migrations succeed mockConnector.query.mockResolvedValue({ rows: [] }); - storage = new PersistentStorage({ maxSize: 100 }, mockConnector as any); + storage = new PersistentStorage( + { maxBytes: 1024 * 1024 }, // 1MB + mockConnector as any, + ); }); describe("initialization", () => { @@ -30,9 +33,9 @@ describe("PersistentStorage", () => { expect.stringContaining("CREATE TABLE IF NOT EXISTS"), ); - // Should create indexes + // Should create unique index on key_hash expect(mockConnector.query).toHaveBeenCalledWith( - expect.stringContaining("CREATE INDEX IF NOT EXISTS"), + expect.stringContaining("CREATE UNIQUE INDEX IF NOT EXISTS"), ); }); @@ -68,8 +71,13 @@ describe("PersistentStorage", () => { test("should return cached entry", async () => { const expiry = Date.now() + 10000; + const valueBuffer = Buffer.from( + JSON.stringify({ data: "test" }), + "utf-8", + ); + mockConnector.query.mockResolvedValueOnce({ - rows: [{ value: { data: "test" }, expiry: String(expiry) }], + rows: [{ value: valueBuffer, expiry: String(expiry) }], }); const result = await storage.get("test-key"); @@ -80,7 +88,7 @@ describe("PersistentStorage", () => { }); expect(mockConnector.query).toHaveBeenCalledWith( expect.stringContaining("SELECT value, expiry"), - ["test-key"], + [expect.any(BigInt)], // key_hash is bigint ); }); @@ -94,9 +102,14 @@ describe("PersistentStorage", () => { test("should update last_accessed on get (fire-and-forget)", async () => { const expiry = Date.now() + 10000; + const valueBuffer = Buffer.from( + JSON.stringify({ data: "test" }), + "utf-8", + ); + mockConnector.query .mockResolvedValueOnce({ - rows: [{ value: { data: "test" }, expiry: String(expiry) }], + rows: [{ value: valueBuffer, expiry: String(expiry) }], }) .mockResolvedValue({ rows: [] }); @@ -107,7 +120,7 @@ describe("PersistentStorage", () => { expect(mockConnector.query).toHaveBeenCalledWith( expect.stringContaining("UPDATE"), - expect.arrayContaining([expect.any(Number), "test-key"]), + [expect.any(BigInt)], // key_hash ); }); }); @@ -119,13 +132,9 @@ describe("PersistentStorage", () => { }); test("should insert new entry", async () => { - // has() returns false + // totalBytes() returns 0 mockConnector.query.mockResolvedValueOnce({ - rows: [{ exists: false }], - }); - // size() returns 0 - mockConnector.query.mockResolvedValueOnce({ - rows: [{ count: "0" }], + rows: [{ total: "0" }], }); // INSERT succeeds mockConnector.query.mockResolvedValueOnce({ rows: [] }); @@ -137,39 +146,26 @@ describe("PersistentStorage", () => { expect(mockConnector.query).toHaveBeenCalledWith( expect.stringContaining("INSERT INTO"), - expect.arrayContaining(["test-key", expect.any(String)]), + expect.arrayContaining([ + expect.any(BigInt), // key_hash + expect.any(Buffer), // key + expect.any(Buffer), // value + expect.any(Number), // byte_size + expect.any(Number), // expiry + ]), ); }); - test("should update existing entry", async () => { - // has() returns true + test("should evict when maxBytes exceeded", async () => { + // totalBytes() returns maxBytes (triggers eviction) mockConnector.query.mockResolvedValueOnce({ - rows: [{ exists: true }], + rows: [{ total: String(1024 * 1024) }], // 1MB (at limit) }); - // INSERT/UPDATE succeeds - mockConnector.query.mockResolvedValueOnce({ rows: [] }); - - await storage.set("test-key", { - value: { data: "updated" }, - expiry: Date.now() + 10000, - }); - - expect(mockConnector.query).toHaveBeenCalledWith( - expect.stringContaining("ON CONFLICT"), - expect.any(Array), - ); - }); - - test("should evict LRU when full", async () => { - // has() returns false - mockConnector.query.mockResolvedValueOnce({ - rows: [{ exists: false }], - }); - // size() returns maxSize (100) + // cleanupExpired returns 0 mockConnector.query.mockResolvedValueOnce({ - rows: [{ count: "100" }], + rows: [{ count: "0" }], }); - // DELETE (eviction) succeeds + // eviction DELETE succeeds mockConnector.query.mockResolvedValueOnce({ rows: [] }); // INSERT succeeds mockConnector.query.mockResolvedValueOnce({ rows: [] }); @@ -186,12 +182,9 @@ describe("PersistentStorage", () => { ); }); - test("should serialize value as JSON", async () => { - mockConnector.query.mockResolvedValueOnce({ - rows: [{ exists: false }], - }); + test("should serialize value to Buffer", async () => { mockConnector.query.mockResolvedValueOnce({ - rows: [{ count: "0" }], + rows: [{ total: "0" }], }); mockConnector.query.mockResolvedValueOnce({ rows: [] }); @@ -205,7 +198,10 @@ describe("PersistentStorage", () => { call[0].includes("INSERT"), ); - expect(insertCall?.[1]?.[1]).toBe(JSON.stringify(value)); + // value is at index 2 (key_hash, key, value, ...) + const valueBuffer = insertCall?.[1]?.[2] as Buffer; + expect(valueBuffer).toBeInstanceOf(Buffer); + expect(valueBuffer.toString("utf-8")).toBe(JSON.stringify(value)); }); }); @@ -215,14 +211,14 @@ describe("PersistentStorage", () => { mockConnector.query.mockClear(); }); - test("should delete entry", async () => { + test("should delete entry by key_hash", async () => { mockConnector.query.mockResolvedValueOnce({ rows: [] }); await storage.delete("test-key"); expect(mockConnector.query).toHaveBeenCalledWith( expect.stringContaining("DELETE FROM"), - ["test-key"], + [expect.any(BigInt)], // key_hash ); }); }); @@ -258,6 +254,10 @@ describe("PersistentStorage", () => { const result = await storage.has("test-key"); expect(result).toBe(true); + expect(mockConnector.query).toHaveBeenCalledWith( + expect.stringContaining("SELECT EXISTS"), + [expect.any(BigInt)], // key_hash + ); }); test("should return false when key does not exist", async () => { @@ -314,6 +314,33 @@ describe("PersistentStorage", () => { }); }); + describe("totalBytes", () => { + beforeEach(async () => { + await storage.initialize(); + mockConnector.query.mockClear(); + }); + + test("should return sum of byte_size", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ total: "1048576" }], // 1MB + }); + + const result = await storage.totalBytes(); + + expect(result).toBe(1048576); + }); + + test("should return 0 when empty", async () => { + mockConnector.query.mockResolvedValueOnce({ + rows: [{ total: "0" }], + }); + + const result = await storage.totalBytes(); + + expect(result).toBe(0); + }); + }); + describe("cleanupExpired", () => { beforeEach(async () => { await storage.initialize(); @@ -377,7 +404,7 @@ describe("PersistentStorage", () => { describe("auto-initialization", () => { test("should auto-initialize on get if not initialized", async () => { const uninitializedStorage = new PersistentStorage( - { maxSize: 100 }, + { maxBytes: 1024 * 1024 }, mockConnector as any, ); @@ -393,7 +420,7 @@ describe("PersistentStorage", () => { test("should auto-initialize on set if not initialized", async () => { const uninitializedStorage = new PersistentStorage( - { maxSize: 100 }, + { maxBytes: 1024 * 1024 }, mockConnector as any, ); diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index 3ece4a5ef..5465acb8e 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -4,8 +4,8 @@ export interface CacheConfig { enabled?: boolean; /** Time to live in seconds */ ttl?: number; - /** Maximum number of entries in the cache */ - maxSize?: number; + /** Maximum number of bytes in the cache */ + maxBytes?: number; /** Cache key */ cacheKey?: (string | number | object)[]; /** Whether to use persistent cache */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f89b1bed..a87c7836f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: lint-staged: specifier: ^15.5.1 version: 15.5.2 + pg: + specifier: ^8.16.3 + version: 8.16.3 plop: specifier: ^4.0.4 version: 4.0.4(@types/node@24.7.2) From 0e1fd6617d318c7cddb65d60035cce61a24395b5 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Thu, 27 Nov 2025 12:44:14 +0000 Subject: [PATCH 07/18] fix: maxSize for in-memory --- packages/app-kit/src/cache/storage/defaults.ts | 2 ++ packages/shared/src/cache.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/app-kit/src/cache/storage/defaults.ts b/packages/app-kit/src/cache/storage/defaults.ts index 0268beff3..e71d7cf66 100644 --- a/packages/app-kit/src/cache/storage/defaults.ts +++ b/packages/app-kit/src/cache/storage/defaults.ts @@ -10,6 +10,8 @@ export const lakebaseStorageDefaults = { tableName: "appkit_cache_entries", /** Maximum number of bytes in the cache */ maxBytes: 256 * 1024 * 1024, // 256MB + /** Maximum number of entries in the cache */ + maxSize: 1000, /** Number of entries to evict when cache is full */ evictionBatchSize: 100, }; diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index 5465acb8e..d81b2dbdd 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -6,6 +6,8 @@ export interface CacheConfig { ttl?: number; /** Maximum number of bytes in the cache */ maxBytes?: number; + /** Maximum number of entries in the cache */ + maxSize?: number; /** Cache key */ cacheKey?: (string | number | object)[]; /** Whether to use persistent cache */ From a3646c29730d3cace901121bdda5e488db5efa47 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Thu, 27 Nov 2025 17:25:58 +0000 Subject: [PATCH 08/18] fix: restore telemetry in cache manager --- .../src/analytics/tests/analytics.test.ts | 2 +- packages/app-kit/src/cache/index.ts | 323 +++++++++++++----- .../app-kit/src/plugin/tests/cache.test.ts | 28 -- 3 files changed, 241 insertions(+), 112 deletions(-) diff --git a/packages/app-kit/src/analytics/tests/analytics.test.ts b/packages/app-kit/src/analytics/tests/analytics.test.ts index 10a3b4e69..1fd11abb6 100644 --- a/packages/app-kit/src/analytics/tests/analytics.test.ts +++ b/packages/app-kit/src/analytics/tests/analytics.test.ts @@ -44,7 +44,7 @@ const { mockCacheStore, mockCacheInstance } = vi.hoisted(() => { return { mockCacheStore: store, mockCacheInstance: instance }; }); -vi.mock("@databricks-apps/cache", () => ({ +vi.mock("../../cache", () => ({ CacheManager: { getInstanceSync: vi.fn(() => mockCacheInstance), }, diff --git a/packages/app-kit/src/cache/index.ts b/packages/app-kit/src/cache/index.ts index 301670470..1bb968141 100644 --- a/packages/app-kit/src/cache/index.ts +++ b/packages/app-kit/src/cache/index.ts @@ -1,43 +1,64 @@ +import { createHash } from "node:crypto"; +import { WorkspaceClient } from "@databricks/sdk-experimental"; import type { CacheConfig } from "shared"; -import type { ITelemetry } from "../telemetry"; -import { type Counter, SpanStatusCode } from "../telemetry"; +import { LakebaseConnector } from "../connectors"; +import type { Counter, ITelemetry } from "../telemetry"; +import { SpanStatusCode, TelemetryManager } from "../telemetry"; +import { deepMerge } from "../utils"; +import { cacheDefaults } from "./defaults"; +import { + type CacheStorage, + InMemoryStorage, + PersistentStorage, +} from "./storage"; -export interface CacheEntry { - value: T; - expiry: number; -} +export type { CacheEntry, CacheStorage } from "./storage"; +/** + * Cache manager class to handle cache operations. + * Can be used with in-memory storage or persistent storage (Lakebase). + * + * The cache is automatically initialized by AppKit. Use `getInstanceSync()` to access + * the singleton instance after initialization. + * + * @example + * ```typescript + * const cache = CacheManager.getInstanceSync(); + * const result = await cache.getOrExecute(["users", userId], () => fetchUser(userId), userKey); + * ``` + */ export class CacheManager { - private static readonly TELEMETRY_INSTRUMENT_CONFIG = { - name: "cache-manager", - includePrefix: true, - }; - private static readonly DEFAULT_ENABLED = true; - private static readonly DEFAULT_TTL = 3600; // 1 hour - private static readonly DEFAULT_MAX_SIZE = 1000; // default max 1000 entries - - private cache = new Map(); - private accessOrder = new Map(); - private accessCounter = 0; - private config: Required; - private inFlightRequests = new Map>(); - private telemetry: ITelemetry; + private static readonly CLEANUP_PROBABILITY = 0.01; + private static readonly NAME: string = "cache-manager"; + private static instance: CacheManager | null = null; + private static initPromise: Promise | null = null; + + private storage: CacheStorage; + private config: CacheConfig; + private inFlightRequests: Map>; + private cleanupInProgress: boolean; - // Create metrics once at class level + // Telemetry + private telemetry: ITelemetry; private cacheHitCounter: Counter; private cacheMissCounter: Counter; - constructor(config: CacheConfig = {}, telemetry: ITelemetry) { - this.config = { - enabled: config.enabled ?? CacheManager.DEFAULT_ENABLED, - ttl: config.ttl ?? CacheManager.DEFAULT_TTL, - maxSize: config.maxSize ?? CacheManager.DEFAULT_MAX_SIZE, - cacheKey: config.cacheKey ?? [], - }; + private constructor( + storage: CacheStorage, + config: CacheConfig, + telemetry: ITelemetry, + ) { + this.storage = storage; + this.config = config; + this.inFlightRequests = new Map(); + this.cleanupInProgress = false; + this.telemetry = telemetry; - const meter = this.telemetry.getMeter( - CacheManager.TELEMETRY_INSTRUMENT_CONFIG, - ); + const meter = this.telemetry.getMeter({ + name: CacheManager.NAME, + includePrefix: true, + }); + this.cacheHitCounter = meter.createCounter("cache.hit", { description: "Total number of cache hits", unit: "1", @@ -48,13 +69,109 @@ export class CacheManager { }); } - // Get or execute a function and cache the result + /** + * Get the singleton instance of the cache manager (sync version). + * Throws if not initialized - ensure AppKit.create() has completed first. + * @returns CacheManager instance + */ + static getInstanceSync(): CacheManager { + if (!CacheManager.instance) { + throw new Error( + "CacheManager not initialized. Ensure AppKit.create() has completed before accessing the cache.", + ); + } + + return CacheManager.instance; + } + + /** + * Initialize and get the singleton instance of the cache manager. + * Called internally by AppKit - prefer `getInstanceSync()` for plugin access. + * @param userConfig - User configuration for the cache manager + * @returns CacheManager instance + * @internal + */ + static async getInstance( + userConfig?: Partial, + ): Promise { + if (CacheManager.instance) { + return CacheManager.instance; + } + + if (!CacheManager.initPromise) { + CacheManager.initPromise = CacheManager.create(userConfig).then( + (instance) => { + CacheManager.instance = instance; + return instance; + }, + ); + } + + return CacheManager.initPromise; + } + + /** + * Create a new cache manager instance + * @param userConfig - User configuration for the cache manager + * @returns CacheManager instance + */ + private static async create( + userConfig?: Partial, + ): Promise { + const config = deepMerge(cacheDefaults, userConfig); + const telemetry = TelemetryManager.getProvider(CacheManager.NAME); + + if (!config.persistentCache) { + return new CacheManager(new InMemoryStorage(config), config, telemetry); + } + + try { + const workspaceClient = new WorkspaceClient({}); + const connector = new LakebaseConnector({ workspaceClient }); + const isHealthy = await connector.healthCheck(); + + if (isHealthy) { + const persistentStorage = new PersistentStorage(config, connector); + await persistentStorage.initialize(); + return new CacheManager(persistentStorage, config, telemetry); + } + } catch (error) { + console.warn("[Cache] Persistent storage unavailable:", error); + } + + // if strict persistence is enabled, do not fallback to in-memory storage + if (config.strictPersistence) { + console.warn( + "[Cache] strictPersistence enabled but persistent storage unavailable. Cache disabled.", + ); + const disabledConfig = { ...config, enabled: false }; + return new CacheManager( + new InMemoryStorage(disabledConfig), + disabledConfig, + telemetry, + ); + } + + console.warn("[Cache] Falling back to in-memory cache."); + return new CacheManager(new InMemoryStorage(config), config, telemetry); + } + + /** + * Get or execute a function and cache the result + * @param key - Cache key + * @param fn - Function to execute + * @param userKey - User key + * @param options - Options for the cache + * @returns Promise of the result + */ async getOrExecute( key: (string | number | object)[], fn: () => Promise, userKey: string, options?: { ttl?: number }, ): Promise { + if (!this.config.enabled) return fn(); + const cacheKey = this.generateKey(key, userKey); return this.telemetry.startActiveSpan( @@ -63,17 +180,18 @@ export class CacheManager { attributes: { "cache.key": cacheKey, "cache.enabled": this.config.enabled, + "cache.persistent": this.storage.isPersistent(), }, }, async (span) => { try { // Check cache first - const cached = this.get(cacheKey); - if (cached) { + const cached = await this.storage.get(cacheKey); + if (cached !== null) { span.setAttribute("cache.hit", true); span.setStatus({ code: SpanStatusCode.OK }); this.cacheHitCounter.add(1, { "cache.key": cacheKey }); - return cached; + return cached.value as T; } // Check in-flight requests for deduplication @@ -90,7 +208,7 @@ export class CacheManager { "cache.deduplication": "true", }); span.end(); - return inFlight; + return inFlight as Promise; } // Cache miss - execute function @@ -99,11 +217,11 @@ export class CacheManager { this.cacheMissCounter.add(1, { "cache.key": cacheKey }); const promise = fn() - .then((result) => { - this.set(cacheKey, result, options); + .then(async (result) => { + await this.set(cacheKey, result, options); span.addEvent("cache.value_stored", { "cache.key": cacheKey, - "cache.ttl": options?.ttl ?? this.config.ttl, + "cache.ttl": options?.ttl ?? this.config.ttl ?? 3600, }); return result; }) @@ -129,84 +247,123 @@ export class CacheManager { span.end(); } }, - CacheManager.TELEMETRY_INSTRUMENT_CONFIG, + { name: CacheManager.NAME, includePrefix: true }, ); } - get(key: string): T | null { + /** + * Get a cached value + * @param key - Cache key + * @returns Promise of the value or null if not found or expired + */ + async get(key: string): Promise { if (!this.config.enabled) return null; - const entry = this.cache.get(key); + // probabilistic cleanup trigger + this.maybeCleanup(); + + const entry = await this.storage.get(key); if (!entry) return null; if (Date.now() > entry.expiry) { - this.cache.delete(key); - this.accessOrder.delete(key); + await this.storage.delete(key); return null; } - - // Update access order for LRU - this.accessOrder.set(key, ++this.accessCounter); return entry.value as T; } - set(key: string, value: T, options?: { ttl?: number }): void { - if (!this.config.enabled) return; + /** Probabilistically trigger cleanup of expired entries (fire-and-forget) */ + private maybeCleanup(): void { + if (this.cleanupInProgress) return; + if (!this.storage.isPersistent()) return; + if (Math.random() > CacheManager.CLEANUP_PROBABILITY) return; - if (this.cache.size >= this.config.maxSize && !this.cache.has(key)) { - this.evictLRU(); - } + this.cleanupInProgress = true; + (this.storage as PersistentStorage) + .cleanupExpired() + .catch((error) => { + console.debug("Error cleaning up expired entries:", error); + }) + .finally(() => { + this.cleanupInProgress = false; + }); + } + + /** + * Set a value in the cache + * @param key - Cache key + * @param value - Value to set + * @param options - Options for the cache + * @returns Promise of the result + */ + async set( + key: string, + value: T, + options?: { ttl?: number }, + ): Promise { + if (!this.config.enabled) return; - const expiryTime = Date.now() + (options?.ttl ?? this.config.ttl) * 1000; - this.cache.set(key, { value, expiry: expiryTime }); - this.accessOrder.set(key, ++this.accessCounter); + const ttl = options?.ttl ?? this.config.ttl ?? 3600; + const expiryTime = Date.now() + ttl * 1000; + await this.storage.set(key, { value, expiry: expiryTime }); } - delete(key: string): void { - this.cache.delete(key); - this.accessOrder.delete(key); + /** + * Delete a value from the cache + * @param key - Cache key + * @returns Promise of the result + */ + async delete(key: string): Promise { + if (!this.config.enabled) return; + await this.storage.delete(key); } - clear(): void { - this.cache.clear(); - this.accessOrder.clear(); - this.accessCounter = 0; + /** Clear the cache */ + async clear(): Promise { + await this.storage.clear(); this.inFlightRequests.clear(); } - has(key: string): boolean { + /** + * Check if a value exists in the cache + * @param key - Cache key + * @returns Promise of true if the value exists, false otherwise + */ + async has(key: string): Promise { if (!this.config.enabled) return false; - const entry = this.cache.get(key); + + const entry = await this.storage.get(key); if (!entry) return false; if (Date.now() > entry.expiry) { - this.cache.delete(key); - this.accessOrder.delete(key); + await this.storage.delete(key); return false; } return true; } + /** + * Generate a cache key + * @param parts - Parts of the key + * @param userKey - User key + * @returns Cache key + */ generateKey(parts: (string | number | object)[], userKey: string): string { - parts = [userKey, ...parts]; - return parts.map((p) => JSON.stringify(p)).join(":"); + const allParts = [userKey, ...parts]; + const serialized = JSON.stringify(allParts); + return createHash("sha256").update(serialized).digest("hex"); } - // Evict the least recently used entry (LRU) - private evictLRU(): void { - let oldestKey: string | null = null; - let oldestAccess = Infinity; - - for (const [key, accessTime] of this.accessOrder) { - if (accessTime < oldestAccess) { - oldestAccess = accessTime; - oldestKey = key; - } - } + /** Close the cache */ + async close(): Promise { + await this.storage.close(); + } - if (oldestKey) { - this.cache.delete(oldestKey); - this.accessOrder.delete(oldestKey); - } + /** + * Check if the storage is healthy + * @returns Promise of true if the storage is healthy, false otherwise + */ + async isStorageHealthy(): Promise { + return this.storage.healthCheck(); } } diff --git a/packages/app-kit/src/plugin/tests/cache.test.ts b/packages/app-kit/src/plugin/tests/cache.test.ts index daee2c208..3acf17f09 100644 --- a/packages/app-kit/src/plugin/tests/cache.test.ts +++ b/packages/app-kit/src/plugin/tests/cache.test.ts @@ -1,6 +1,3 @@ -import { TelemetryManager, type TelemetryProvider } from "../../telemetry"; -import { createMockTelemetry } from "@tools/test-helpers"; -import { CacheManager } from "../../cache"; import type { CacheConfig } from "shared"; import { beforeEach, describe, expect, test, vi } from "vitest"; import { CacheInterceptor } from "../interceptors/cache"; @@ -266,29 +263,4 @@ describe("CacheInterceptor", () => { expect(fn).toHaveBeenCalledTimes(2); }); - - test("should work correctly with telemetry enabled", async () => { - // Create telemetry with traces enabled - const mockTelemetryWithTraces = createMockTelemetry(); - vi.mocked(TelemetryManager.getProvider).mockReturnValue( - mockTelemetryWithTraces as TelemetryProvider, - ); - - const telemetryProvider = - TelemetryManager.getProvider("cache-test-enabled"); - const cacheManagerWithTelemetry = new CacheManager({}, telemetryProvider); - - const config: CacheConfig = { - enabled: true, - cacheKey: ["telemetry-test"], - }; - const interceptor = new CacheInterceptor(cacheManagerWithTelemetry, config); - const fn = vi.fn().mockResolvedValue("result"); - - const result = await interceptor.intercept(fn, context); - - // Verify the cache works correctly with telemetry - expect(result).toBe("result"); - expect(fn).toHaveBeenCalledTimes(1); - }); }); From b632c51d2a45c97f85cfbb071afc3068dc635bb3 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Thu, 27 Nov 2025 17:39:51 +0000 Subject: [PATCH 09/18] fix: removing some files --- .cursor/rules/test-strategy.en.mdc | 91 ---------------- .cursor/rules/v5.en.mdc | 169 ----------------------------- 2 files changed, 260 deletions(-) delete mode 100644 .cursor/rules/test-strategy.en.mdc delete mode 100644 .cursor/rules/v5.en.mdc diff --git a/.cursor/rules/test-strategy.en.mdc b/.cursor/rules/test-strategy.en.mdc deleted file mode 100644 index 8783e722c..000000000 --- a/.cursor/rules/test-strategy.en.mdc +++ /dev/null @@ -1,91 +0,0 @@ ---- -alwaysApply: true ---- - -## Test strategy rules - -These rules define the test process that **must** be followed whenever you implement or modify test code. A test task is **not** considered complete unless all of the steps below are satisfied. - ---- - -## 1. Test perspective table (equivalence partitioning / boundary values) - -1. Before starting any test work, you **must** first present a “test perspectives table” in Markdown table format. -2. The table must include at least the following columns: `Case ID`, `Input / Precondition`, `Perspective (Equivalence / Boundary)`, `Expected Result`, `Notes`. -3. Rows must comprehensively cover normal, abnormal, and boundary cases. For boundary values, you must include `0 / minimum / maximum / ±1 / empty / NULL` at a minimum. - Among the boundary candidates (0 / minimum / maximum / ±1 / empty / NULL), you may omit those that are not meaningful for the given specification, as long as you record in `Notes` why they are out of scope. -4. If you later discover missing perspectives, update the table after self‑review and add the necessary cases. -5. Note: For minor adjustments to existing tests (such as tweaking messages or slightly updating expectations) that do not introduce new branches or constraints, creating or updating a test perspectives table is optional. - -### Template example - -| Case ID | Input / Precondition | Perspective (Equivalence / Boundary) | Expected Result | Notes | -|--------|----------------------|---------------------------------------|----------------------------------------------|-------| -| TC-N-01 | Valid input A | Equivalence – normal | Processing succeeds and returns expected value | - | -| TC-A-01 | NULL | Boundary – NULL | Validation error (required field) | - | -| ... | ... | ... | ... | ... | - ---- - -## 2. Test code implementation policy - -1. Implement **all** cases listed in the table above as automated tests. -2. Ensure you include **at least as many failure cases as success cases** (validation errors, exceptions, external dependency failures, etc.). -3. Your tests must cover the following perspectives: - - Normal paths (main scenarios) - - Abnormal paths (validation errors, exception paths) - - Boundary values (0, minimum, maximum, ±1, empty, NULL) - - Inputs with invalid types or formats - - Failures of external dependencies (e.g. API / DB / messaging, when applicable) - - Exception types and error messages -4. Additionally, aim for 100% branch coverage, and design extra cases yourself as needed to achieve it. - Treat 100% branch coverage as a target. When it is not reasonably achievable, at minimum cover all high‑impact business branches and primary error paths. - If any branches remain uncovered, explicitly document the reasons and impact in `Notes` or the PR description. - ---- - -## 3. Given / When / Then comments - -Each test case must include the following comment format: - -```text -// Given: Preconditions -// When: Operation to execute -// Then: Expected result / assertions -``` - -Place these comments directly above the test code or within the steps so that readers can easily follow the scenario. - ---- - -## 4. Exception and error verification - -1. For cases where exceptions occur, explicitly verify both the **exception type** and the **message**. -2. For validation‑related abnormal cases, also verify error codes and field information, if available. -3. When simulating failures of external dependencies, use stubs/mocks and confirm that the expected exceptions, retries, and fallbacks are invoked. - ---- - -## 5. Execution commands and coverage - -1. At the end of test implementation, always document the **execution command** and **coverage collection method** at the end of the documentation or PR description. - - Examples: `npm run test`, `pnpm vitest run --coverage`, `pytest --cov=...` -2. Check branch coverage and statement coverage, aiming for 100% branch coverage as a target (when it is not reasonably achievable, prioritize covering high‑impact business branches and primary error paths). -3. Attach coverage report results (screenshots or summaries) where reasonably possible. - ---- - -## 6. Operational notes - -1. Any test changes that do not comply with these rules should be sent back during review. -2. Even when there are no external dependencies, you must still include failure cases by **using mocks to simulate failures**. -3. When new branches or constraints are added to the target specification, update both the test perspective table and the test code at the same time. -4. If there are cases that are difficult to automate, clearly document the reasons and alternative measures, and obtain agreement with the reviewer. - The alternative measures must at least describe the affected functionality and risks, the manual verification steps, the expected results, and how logs or screenshots will be recorded. -5. In principle, any PR that includes a meaningful change to production code (such as new features, bug fixes, or refactors that may affect behavior) must also include corresponding additions or updates to automated tests. -6. If adding or updating tests is reasonably difficult, clearly document the reasons and the alternative verification steps (such as manual test procedures) in the PR description and obtain agreement from the reviewer. -7. Even for refactors that are not intended to change behavior, confirm that the changed areas are sufficiently covered by existing tests, and add tests when coverage is insufficient. - ---- - -Always adhere to these rules and continuously self‑check for missing perspectives when designing and implementing tests. diff --git a/.cursor/rules/v5.en.mdc b/.cursor/rules/v5.en.mdc deleted file mode 100644 index 072b01aa9..000000000 --- a/.cursor/rules/v5.en.mdc +++ /dev/null @@ -1,169 +0,0 @@ ---- -alwaysApply: true ---- - -# v5: Coding support rules - -You are a highly capable AI assistant. This file defines only the behaviour required to achieve maximum productivity and safety for **code‑centric tasks**. -This file provides the foundational rules for carrying out coding‑related tasks. - ---- - -## 0. Common assumptions - -- **Target tasks**: Coding assistance, refactoring, debugging, and authoring development‑related documentation -- **Language**: Follow the language used in the user’s instructions and input (if not explicitly specified, reply in the language the user is using). -- **Rule precedence**: System > Workspace‑common rules > This file (v5) -- **Completion policy**: Do not stop halfway. Keep working persistently until the user’s request is satisfied. If constraints prevent completion, clearly state current progress and remaining tasks. -- **Priority and conflicts between instructions**: Follow the user’s instructions based on system and workspace‑common rules. If instructions conflict or are ambiguous, do not arbitrarily interpret them for convenience; ask a brief clarification before proceeding. -- **User‑specified preferences take precedence**: When the user specifies an output format (bullet list, code only, etc.) or length, treat that preference as higher priority than the defaults in this file. -- **Response style**: - - Avoid excessive preambles; state conclusions and changes first. - - Keep explanations to what is necessary and sufficient, and be especially brief for lightweight tasks. - - Limit example code to only what is needed (avoid huge code blocks). - - Only share deep reasoning processes or long thought logs when the user explicitly asks; otherwise stick to conclusions and key rationales. - ---- - -## 1. Task classification and reasoning depth - -Task classification (🟢/🟡/🔴) and approval conditions follow the workspace‑common rules. -This section only defines **differences in reasoning depth and procedure for coding assistance**. -If the user explicitly requests a different way of working (e.g. “design only first”), prioritize that instruction. - -### 🟢 Lightweight tasks (e.g. small fixes / simple investigation) - -- Examples: A few‑line change in a single file, quick root‑cause check for a bug, checking configuration values. -- Design consultations without code changes, refactor strategy discussions, and general Q&A should also, in principle, be handled as 🟢 tasks with concise answers. -- **Reasoning policy**: - - Avoid deep brainstorming; aim for the shortest path to a solution. - - Do not perform large‑scale design discussions or present a Plan. -- **Execution flow**: - 1. Summarize the task in one line. - 2. Read only the necessary files with `read_file` / `grep`, then immediately apply the fix with `apply_patch`. - 3. Report the result in 1–2 sentences (do not use checklists or detailed templates). - -### 🟡 Standard tasks (e.g. feature additions / small refactors) - -- Examples: Changes spanning multiple files, implementing a single API endpoint, creating a component. -- **Reasoning policy**: - - Present a brief analysis and a "todo list" before implementation. - - Leverage adaptive reasoning while avoiding unnecessarily long thought logs. -- **Execution flow**: - 1. Present 3–7 key subtasks in a checklist. - 2. Read relevant files and apply staged changes with `apply_patch`. - 3. When possible, check for basic errors with `read_lints`. - 4. Finally, summarize in a few sentences **what you changed, in which files, and to what extent**. - -### 🔴 Critical tasks (e.g. architecture/security/cost‑impacting work) - -- Examples: Authentication/authorization changes, DB schema changes, infrastructure changes, modifications likely to affect production. -- **Reasoning policy**: - - First carefully analyze impact and risk, then present a Plan and wait for approval. - - Consider rollback steps and security/cost impact. -- **Execution flow**: - - Always use `create_plan`, and only start work after the user explicitly approves (following the common rules). - ---- - -## 2. Tool usage policy for coding - -### 2.1 Core tools - -- **`read_file`**: Always read relevant files before making changes. For large files, focus on only the necessary ranges. -- **`apply_patch`**: Primary method for code changes. - - When the user asks you to “implement” something, **do not stop at a proposal—actually apply patches** unless there is a blocker. - - Keep each patch to a semantically coherent unit of change. -- **`grep` / `codebase_search`**: - - Use `grep` to locate strings and symbols. - - Use `codebase_search` when searching by meaning or behavior. - -### 2.2 Parallel execution and long‑running operations - -- **`multi_tool_use.parallel`**: - - For read‑only tools like `read_file` / `grep` / `codebase_search` / `web_search`, actively execute them in parallel when there are no dependencies. - - Do not run them in parallel with `apply_patch` or other state‑changing commands. -- **`run_terminal_cmd`**: - - Use only when the user explicitly requests it or when builds/tests are clearly necessary. - - Add non‑interactive flags (e.g. `--yes`) for commands that would otherwise require input. - - For commands that run for a long time, use `is_background: true`. - -### 2.3 Web and browser‑related tools - -- **`web_search`** usage: - - Actively search even without user instruction in cases such as: - - **External services** (models, AI services, clouds) where latest specs/pricing matter - - **Version‑dependent behavior or breaking changes** in libraries/frameworks - - Specific error messages or compatibility issues where built‑in knowledge may be risky - - Only when you actually search, briefly (1–2 sentences) share **what you searched for**. -- **`mcp_cursor-ide-browser_browser_script`** (hereafter `browser_script`): - - Use for checking web app behavior or doing E2E‑like verification. - - Do not start local servers on your own; only do so when instructed by the user. - -### 2.4 Static analysis tools - -- **`read_lints`**: - - For files where you made non‑trivial code changes, check for lint errors when feasible and fix those you can quickly resolve. - ---- - -## 3. Standard flow for coding tasks - -- For any task type, do not leave the flow half‑finished; if constraints prevent completion, clearly indicate “what is done so far and what remains”. - -### 3.1 Lightweight tasks (🟢) - -1. Summarize the task in one line. -2. Check 1–2 related files with `read_file` / `grep`. -3. Immediately fix using `apply_patch`. -4. Perform minimal verification as needed (e.g. visually confirm there are no type errors). -5. Communicate the result in 1–2 sentences. - -### 3.2 Standard tasks (🟡) - -1. Organize the goal, constraints, and expected impact in 2–3 sentences. -2. Present a checklist with about 3–7 items. -3. Read related files and apply changes in multiple passes using `apply_patch`. -4. Use `read_lints` to check for basic errors and fix them on the spot when possible. -5. Finally, concisely summarize what you changed (which files, how they changed, and any known limitations). - -### 3.3 Critical tasks (🔴) - -- Follow the existing rule: `create_plan` → approval → phased execution. -- Break code changes into **small, safe steps**, and check state at each step. -- In `create_plan`, include at least: purpose, expected impact, major risks, and rollback approach (how to revert). - ---- - -## 4. Errors, types, security, and cost - -- **Lint/type errors**: - - Resolve errors you introduced as much as possible on the spot. - - If the root cause is complex and cannot be fixed immediately, explicitly state that, and either revert to a safe state or limit the impact. -- **No `any` / no degradation**: - - Do not add `any` or intentionally degrade features just to “hide” errors. - - Even when a temporary workaround is necessary, briefly explain the rationale and risks. -- **Security / production / cost**: - - Treat changes involving authentication/authorization, network boundaries, data retention, or pricing as “critical tasks”. - - In such cases, present a Plan and obtain user approval before implementation. - ---- - -## 5. Output style and explanation granularity - -- **Lightweight tasks**: - - 1–2 sentence result reports are sufficient. Do not use detailed templates or long text. -- **Standard tasks and above**: - - Use headings (`##` / `###`) and bullet lists to organize changes, impact, and caveats. - - When quoting code, show only the necessary surrounding lines. -- **Code block usage**: - - When quoting existing code, include the file path so it is clear where it comes from. - - For new proposal code, show only the smallest copyable unit. -- **User‑specified preferences take precedence**: - - If the user requests “short”, “longer”, “bullet list”, or “code only”, prioritize that over the defaults here. -- **Disclosure of reasoning process**: - - Only provide deep reasoning logs or long thought processes when the user explicitly asks; by default, stick to conclusions and the main rationale. - ---- - -By following these rules and leveraging adaptive reasoning and the toolset, autonomously execute coding tasks **safely and efficiently**. From be7a17df119368549dd3056e7641eb59f8b76de4 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Fri, 28 Nov 2025 12:46:30 +0000 Subject: [PATCH 10/18] feat: add observability on cache and lakebase --- packages/app-kit/src/analytics/analytics.ts | 2 +- packages/app-kit/src/analytics/types.ts | 3 +- packages/app-kit/src/cache/index.ts | 68 +++--- .../src}/cache/tests/cache-manager.test.ts | 6 +- .../src}/cache/tests/memory.test.ts | 2 +- .../src}/cache/tests/persistent.test.ts | 2 +- .../app-kit/src/connectors/lakebase/client.ts | 217 ++++++++++++------ .../app-kit/src/connectors/lakebase/types.ts | 4 + .../src/connectors/sql-warehouse/client.ts | 60 ++--- .../src}/connectors/tests/lakebase.test.ts | 2 +- packages/app-kit/src/core/app-kit.ts | 19 +- packages/app-kit/src/telemetry/index.ts | 2 + packages/shared/src/cache.ts | 4 + 13 files changed, 254 insertions(+), 137 deletions(-) rename packages/{backend => app-kit/src}/cache/tests/cache-manager.test.ts (98%) rename packages/{backend => app-kit/src}/cache/tests/memory.test.ts (99%) rename packages/{backend => app-kit/src}/cache/tests/persistent.test.ts (99%) rename packages/{backend => app-kit/src}/connectors/tests/lakebase.test.ts (99%) diff --git a/packages/app-kit/src/analytics/analytics.ts b/packages/app-kit/src/analytics/analytics.ts index 9b6a40870..37233f228 100644 --- a/packages/app-kit/src/analytics/analytics.ts +++ b/packages/app-kit/src/analytics/analytics.ts @@ -41,7 +41,7 @@ export class AnalyticsPlugin extends Plugin { this.SQLClient = new SQLWarehouseConnector({ timeout: config.timeout, - telemetry: this.telemetry, + telemetry: config.telemetry, }); if (process.env.NODE_ENV === "development") { diff --git a/packages/app-kit/src/analytics/types.ts b/packages/app-kit/src/analytics/types.ts index db3feb889..c3211178c 100644 --- a/packages/app-kit/src/analytics/types.ts +++ b/packages/app-kit/src/analytics/types.ts @@ -1,9 +1,10 @@ -import type { BasePluginConfig } from "shared"; +import type { BasePluginConfig, TelemetryOptions } from "shared"; import { z } from "zod"; export interface IAnalyticsConfig extends BasePluginConfig { timeout?: number; typePath?: string; + telemetry?: TelemetryOptions; } export interface IAnalyticsQueryRequest { diff --git a/packages/app-kit/src/cache/index.ts b/packages/app-kit/src/cache/index.ts index 1bb968141..21dd3f187 100644 --- a/packages/app-kit/src/cache/index.ts +++ b/packages/app-kit/src/cache/index.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { WorkspaceClient } from "@databricks/sdk-experimental"; import type { CacheConfig } from "shared"; import { LakebaseConnector } from "../connectors"; -import type { Counter, ITelemetry } from "../telemetry"; +import type { Counter, TelemetryProvider } from "../telemetry"; import { SpanStatusCode, TelemetryManager } from "../telemetry"; import { deepMerge } from "../utils"; import { cacheDefaults } from "./defaults"; @@ -29,7 +29,7 @@ export type { CacheEntry, CacheStorage } from "./storage"; */ export class CacheManager { private static readonly CLEANUP_PROBABILITY = 0.01; - private static readonly NAME: string = "cache-manager"; + private readonly name: string = "cache-manager"; private static instance: CacheManager | null = null; private static initPromise: Promise | null = null; @@ -39,34 +39,32 @@ export class CacheManager { private cleanupInProgress: boolean; // Telemetry - private telemetry: ITelemetry; - private cacheHitCounter: Counter; - private cacheMissCounter: Counter; - - private constructor( - storage: CacheStorage, - config: CacheConfig, - telemetry: ITelemetry, - ) { + private telemetry: TelemetryProvider; + private telemetryMetrics: { + cacheHitCount: Counter; + cacheMissCount: Counter; + }; + + private constructor(storage: CacheStorage, config: CacheConfig) { this.storage = storage; this.config = config; this.inFlightRequests = new Map(); this.cleanupInProgress = false; - this.telemetry = telemetry; - const meter = this.telemetry.getMeter({ - name: CacheManager.NAME, - includePrefix: true, - }); - - this.cacheHitCounter = meter.createCounter("cache.hit", { - description: "Total number of cache hits", - unit: "1", - }); - this.cacheMissCounter = meter.createCounter("cache.miss", { - description: "Total number of cache misses", - unit: "1", - }); + this.telemetry = TelemetryManager.getProvider( + this.name, + this.config.telemetry, + ); + this.telemetryMetrics = { + cacheHitCount: this.telemetry.getMeter().createCounter("cache.hit", { + description: "Total number of cache hits", + unit: "1", + }), + cacheMissCount: this.telemetry.getMeter().createCounter("cache.miss", { + description: "Total number of cache misses", + unit: "1", + }), + }; } /** @@ -119,10 +117,9 @@ export class CacheManager { userConfig?: Partial, ): Promise { const config = deepMerge(cacheDefaults, userConfig); - const telemetry = TelemetryManager.getProvider(CacheManager.NAME); if (!config.persistentCache) { - return new CacheManager(new InMemoryStorage(config), config, telemetry); + return new CacheManager(new InMemoryStorage(config), config); } try { @@ -133,7 +130,7 @@ export class CacheManager { if (isHealthy) { const persistentStorage = new PersistentStorage(config, connector); await persistentStorage.initialize(); - return new CacheManager(persistentStorage, config, telemetry); + return new CacheManager(persistentStorage, config); } } catch (error) { console.warn("[Cache] Persistent storage unavailable:", error); @@ -148,12 +145,11 @@ export class CacheManager { return new CacheManager( new InMemoryStorage(disabledConfig), disabledConfig, - telemetry, ); } console.warn("[Cache] Falling back to in-memory cache."); - return new CacheManager(new InMemoryStorage(config), config, telemetry); + return new CacheManager(new InMemoryStorage(config), config); } /** @@ -190,7 +186,9 @@ export class CacheManager { if (cached !== null) { span.setAttribute("cache.hit", true); span.setStatus({ code: SpanStatusCode.OK }); - this.cacheHitCounter.add(1, { "cache.key": cacheKey }); + this.telemetryMetrics.cacheHitCount.add(1, { + "cache.key": cacheKey, + }); return cached.value as T; } @@ -203,7 +201,7 @@ export class CacheManager { "cache.key": cacheKey, }); span.setStatus({ code: SpanStatusCode.OK }); - this.cacheHitCounter.add(1, { + this.telemetryMetrics.cacheHitCount.add(1, { "cache.key": cacheKey, "cache.deduplication": "true", }); @@ -214,7 +212,9 @@ export class CacheManager { // Cache miss - execute function span.setAttribute("cache.hit", false); span.addEvent("cache.miss", { "cache.key": cacheKey }); - this.cacheMissCounter.add(1, { "cache.key": cacheKey }); + this.telemetryMetrics.cacheMissCount.add(1, { + "cache.key": cacheKey, + }); const promise = fn() .then(async (result) => { @@ -247,7 +247,7 @@ export class CacheManager { span.end(); } }, - { name: CacheManager.NAME, includePrefix: true }, + { name: this.name, includePrefix: true }, ); } diff --git a/packages/backend/cache/tests/cache-manager.test.ts b/packages/app-kit/src/cache/tests/cache-manager.test.ts similarity index 98% rename from packages/backend/cache/tests/cache-manager.test.ts rename to packages/app-kit/src/cache/tests/cache-manager.test.ts index b2ba13aef..9c26c6b0e 100644 --- a/packages/backend/cache/tests/cache-manager.test.ts +++ b/packages/app-kit/src/cache/tests/cache-manager.test.ts @@ -1,9 +1,9 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { CacheManager } from "../src/index"; -import type { CacheStorage } from "../src/storage/types"; +import { CacheManager } from "../../index"; +import type { CacheStorage } from "../storage"; // Mock the storage modules -vi.mock("../src/storage/memory", () => ({ +vi.mock("../storage/memory", () => ({ InMemoryStorage: vi.fn().mockImplementation(() => createMockStorage()), })); diff --git a/packages/backend/cache/tests/memory.test.ts b/packages/app-kit/src/cache/tests/memory.test.ts similarity index 99% rename from packages/backend/cache/tests/memory.test.ts rename to packages/app-kit/src/cache/tests/memory.test.ts index 6d80f0013..5f4f0f23e 100644 --- a/packages/backend/cache/tests/memory.test.ts +++ b/packages/app-kit/src/cache/tests/memory.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, test } from "vitest"; -import { InMemoryStorage } from "../src/storage/memory"; +import { InMemoryStorage } from "../storage"; describe("InMemoryStorage", () => { let storage: InMemoryStorage; diff --git a/packages/backend/cache/tests/persistent.test.ts b/packages/app-kit/src/cache/tests/persistent.test.ts similarity index 99% rename from packages/backend/cache/tests/persistent.test.ts rename to packages/app-kit/src/cache/tests/persistent.test.ts index 0d3ae957c..7bdc105d1 100644 --- a/packages/backend/cache/tests/persistent.test.ts +++ b/packages/app-kit/src/cache/tests/persistent.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { PersistentStorage } from "../src/storage/persistent"; +import { PersistentStorage } from "../storage"; /** Mock LakebaseConnector for testing */ const createMockConnector = () => ({ diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index 0ebbc1144..b3db22d1d 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -1,8 +1,15 @@ import { randomUUID } from "node:crypto"; import type { WorkspaceClient } from "@databricks/sdk-experimental"; import { ApiClient, Config } from "@databricks/sdk-experimental"; -import { deepMerge } from "../../utils"; import pg from "pg"; +import { + type Counter, + type Histogram, + SpanStatusCode, + TelemetryManager, + type TelemetryProvider, +} from "@/telemetry"; +import { deepMerge } from "../../utils"; import { lakebaseDefaults } from "./defaults"; import type { LakebaseConfig, @@ -26,17 +33,43 @@ import type { * ``` */ export class LakebaseConnector { + private readonly name: string = "lakebase"; private readonly CACHE_BUFFER_MS = 2 * 60 * 1000; private readonly config: LakebaseConfig; private readonly connectionConfig: LakebaseConnectionConfig; private pool: pg.Pool | null = null; private credentials: LakebaseCredentials | null = null; + // telemetry + private readonly telemetry: TelemetryProvider; + private readonly telemetryMetrics: { + queryCount: Counter; + queryDuration: Histogram; + }; + constructor(userConfig?: Partial) { this.config = deepMerge(lakebaseDefaults, userConfig); - this.connectionConfig = this.parseConnectionConfig(); + this.telemetry = TelemetryManager.getProvider( + this.name, + this.config.telemetry, + ); + this.telemetryMetrics = { + queryCount: this.telemetry + .getMeter() + .createCounter("lakebase.query.count", { + description: "Total number of queries executed", + unit: "1", + }), + queryDuration: this.telemetry + .getMeter() + .createHistogram("lakebase.query.duration", { + description: "Duration of queries executed", + unit: "ms", + }), + }; + // validate configuration if (this.config.maxPoolSize < 1) { throw new Error("maxPoolSize must be at least 1"); @@ -60,26 +93,52 @@ export class LakebaseConnector { params?: any[], retryCount: number = 0, ): Promise> { - const pool = await this.getPool(); - - try { - return await pool.query(sql, params); - } catch (error) { - // retry on auth failure - if (this.isAuthError(error)) { - await this.rotateCredentials(); - const newPool = await this.getPool(); - return await newPool.query(sql, params); - } - - // retry on transient errors, but only once - if (this.isTransientError(error) && retryCount < 1) { - await new Promise((resolve) => setTimeout(resolve, 100)); - return await this.query(sql, params, retryCount + 1); - } - - throw error; - } + const startTime = Date.now(); + + return this.telemetry.startActiveSpan( + "lakebase.query", + { + attributes: { + "db.system": "lakebase", + "db.statement": sql.substring(0, 500), + "db.retry_count": retryCount, + }, + }, + async (span) => { + try { + const pool = await this.getPool(); + const result = await pool.query(sql, params); + span.setAttribute("db.rows_affected", result.rowCount ?? 0); + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } catch (error) { + // retry on auth failure + if (this.isAuthError(error)) { + span.addEvent("auth_error_retry"); + await this.rotateCredentials(); + const newPool = await this.getPool(); + return await newPool.query(sql, params); + } + + // retry on transient errors, but only once + if (this.isTransientError(error) && retryCount < 1) { + span.addEvent("transient_error_retry"); + await new Promise((resolve) => setTimeout(resolve, 100)); + return await this.query(sql, params, retryCount + 1); + } + + span.recordException(error as Error); + span.setStatus({ code: SpanStatusCode.ERROR }); + + throw error; + } finally { + const duration = Date.now() - startTime; + this.telemetryMetrics.queryCount.add(1); + this.telemetryMetrics.queryDuration.record(duration); + span.end(); + } + }, + ); } /** @@ -99,57 +158,87 @@ export class LakebaseConnector { callback: (client: pg.PoolClient) => Promise, retryCount: number = 0, ): Promise { - const pool = await this.getPool(); - const client = await pool.connect(); - - try { - return await callback(client); - } catch (error) { - // retry on auth failure - if (this.isAuthError(error)) { - client.release(); - await this.rotateCredentials(); - const newPool = await this.getPool(); - const retryClient = await newPool.connect(); - try { - return await callback(retryClient); - } finally { - retryClient.release(); - } - } - - // retry on transient errors, but only once - if (this.isTransientError(error) && retryCount < 1) { - client.release(); - await new Promise((resolve) => setTimeout(resolve, 100)); - const retryClient = await pool.connect(); + const startTime = Date.now(); + return this.telemetry.startActiveSpan( + "lakebase.transaction", + { + attributes: { + "db.system": "lakebase", + "db.retry_count": retryCount, + }, + }, + async (span) => { + const pool = await this.getPool(); + const client = await pool.connect(); try { - return await this.transaction(callback, retryCount + 1); + const result = await callback(client); + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } catch (error) { + // retry on auth failure + if (this.isAuthError(error)) { + span.addEvent("auth_error_retry"); + client.release(); + await this.rotateCredentials(); + const newPool = await this.getPool(); + const retryClient = await newPool.connect(); + try { + return await callback(retryClient); + } finally { + retryClient.release(); + } + } + + // retry on transient errors, but only once + if (this.isTransientError(error) && retryCount < 1) { + span.addEvent("transaction_error_retry"); + client.release(); + await new Promise((resolve) => setTimeout(resolve, 100)); + return await this.transaction(callback, retryCount + 1); + } + span.recordException(error as Error); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; } finally { - retryClient.release(); + client.release(); + const duration = Date.now() - startTime; + this.telemetryMetrics.queryCount.add(1); + this.telemetryMetrics.queryDuration.record(duration); + span.end(); } - } - - throw error; - } finally { - client.release(); - } + }, + ); } /** Check if database connection is healthy */ async healthCheck(): Promise { - try { - const result = await this.query<{ result: number }>("SELECT 1 as result"); - return result.rows[0]?.result === 1; - } catch { - return false; - } + return this.telemetry.startActiveSpan( + "lakebase.healthCheck", + {}, + async (span) => { + try { + const result = await this.query<{ result: number }>( + "SELECT 1 as result", + ); + const healthy = result.rows[0]?.result === 1; + span.setAttribute("db.healthy", healthy); + span.setStatus({ code: SpanStatusCode.OK }); + return healthy; + } catch { + span.setAttribute("db.healthy", false); + span.setStatus({ code: SpanStatusCode.ERROR }); + return false; + } finally { + span.end(); + } + }, + ); } /** Close connection pool (call on shutdown) */ async close(): Promise { if (this.pool) { - await this.pool.end().catch((error) => { + await this.pool.end().catch((error: unknown) => { console.error("Error closing connection pool:", error); }); this.pool = null; @@ -219,9 +308,9 @@ export class LakebaseConnector { ssl: sslMode === "require" ? { rejectUnauthorized: true } : false, }); - pool.on("error", (error) => { + pool.on("error", (error: Error & { code?: string }) => { console.error("Connection pool error:", error.message, { - code: (error as any).code, + code: error.code, }); }); @@ -264,7 +353,7 @@ export class LakebaseConnector { if (this.pool) { const oldPool = this.pool; this.pool = null; - oldPool.end().catch((error) => { + oldPool.end().catch((error: unknown) => { console.error( "Error closing old connection pool during rotation:", error, diff --git a/packages/app-kit/src/connectors/lakebase/types.ts b/packages/app-kit/src/connectors/lakebase/types.ts index 8b945a452..d85c9d137 100644 --- a/packages/app-kit/src/connectors/lakebase/types.ts +++ b/packages/app-kit/src/connectors/lakebase/types.ts @@ -1,4 +1,5 @@ import type { WorkspaceClient } from "@databricks/sdk-experimental"; +import type { TelemetryOptions } from "shared"; /** Configuration for LakebaseConnector */ export interface LakebaseConfig { @@ -32,6 +33,9 @@ export interface LakebaseConfig { /** How long credentials are valid (milliseconds) */ credentialTTLMs: number; + /** Telemetry configuration */ + telemetry?: TelemetryOptions; + /** Additional configuration options */ [key: string]: unknown; } diff --git a/packages/app-kit/src/connectors/sql-warehouse/client.ts b/packages/app-kit/src/connectors/sql-warehouse/client.ts index 3caa6f207..db953ba8d 100644 --- a/packages/app-kit/src/connectors/sql-warehouse/client.ts +++ b/packages/app-kit/src/connectors/sql-warehouse/client.ts @@ -3,48 +3,54 @@ import { type sql, type WorkspaceClient, } from "@databricks/sdk-experimental"; -import type { ITelemetry } from "../../telemetry"; +import type { TelemetryProvider } from "../../telemetry"; import { type Counter, type Histogram, - type Meter, type Span, SpanKind, SpanStatusCode, + TelemetryManager, } from "../../telemetry"; import { executeStatementDefaults } from "./defaults"; +import type { TelemetryOptions } from "shared"; export interface SQLWarehouseConfig { timeout?: number; - telemetry: ITelemetry; + telemetry?: TelemetryOptions; } export class SQLWarehouseConnector { - private static readonly TELEMETRY_INSTRUMENT_CONFIG = { - name: "sql-warehouse", - includePrefix: true, - }; + private readonly name = "sql-warehouse"; private config: SQLWarehouseConfig; - private meter: Meter; - private queryCounter: Counter; - private queryDuration: Histogram; + // telemetry + private readonly telemetry: TelemetryProvider; + private readonly telemetryMetrics: { + queryCount: Counter; + queryDuration: Histogram; + }; constructor(config: SQLWarehouseConfig) { this.config = config; - this.meter = this.config.telemetry.getMeter( - SQLWarehouseConnector.TELEMETRY_INSTRUMENT_CONFIG, - ); - this.queryCounter = this.meter.createCounter("db.query.count", { - description: "Total number of database queries", - unit: "1", - }); - this.queryDuration = this.meter.createHistogram("db.query.duration", { - description: "Duration of database queries", - unit: "ms", - }); + this.telemetry = TelemetryManager.getProvider( + this.name, + this.config.telemetry, + ); + this.telemetryMetrics = { + queryCount: this.telemetry.getMeter().createCounter("query.count", { + description: "Total number of queries executed", + unit: "1", + }), + queryDuration: this.telemetry + .getMeter() + .createHistogram("query.duration", { + description: "Duration of queries executed", + unit: "ms", + }), + }; } async executeStatement( @@ -55,7 +61,7 @@ export class SQLWarehouseConnector { const startTime = Date.now(); let success = false; - return this.config.telemetry.startActiveSpan( + return this.telemetry.startActiveSpan( "sql.query", { kind: SpanKind.CLIENT, @@ -185,11 +191,11 @@ export class SQLWarehouseConnector { success: success.toString(), }; - this.queryCounter.add(1, attributes); - this.queryDuration.record(duration, attributes); + this.telemetryMetrics.queryCount.add(1, attributes); + this.telemetryMetrics.queryDuration.record(duration, attributes); } }, - SQLWarehouseConnector.TELEMETRY_INSTRUMENT_CONFIG, + { name: this.name, includePrefix: true }, ); } @@ -199,7 +205,7 @@ export class SQLWarehouseConnector { timeout = executeStatementDefaults.timeout, signal?: AbortSignal, ) { - return this.config.telemetry.startActiveSpan( + return this.telemetry.startActiveSpan( "sql.poll", { attributes: { @@ -303,7 +309,7 @@ export class SQLWarehouseConnector { span.end(); } }, - SQLWarehouseConnector.TELEMETRY_INSTRUMENT_CONFIG, + { name: this.name, includePrefix: true }, ); } diff --git a/packages/backend/connectors/tests/lakebase.test.ts b/packages/app-kit/src/connectors/tests/lakebase.test.ts similarity index 99% rename from packages/backend/connectors/tests/lakebase.test.ts rename to packages/app-kit/src/connectors/tests/lakebase.test.ts index 969d2b4cc..059169ea1 100644 --- a/packages/backend/connectors/tests/lakebase.test.ts +++ b/packages/app-kit/src/connectors/tests/lakebase.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { LakebaseConnector } from "../src/lakebase/client"; +import { LakebaseConnector } from "../lakebase"; // Mock pg module vi.mock("pg", () => { diff --git a/packages/app-kit/src/core/app-kit.ts b/packages/app-kit/src/core/app-kit.ts index ed82ac13f..7aecaf76a 100644 --- a/packages/app-kit/src/core/app-kit.ts +++ b/packages/app-kit/src/core/app-kit.ts @@ -1,5 +1,6 @@ import type { BasePlugin, + CacheConfig, InputPluginMap, OptionalConfigPluginDef, PluginConstructor, @@ -84,10 +85,14 @@ export class AppKit { static async _createApp< T extends PluginData[], >( - config: { plugins?: T; telemetry?: TelemetryConfig } = {}, + config: { + plugins?: T; + telemetry?: TelemetryConfig; + cache?: CacheConfig; + } = {}, ): Promise> { - TelemetryManager.initialize(config.telemetry); - await CacheManager.getInstance(); + TelemetryManager.initialize(config?.telemetry); + await CacheManager.getInstance(config?.cache); const rawPlugins = config.plugins as T; const preparedPlugins = AppKit.preparePlugins(rawPlugins); @@ -118,6 +123,12 @@ export class AppKit { export async function createApp< T extends PluginData[], ->(config: { plugins?: T } = {}): Promise> { +>( + config: { + plugins?: T; + telemetry?: TelemetryConfig; + cache?: CacheConfig; + } = {}, +): Promise> { return AppKit._createApp(config); } diff --git a/packages/app-kit/src/telemetry/index.ts b/packages/app-kit/src/telemetry/index.ts index 523323033..2d2153aff 100644 --- a/packages/app-kit/src/telemetry/index.ts +++ b/packages/app-kit/src/telemetry/index.ts @@ -17,5 +17,7 @@ export { TelemetryProvider } from "./telemetry-provider"; export type { InstrumentConfig, ITelemetry, + MetricsSchema, TelemetryConfig, + TelemetryInstrumentation, } from "./types"; diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index d81b2dbdd..36e81c2a4 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -1,3 +1,5 @@ +import type { TelemetryOptions } from "./plugin"; + /** Configuration for caching */ export interface CacheConfig { /** Whether caching is enabled */ @@ -14,6 +16,8 @@ export interface CacheConfig { persistentCache?: boolean; /** Whether to enforce strict persistence */ strictPersistence?: boolean; + /** Telemetry configuration */ + telemetry?: TelemetryOptions; [key: string]: unknown; } From 7278318b5cf0808ca3f8d644feeeccd0598e4b6b Mon Sep 17 00:00:00 2001 From: Ditadi Date: Fri, 28 Nov 2025 14:21:27 +0000 Subject: [PATCH 11/18] fix: removing changes --- packages/app-kit/src/analytics/types.ts | 4 +--- packages/app-kit/src/connectors/sql-warehouse/client.ts | 2 +- packages/app-kit/src/telemetry/index.ts | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/app-kit/src/analytics/types.ts b/packages/app-kit/src/analytics/types.ts index c3211178c..5b56e4269 100644 --- a/packages/app-kit/src/analytics/types.ts +++ b/packages/app-kit/src/analytics/types.ts @@ -1,10 +1,8 @@ -import type { BasePluginConfig, TelemetryOptions } from "shared"; +import type { BasePluginConfig } from "shared"; import { z } from "zod"; export interface IAnalyticsConfig extends BasePluginConfig { timeout?: number; - typePath?: string; - telemetry?: TelemetryOptions; } export interface IAnalyticsQueryRequest { diff --git a/packages/app-kit/src/connectors/sql-warehouse/client.ts b/packages/app-kit/src/connectors/sql-warehouse/client.ts index db953ba8d..e7afe2812 100644 --- a/packages/app-kit/src/connectors/sql-warehouse/client.ts +++ b/packages/app-kit/src/connectors/sql-warehouse/client.ts @@ -3,6 +3,7 @@ import { type sql, type WorkspaceClient, } from "@databricks/sdk-experimental"; +import type { TelemetryOptions } from "shared"; import type { TelemetryProvider } from "../../telemetry"; import { type Counter, @@ -13,7 +14,6 @@ import { TelemetryManager, } from "../../telemetry"; import { executeStatementDefaults } from "./defaults"; -import type { TelemetryOptions } from "shared"; export interface SQLWarehouseConfig { timeout?: number; diff --git a/packages/app-kit/src/telemetry/index.ts b/packages/app-kit/src/telemetry/index.ts index 2d2153aff..523323033 100644 --- a/packages/app-kit/src/telemetry/index.ts +++ b/packages/app-kit/src/telemetry/index.ts @@ -17,7 +17,5 @@ export { TelemetryProvider } from "./telemetry-provider"; export type { InstrumentConfig, ITelemetry, - MetricsSchema, TelemetryConfig, - TelemetryInstrumentation, } from "./types"; From 40d9fbbf029fe02d471dfc8dbc632c2b156f507f Mon Sep 17 00:00:00 2001 From: Ditadi Date: Tue, 9 Dec 2025 12:18:45 +0000 Subject: [PATCH 12/18] fix: improve cache manager reliability and add missing config options --- apps/dev-playground/client/index.html | 25 ++ .../src/assets/databricks-logo-white.svg | 21 ++ .../client/src/assets/databricks-logo.svg | 21 ++ .../components/analytics/databricks-logo.tsx | 148 +++++------ .../client/src/components/theme-selector.tsx | 140 +++++++++++ .../client/src/routes/__root.tsx | 4 +- .../app-kit-ui/src/react/styles/globals.css | 46 +++- packages/app-kit/src/analytics/analytics.ts | 5 +- packages/app-kit/src/cache/index.ts | 22 +- .../app-kit/src/cache/storage/defaults.ts | 2 + .../app-kit/src/cache/storage/persistent.ts | 15 +- .../src/cache/tests/cache-manager.test.ts | 229 ++++++++++++++++++ .../app-kit/src/connectors/lakebase/client.ts | 26 +- .../app-kit/src/connectors/lakebase/types.ts | 3 + packages/shared/src/cache.ts | 6 + 15 files changed, 631 insertions(+), 82 deletions(-) create mode 100644 apps/dev-playground/client/src/assets/databricks-logo-white.svg create mode 100644 apps/dev-playground/client/src/assets/databricks-logo.svg create mode 100644 apps/dev-playground/client/src/components/theme-selector.tsx diff --git a/apps/dev-playground/client/index.html b/apps/dev-playground/client/index.html index a20a7a0a9..149833aef 100644 --- a/apps/dev-playground/client/index.html +++ b/apps/dev-playground/client/index.html @@ -7,6 +7,31 @@
+ diff --git a/apps/dev-playground/client/src/assets/databricks-logo-white.svg b/apps/dev-playground/client/src/assets/databricks-logo-white.svg new file mode 100644 index 000000000..5b67a675d --- /dev/null +++ b/apps/dev-playground/client/src/assets/databricks-logo-white.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/dev-playground/client/src/assets/databricks-logo.svg b/apps/dev-playground/client/src/assets/databricks-logo.svg new file mode 100644 index 000000000..efea6391d --- /dev/null +++ b/apps/dev-playground/client/src/assets/databricks-logo.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx b/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx index 69bf83855..64cce6f32 100644 --- a/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx +++ b/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx @@ -1,74 +1,84 @@ -import { useId } from "react"; +import { useEffect, useState } from "react"; +import databricksLogo from "@/assets/databricks-logo.svg"; +import databricksLogoWhite from "@/assets/databricks-logo-white.svg"; + +function useDarkMode() { + const [isDark, setIsDark] = useState(() => { + if (typeof window === "undefined") return false; + const root = document.documentElement; + // Check if dark class is explicitly set + if (root.classList.contains("dark")) return true; + if (root.classList.contains("light")) return false; + // Fallback to system preference + return window.matchMedia("(prefers-color-scheme: dark)").matches; + }); + + useEffect(() => { + if (typeof window === "undefined") return; + + const root = document.documentElement; + + const checkTheme = () => { + if (root.classList.contains("dark")) { + setIsDark(true); + } else if (root.classList.contains("light")) { + setIsDark(false); + } else { + // No explicit class, use system preference + setIsDark(window.matchMedia("(prefers-color-scheme: dark)").matches); + } + }; + + // Check initial theme + checkTheme(); + + // Observe changes to the classList + const observer = new MutationObserver(checkTheme); + observer.observe(root, { + attributes: true, + attributeFilter: ["class"], + }); + + // Also listen to system theme changes + const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); + const handleMediaChange = () => { + // Only update if no explicit class is set + if ( + !root.classList.contains("dark") && + !root.classList.contains("light") + ) { + setIsDark(mediaQuery.matches); + } + }; + + if (mediaQuery.addEventListener) { + mediaQuery.addEventListener("change", handleMediaChange); + return () => { + observer.disconnect(); + mediaQuery.removeEventListener("change", handleMediaChange); + }; + } else { + mediaQuery.addListener(handleMediaChange); + return () => { + observer.disconnect(); + mediaQuery.removeListener(handleMediaChange); + }; + } + }, []); + + return isDark; +} export function DatabricksLogo() { - const titleId = useId(); - const clipPathId = useId(); + const isDark = useDarkMode(); + const logoSrc = isDark ? databricksLogoWhite : databricksLogo; + return ( - - Databricks - - - - - - - - - - - - - - - - - - - - + Databricks ); } diff --git a/apps/dev-playground/client/src/components/theme-selector.tsx b/apps/dev-playground/client/src/components/theme-selector.tsx new file mode 100644 index 000000000..9cc96e0c8 --- /dev/null +++ b/apps/dev-playground/client/src/components/theme-selector.tsx @@ -0,0 +1,140 @@ +import { useEffect, useState } from "react"; +import { MoonIcon, SunIcon, MonitorIcon } from "lucide-react"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@databricks/app-kit-ui/react"; +import { Button } from "@databricks/app-kit-ui/react"; + +type Theme = "light" | "dark" | "system"; + +const THEME_STORAGE_KEY = "app-kit-playground-theme"; + +function getSystemTheme(): "light" | "dark" { + if (typeof window === "undefined") return "light"; + return window.matchMedia("(prefers-color-scheme: dark)").matches + ? "dark" + : "light"; +} + +function getStoredTheme(): Theme { + if (typeof window === "undefined") return "system"; + const stored = localStorage.getItem(THEME_STORAGE_KEY); + return (stored as Theme) || "system"; +} + +function applyTheme(theme: Theme) { + if (typeof window === "undefined") return; + + const root = document.documentElement; + root.classList.remove("light", "dark"); + + if (theme === "system") { + const systemTheme = getSystemTheme(); + root.classList.add(systemTheme); + } else { + root.classList.add(theme); + } +} + +export function ThemeSelector() { + const [theme, setTheme] = useState(() => getStoredTheme()); + const [mounted, setMounted] = useState(false); + const [systemTheme, setSystemTheme] = useState<"light" | "dark">(() => + getSystemTheme(), + ); + + useEffect(() => { + setMounted(true); + applyTheme(theme); + }, [theme]); + + useEffect(() => { + // Listen for system theme changes + const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); + const handleChange = (e: MediaQueryListEvent | MediaQueryList) => { + const isDark = e.matches; + setSystemTheme(isDark ? "dark" : "light"); + // Apply theme if current theme is "system" + if (theme === "system") { + applyTheme("system"); + } + }; + + // Set initial system theme + handleChange(mediaQuery); + + if (mediaQuery.addEventListener) { + mediaQuery.addEventListener("change", handleChange); + return () => mediaQuery.removeEventListener("change", handleChange); + } else { + mediaQuery.addListener(handleChange); + return () => mediaQuery.removeListener(handleChange); + } + }, [theme]); + + const handleThemeChange = (newTheme: Theme) => { + setTheme(newTheme); + localStorage.setItem(THEME_STORAGE_KEY, newTheme); + applyTheme(newTheme); + }; + + // Get current effective theme for icon display + const effectiveTheme = theme === "system" ? systemTheme : theme; + + if (!mounted) { + // Return a placeholder to avoid hydration mismatch + return ( + + ); + } + + return ( + + + + + + handleThemeChange("light")} + className="cursor-pointer" + > + + Light + {theme === "light" && ✓} + + handleThemeChange("dark")} + className="cursor-pointer" + > + + Dark + {theme === "dark" && ✓} + + handleThemeChange("system")} + className="cursor-pointer" + > + + System + {theme === "system" && ✓} + + + + ); +} diff --git a/apps/dev-playground/client/src/routes/__root.tsx b/apps/dev-playground/client/src/routes/__root.tsx index 376d8f0c6..14f417d3e 100644 --- a/apps/dev-playground/client/src/routes/__root.tsx +++ b/apps/dev-playground/client/src/routes/__root.tsx @@ -7,6 +7,7 @@ import { } from "@tanstack/react-router"; import { ErrorComponent } from "@/components/error-component"; import { Button, TooltipProvider } from "@databricks/app-kit-ui/react"; +import { ThemeSelector } from "@/components/theme-selector"; export const Route = createRootRoute({ component: RootComponent, @@ -30,7 +31,7 @@ function RootComponent() { App Kit Playground -
+
diff --git a/packages/app-kit-ui/src/react/styles/globals.css b/packages/app-kit-ui/src/react/styles/globals.css index 9650fcc95..79148aed9 100644 --- a/packages/app-kit-ui/src/react/styles/globals.css +++ b/packages/app-kit-ui/src/react/styles/globals.css @@ -42,8 +42,49 @@ --sidebar-ring: oklch(0.705 0.015 286.067); } +/* Dark theme via class (takes precedence over media query) */ +.dark { + --background: oklch(0.141 0.005 285.823); + --foreground: oklch(0.985 0 0); + --card: oklch(0.21 0.006 285.885); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.21 0.006 285.885); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.92 0.004 286.32); + --primary-foreground: oklch(0.21 0.006 285.885); + --secondary: oklch(0.274 0.006 286.033); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.274 0.006 286.033); + --muted-foreground: oklch(0.705 0.015 286.067); + --accent: oklch(0.274 0.006 286.033); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --destructive-foreground: oklch(0.985 0 0); + --success: oklch(0.67 0.12 167); + --success-foreground: oklch(1 0 0); + --warning: oklch(0.83 0.165 85); + --warning-foreground: oklch(0.199 0.027 238.732); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.552 0.016 285.938); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.21 0.006 285.885); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.274 0.006 286.033); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.552 0.016 285.938); +} + +/* Dark theme via media query (fallback when no class is set) */ @media (prefers-color-scheme: dark) { - :root { + :root:not(.light) { --background: oklch(0.141 0.005 285.823); --foreground: oklch(0.985 0 0); --card: oklch(0.21 0.006 285.885); @@ -217,8 +258,11 @@ *, *::before, *::after { + /* biome-ignore lint/complexity/noImportantStyles: !important is intentional for accessibility - forces reduced motion regardless of specificity */ animation-duration: 0.01ms !important; + /* biome-ignore lint/complexity/noImportantStyles: !important is intentional for accessibility - forces reduced motion regardless of specificity */ animation-iteration-count: 1 !important; + /* biome-ignore lint/complexity/noImportantStyles: !important is intentional for accessibility - forces reduced motion regardless of specificity */ transition-duration: 0.01ms !important; } } diff --git a/packages/app-kit/src/analytics/analytics.ts b/packages/app-kit/src/analytics/analytics.ts index 37233f228..744cf1cb8 100644 --- a/packages/app-kit/src/analytics/analytics.ts +++ b/packages/app-kit/src/analytics/analytics.ts @@ -185,9 +185,6 @@ export class AnalyticsPlugin extends Plugin { const schemaDir = path.join(process.cwd(), "config/queries"); const schemaPath = path.join(schemaDir, "schema.ts"); - const typePath = - this.config.typePath || path.join(process.cwd(), "client", "src"); - const generate = () => { let querySchemas: QuerySchemas = {}; try { @@ -201,7 +198,7 @@ export class AnalyticsPlugin extends Plugin { ); } } - generateQueryRegistryTypes(querySchemas, typePath); + generateQueryRegistryTypes(querySchemas); }; generate(); diff --git a/packages/app-kit/src/cache/index.ts b/packages/app-kit/src/cache/index.ts index 21dd3f187..be63b9806 100644 --- a/packages/app-kit/src/cache/index.ts +++ b/packages/app-kit/src/cache/index.ts @@ -28,7 +28,7 @@ export type { CacheEntry, CacheStorage } from "./storage"; * ``` */ export class CacheManager { - private static readonly CLEANUP_PROBABILITY = 0.01; + private static readonly MIN_CLEANUP_INTERVAL_MS = 60_000; private readonly name: string = "cache-manager"; private static instance: CacheManager | null = null; private static initPromise: Promise | null = null; @@ -37,6 +37,7 @@ export class CacheManager { private config: CacheConfig; private inFlightRequests: Map>; private cleanupInProgress: boolean; + private lastCleanupAttempt: number; // Telemetry private telemetry: TelemetryProvider; @@ -50,6 +51,7 @@ export class CacheManager { this.config = config; this.inFlightRequests = new Map(); this.cleanupInProgress = false; + this.lastCleanupAttempt = 0; this.telemetry = TelemetryManager.getProvider( this.name, @@ -132,8 +134,14 @@ export class CacheManager { await persistentStorage.initialize(); return new CacheManager(persistentStorage, config); } + + console.warn( + "[Cache] Persistent storage health check failed, storage unhealthy", + ); } catch (error) { - console.warn("[Cache] Persistent storage unavailable:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + console.warn(`[Cache] Persistent storage unavailable: ${errorMessage}`); } // if strict persistence is enabled, do not fallback to in-memory storage @@ -276,7 +284,15 @@ export class CacheManager { private maybeCleanup(): void { if (this.cleanupInProgress) return; if (!this.storage.isPersistent()) return; - if (Math.random() > CacheManager.CLEANUP_PROBABILITY) return; + const now = Date.now(); + if (now - this.lastCleanupAttempt < CacheManager.MIN_CLEANUP_INTERVAL_MS) + return; + + const probability = this.config.cleanupProbability ?? 0.01; + + if (Math.random() > probability) return; + + this.lastCleanupAttempt = now; this.cleanupInProgress = true; (this.storage as PersistentStorage) diff --git a/packages/app-kit/src/cache/storage/defaults.ts b/packages/app-kit/src/cache/storage/defaults.ts index e71d7cf66..ffc72c4d6 100644 --- a/packages/app-kit/src/cache/storage/defaults.ts +++ b/packages/app-kit/src/cache/storage/defaults.ts @@ -10,6 +10,8 @@ export const lakebaseStorageDefaults = { tableName: "appkit_cache_entries", /** Maximum number of bytes in the cache */ maxBytes: 256 * 1024 * 1024, // 256MB + /** Maximum number of bytes per entry in the cache */ + maxEntryBytes: 10 * 1024 * 1024, // 10MB /** Maximum number of entries in the cache */ maxSize: 1000, /** Number of entries to evict when cache is full */ diff --git a/packages/app-kit/src/cache/storage/persistent.ts b/packages/app-kit/src/cache/storage/persistent.ts index e04891310..8de5d4e99 100644 --- a/packages/app-kit/src/cache/storage/persistent.ts +++ b/packages/app-kit/src/cache/storage/persistent.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import type { LakebaseConnector } from "../../connectors"; import type { CacheConfig } from "shared"; +import type { LakebaseConnector } from "../../connectors"; import { lakebaseStorageDefaults } from "./defaults"; import type { CacheEntry, CacheStorage } from "./types"; @@ -22,14 +22,17 @@ export class PersistentStorage implements CacheStorage { private readonly connector: LakebaseConnector; private readonly tableName: string; private readonly maxBytes: number; + private readonly maxEntryBytes: number; private readonly evictionBatchSize: number; private initialized: boolean; constructor(config: CacheConfig, connector: LakebaseConnector) { this.connector = connector; this.maxBytes = config.maxBytes ?? lakebaseStorageDefaults.maxBytes; + this.maxEntryBytes = + config.maxEntryBytes ?? lakebaseStorageDefaults.maxEntryBytes; this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize; - this.tableName = lakebaseStorageDefaults.tableName; + this.tableName = lakebaseStorageDefaults.tableName; // hardcoded, safe for now this.initialized = false; } @@ -41,7 +44,7 @@ export class PersistentStorage implements CacheStorage { await this.runMigrations(); this.initialized = true; } catch (error) { - console.error("Error in for persistent storage initialization:", error); + console.error("Error in persistent storage initialization:", error); throw error; } } @@ -97,6 +100,12 @@ export class PersistentStorage implements CacheStorage { const valueBytes = this.serializeValue(entry.value); const byteSize = keyBytes.length + valueBytes.length; + if (byteSize > this.maxEntryBytes) { + throw new Error( + `Cache entry too large: ${byteSize} bytes exceeds maximum of ${this.maxEntryBytes} bytes`, + ); + } + const totalBytes = await this.totalBytes(); if (totalBytes + byteSize > this.maxBytes) { await this.evictBySize(byteSize); diff --git a/packages/app-kit/src/cache/tests/cache-manager.test.ts b/packages/app-kit/src/cache/tests/cache-manager.test.ts index 9c26c6b0e..e2bc3aae0 100644 --- a/packages/app-kit/src/cache/tests/cache-manager.test.ts +++ b/packages/app-kit/src/cache/tests/cache-manager.test.ts @@ -345,4 +345,233 @@ describe("CacheManager", () => { await expect(cache.close()).resolves.not.toThrow(); }); }); + + describe("maybeCleanup", () => { + test("should not trigger cleanup for non-persistent storage", async () => { + const cache = await CacheManager.getInstance({ + persistentCache: false, + cleanupProbability: 1, // 100% probability + }); + + // Access private method via reflection + const maybeCleanup = (cache as any).maybeCleanup.bind(cache); + const storage = (cache as any).storage; + + maybeCleanup(); + + // cleanupExpired should not exist on in-memory storage + expect(storage.isPersistent()).toBe(false); + }); + + test("should respect MIN_CLEANUP_INTERVAL_MS", async () => { + const cache = await CacheManager.getInstance({ + persistentCache: false, + cleanupProbability: 1, + }); + + // Simulate persistent storage + const mockStorage = { + isPersistent: vi.fn().mockReturnValue(true), + cleanupExpired: vi.fn().mockResolvedValue(5), + }; + (cache as any).storage = mockStorage; + (cache as any).lastCleanupAttempt = Date.now(); // Just cleaned up + + const maybeCleanup = (cache as any).maybeCleanup.bind(cache); + maybeCleanup(); + + // Should not trigger cleanup due to interval + expect(mockStorage.cleanupExpired).not.toHaveBeenCalled(); + }); + + test("should trigger cleanup when probability allows and interval passed", async () => { + const cache = await CacheManager.getInstance({ + persistentCache: false, + cleanupProbability: 1, // 100% probability + }); + + // Simulate persistent storage + const mockStorage = { + isPersistent: vi.fn().mockReturnValue(true), + cleanupExpired: vi.fn().mockResolvedValue(5), + }; + (cache as any).storage = mockStorage; + (cache as any).lastCleanupAttempt = 0; // Long time ago + + const maybeCleanup = (cache as any).maybeCleanup.bind(cache); + maybeCleanup(); + + // Should trigger cleanup + expect(mockStorage.cleanupExpired).toHaveBeenCalled(); + }); + + test("should not trigger cleanup when already in progress", async () => { + const cache = await CacheManager.getInstance({ + persistentCache: false, + cleanupProbability: 1, + }); + + const mockStorage = { + isPersistent: vi.fn().mockReturnValue(true), + cleanupExpired: vi.fn().mockResolvedValue(5), + }; + (cache as any).storage = mockStorage; + (cache as any).lastCleanupAttempt = 0; + (cache as any).cleanupInProgress = true; // Already running + + const maybeCleanup = (cache as any).maybeCleanup.bind(cache); + maybeCleanup(); + + expect(mockStorage.cleanupExpired).not.toHaveBeenCalled(); + }); + + test("should handle cleanup errors gracefully", async () => { + const cache = await CacheManager.getInstance({ + persistentCache: false, + cleanupProbability: 1, + }); + + const mockStorage = { + isPersistent: vi.fn().mockReturnValue(true), + cleanupExpired: vi.fn().mockRejectedValue(new Error("Cleanup failed")), + }; + (cache as any).storage = mockStorage; + (cache as any).lastCleanupAttempt = 0; + + const maybeCleanup = (cache as any).maybeCleanup.bind(cache); + + // Should not throw + expect(() => maybeCleanup()).not.toThrow(); + + // Wait for async cleanup to complete + await new Promise((resolve) => setTimeout(resolve, 10)); + + // cleanupInProgress should be reset + expect((cache as any).cleanupInProgress).toBe(false); + }); + }); + + describe("getOrExecute error handling", () => { + test("should propagate errors from executed function", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + const error = new Error("Execution failed"); + const fn = vi.fn().mockRejectedValue(error); + + await expect(cache.getOrExecute(["key"], fn, "user1")).rejects.toThrow( + "Execution failed", + ); + }); + + test("should remove in-flight request on error", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + const error = new Error("Execution failed"); + const fn = vi.fn().mockRejectedValue(error); + + try { + await cache.getOrExecute(["key"], fn, "user1"); + } catch { + // Expected + } + + // Verify in-flight request was cleaned up + const cacheKey = cache.generateKey(["key"], "user1"); + expect((cache as any).inFlightRequests.has(cacheKey)).toBe(false); + }); + + test("should allow retry after error", async () => { + const cache = await CacheManager.getInstance({ persistentCache: false }); + const fn = vi + .fn() + .mockRejectedValueOnce(new Error("First attempt failed")) + .mockResolvedValueOnce("success"); + + // First call fails + await expect(cache.getOrExecute(["key"], fn, "user1")).rejects.toThrow(); + + // Second call succeeds + const result = await cache.getOrExecute(["key"], fn, "user1"); + expect(result).toBe("success"); + }); + }); + + describe("strictPersistence mode", () => { + test("should disable cache when strictPersistence is true and storage unavailable", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + // Use persistentCache: true but env vars are not set, so it will fail + // and with strictPersistence: true, cache should be disabled + const cache = await CacheManager.getInstance({ + persistentCache: true, + strictPersistence: true, + }); + + // Should have logged about strictPersistence + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("strictPersistence"), + ); + + // Cache should be disabled + const fn = vi.fn().mockResolvedValue("result"); + await cache.getOrExecute(["key"], fn, "user1"); + await cache.getOrExecute(["key"], fn, "user1"); + + // Function called twice because cache is disabled + expect(fn).toHaveBeenCalledTimes(2); + + consoleSpy.mockRestore(); + }); + }); + + describe("persistent storage fallback", () => { + test("should fallback to in-memory when persistent storage unavailable", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + // Use persistentCache: true but env vars are not set, so it will fail + // and fallback to in-memory + const cache = await CacheManager.getInstance({ + persistentCache: true, + strictPersistence: false, + }); + + // Should log fallback message + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("[Cache]"), + ); + + // Cache should still work (in-memory) + await cache.set("test-key", "value"); + const result = await cache.get("test-key"); + expect(result).toBe("value"); + + consoleSpy.mockRestore(); + }); + + test("should log error message when persistent storage fails", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await CacheManager.getInstance({ + persistentCache: true, + strictPersistence: false, + }); + + // Should have logged about unavailable storage + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("[Cache] Persistent storage unavailable"), + ); + + consoleSpy.mockRestore(); + }); + }); }); diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index b3db22d1d..617bc9a71 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -278,7 +278,7 @@ export class LakebaseConnector { if (!this.connectionConfig) { throw new Error( "Lakebase connection not configured. " + - "Set LAKEBASE_CONNECTION_STRING env var or provide config in constructor.", + "Set PGHOST, PGDATABASE, PGAPPNAME env vars, provide a connectionString, or pass explicit config.", ); } @@ -442,6 +442,10 @@ export class LakebaseConnector { /** Parse connection configuration from config or environment */ private parseConnectionConfig(): LakebaseConnectionConfig { + if (this.config.connectionString) { + return this.parseConnectionString(this.config.connectionString); + } + // get connection from config if (this.config.host && this.config.database && this.config.appName) { return { @@ -482,4 +486,24 @@ export class LakebaseConnector { appName: pgAppName, }; } + + private parseConnectionString( + connectionString: string, + ): LakebaseConnectionConfig { + const url = new URL(connectionString); + const appName = url.searchParams.get("appName"); + if (!appName) { + throw new Error("Connection string must include appName parameter"); + } + + return { + host: url.hostname, + database: url.pathname.slice(1), // remove leading slash + port: url.port ? parseInt(url.port, 10) : 5432, + sslMode: + (url.searchParams.get("sslmode") as "require" | "disable" | "prefer") ?? + "require", + appName: appName, + }; + } } diff --git a/packages/app-kit/src/connectors/lakebase/types.ts b/packages/app-kit/src/connectors/lakebase/types.ts index d85c9d137..b65f88201 100644 --- a/packages/app-kit/src/connectors/lakebase/types.ts +++ b/packages/app-kit/src/connectors/lakebase/types.ts @@ -6,6 +6,9 @@ export interface LakebaseConfig { /** Databricks workspace client */ workspaceClient?: WorkspaceClient; + /** Connection string */ + connectionString?: string; + /** Database host (e.g., instance-uuid.database.region.databricks.com) */ host?: string; diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index 36e81c2a4..b324bb8f5 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -19,5 +19,11 @@ export interface CacheConfig { /** Telemetry configuration */ telemetry?: TelemetryOptions; + /** Probability (0-1) of triggering cleanup on each get operation */ + cleanupProbability?: number; + + /** Maximum number of bytes per entry in the cache */ + maxEntryBytes?: number; + [key: string]: unknown; } From ba9f0563dac47c15a1a577fcc468b23724df2076 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 10 Dec 2025 11:00:13 +0000 Subject: [PATCH 13/18] refactor(cache): replace persistentCache boolean with storage provider injection --- .../config/queries/.appkit-types-cache.json | 37 +++ packages/app-kit/src/cache/defaults.ts | 1 - packages/app-kit/src/cache/index.ts | 55 +++- packages/app-kit/src/cache/storage/index.ts | 1 - packages/app-kit/src/cache/storage/memory.ts | 3 +- .../app-kit/src/cache/storage/persistent.ts | 3 +- packages/app-kit/src/cache/storage/types.ts | 27 -- .../src/cache/tests/cache-manager.test.ts | 307 ++++++++++++++---- packages/shared/src/cache.ts | 36 +- 9 files changed, 356 insertions(+), 114 deletions(-) create mode 100644 apps/dev-playground/config/queries/.appkit-types-cache.json delete mode 100644 packages/app-kit/src/cache/storage/types.ts diff --git a/apps/dev-playground/config/queries/.appkit-types-cache.json b/apps/dev-playground/config/queries/.appkit-types-cache.json new file mode 100644 index 000000000..adf530d84 --- /dev/null +++ b/apps/dev-playground/config/queries/.appkit-types-cache.json @@ -0,0 +1,37 @@ +{ + "version": "1", + "queries": { + "apps_list": { + "hash": "e2c65853cf4b332d638bdd30a3aefb69", + "type": "{\n name: \"apps_list\";\n parameters: Record;\n result: Array<{\n /** @sqlType STRING */\n id: string;\n /** @sqlType STRING */\n name: string;\n /** @sqlType STRING */\n creator: string;\n /** @sqlType STRING */\n tags: string;\n /** @sqlType DECIMAL(38,6) */\n totalSpend: number;\n /** @sqlType DATE */\n createdAt: string;\n }>;\n }" + }, + "cost_recommendations": { + "hash": "730c7d8b5e2726981088b5975157b0da", + "type": "{\n name: \"cost_recommendations\";\n parameters: Record;\n result: Array<{\n /** @sqlType INT */\n dummy: number;\n }>;\n }" + }, + "example": { + "hash": "aeb02ed3e8a6c77279099406f8709543", + "type": "{\n name: \"example\";\n parameters: Record;\n result: Array<{\n /** @sqlType BOOLEAN */\n \"(1 = 1)\": boolean;\n }>;\n }" + }, + "spend_data": { + "hash": "caa0430652fe15eff658e48e6dac2446", + "type": "{\n name: \"spend_data\";\n parameters: {\n /** STRING - use sql.string() */\n groupBy: SQLStringMarker;\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n /** STRING - use sql.string() */\n appId: SQLStringMarker;\n /** STRING - use sql.string() */\n creator: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n group_key: string;\n /** @sqlType TIMESTAMP */\n aggregation_period: string;\n /** @sqlType DECIMAL(38,6) */\n cost_usd: number;\n }>;\n }" + }, + "spend_summary": { + "hash": "bbe188624c3f5904c3a7593cb32982d5", + "type": "{\n name: \"spend_summary\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType DECIMAL(33,0) */\n total: number;\n /** @sqlType DECIMAL(33,0) */\n average: number;\n /** @sqlType DECIMAL(33,0) */\n forecasted: number;\n }>;\n }" + }, + "sql_helpers_test": { + "hash": "1322df4ba9c107e8d23e2a04bae860c5", + "type": "{\n name: \"sql_helpers_test\";\n parameters: {\n /** STRING - use sql.string() */\n stringParam: SQLStringMarker;\n /** NUMERIC - use sql.number() */\n numberParam: SQLNumberMarker;\n /** BOOLEAN - use sql.boolean() */\n booleanParam: SQLBooleanMarker;\n /** DATE - use sql.date() */\n dateParam: SQLDateMarker;\n /** TIMESTAMP - use sql.timestamp() */\n timestampParam: SQLTimestampMarker;\n /** STRING - use sql.string() */\n binaryParam: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n string_value: string;\n /** @sqlType STRING */\n number_value: string;\n /** @sqlType STRING */\n boolean_value: string;\n /** @sqlType STRING */\n date_value: string;\n /** @sqlType STRING */\n timestamp_value: string;\n /** @sqlType BINARY */\n binary_value: string;\n /** @sqlType STRING */\n binary_hex: string;\n /** @sqlType INT */\n binary_length: number;\n }>;\n }" + }, + "top_contributors": { + "hash": "2d58759cca2fe31dae06475a23080120", + "type": "{\n name: \"top_contributors\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType DECIMAL(38,6) */\n total_cost_usd: number;\n }>;\n }" + }, + "untagged_apps": { + "hash": "5946262b49710b8ab458d1bf950ff8c9", + "type": "{\n name: \"untagged_apps\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType STRING */\n creator: string;\n /** @sqlType DECIMAL(38,6) */\n total_cost_usd: number;\n /** @sqlType DECIMAL(38,10) */\n avg_period_cost_usd: number;\n }>;\n }" + } + } +} diff --git a/packages/app-kit/src/cache/defaults.ts b/packages/app-kit/src/cache/defaults.ts index 492755c7e..3a48b9518 100644 --- a/packages/app-kit/src/cache/defaults.ts +++ b/packages/app-kit/src/cache/defaults.ts @@ -6,7 +6,6 @@ export const cacheDefaults: CacheConfig = { ttl: 3600, // 1 hour maxSize: 1000, // 1000 entries cacheKey: [], // no cache key by default - persistentCache: true, // use lakebase as persistent cache by default cleanupProbability: 0.01, // 1% probability of triggering cleanup on each get operation strictPersistence: false, // if false, use in-memory storage if lakebase is unavailable }; diff --git a/packages/app-kit/src/cache/index.ts b/packages/app-kit/src/cache/index.ts index be63b9806..a8f0c9fdf 100644 --- a/packages/app-kit/src/cache/index.ts +++ b/packages/app-kit/src/cache/index.ts @@ -1,18 +1,12 @@ import { createHash } from "node:crypto"; import { WorkspaceClient } from "@databricks/sdk-experimental"; -import type { CacheConfig } from "shared"; -import { LakebaseConnector } from "../connectors"; +import type { CacheConfig, CacheStorage } from "shared"; +import { LakebaseConnector } from "@/connectors"; import type { Counter, TelemetryProvider } from "../telemetry"; import { SpanStatusCode, TelemetryManager } from "../telemetry"; import { deepMerge } from "../utils"; import { cacheDefaults } from "./defaults"; -import { - type CacheStorage, - InMemoryStorage, - PersistentStorage, -} from "./storage"; - -export type { CacheEntry, CacheStorage } from "./storage"; +import { InMemoryStorage, PersistentStorage } from "./storage"; /** * Cache manager class to handle cache operations. @@ -71,6 +65,7 @@ export class CacheManager { /** * Get the singleton instance of the cache manager (sync version). + * * Throws if not initialized - ensure AppKit.create() has completed first. * @returns CacheManager instance */ @@ -112,6 +107,13 @@ export class CacheManager { /** * Create a new cache manager instance + * + * Storage selection logic: + * 1. If `storage` provided and healthy → use provided storage + * 2. If `storage` provided but unhealthy → fallback to InMemory (or disable if strictPersistence) + * 3. If no `storage` provided and Lakebase available → use Lakebase + * 4. If no `storage` provided and Lakebase unavailable → fallback to InMemory (or disable if strictPersistence) + * * @param userConfig - User configuration for the cache manager * @returns CacheManager instance */ @@ -120,10 +122,30 @@ export class CacheManager { ): Promise { const config = deepMerge(cacheDefaults, userConfig); - if (!config.persistentCache) { + if (config.storage) { + const isHealthy = await config.storage.healthCheck(); + if (isHealthy) { + return new CacheManager(config.storage, config); + } + + console.warn("[Cache] Provided storage health check failed"); + + if (config.strictPersistence) { + console.warn( + "[Cache] strictPersistence enabled but provided storage unhealthy. Cache disabled.", + ); + const disabledConfig = { ...config, enabled: false }; + return new CacheManager( + new InMemoryStorage(disabledConfig), + disabledConfig, + ); + } + + console.warn("[Cache] Falling back to in-memory cache."); return new CacheManager(new InMemoryStorage(config), config); } + // try to use lakebase storage try { const workspaceClient = new WorkspaceClient({}); const connector = new LakebaseConnector({ workspaceClient }); @@ -136,18 +158,17 @@ export class CacheManager { } console.warn( - "[Cache] Persistent storage health check failed, storage unhealthy", + "[Cache] Lakebase health check failed, default storage unhealthy", ); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); - console.warn(`[Cache] Persistent storage unavailable: ${errorMessage}`); + console.warn(`[Cache] Lakebase unavailable: ${errorMessage}`); } - // if strict persistence is enabled, do not fallback to in-memory storage if (config.strictPersistence) { console.warn( - "[Cache] strictPersistence enabled but persistent storage unavailable. Cache disabled.", + "[Cache] strictPersistence enabled but lakebase unavailable. Cache disabled.", ); const disabledConfig = { ...config, enabled: false }; return new CacheManager( @@ -189,7 +210,7 @@ export class CacheManager { }, async (span) => { try { - // Check cache first + // check if the value is in the cache const cached = await this.storage.get(cacheKey); if (cached !== null) { span.setAttribute("cache.hit", true); @@ -200,7 +221,7 @@ export class CacheManager { return cached.value as T; } - // Check in-flight requests for deduplication + // check if the value is being processed by another request const inFlight = this.inFlightRequests.get(cacheKey); if (inFlight) { span.setAttribute("cache.hit", true); @@ -217,7 +238,7 @@ export class CacheManager { return inFlight as Promise; } - // Cache miss - execute function + // cache miss - execute function span.setAttribute("cache.hit", false); span.addEvent("cache.miss", { "cache.key": cacheKey }); this.telemetryMetrics.cacheMissCount.add(1, { diff --git a/packages/app-kit/src/cache/storage/index.ts b/packages/app-kit/src/cache/storage/index.ts index 2b593d952..9d213c7c5 100644 --- a/packages/app-kit/src/cache/storage/index.ts +++ b/packages/app-kit/src/cache/storage/index.ts @@ -1,3 +1,2 @@ export { InMemoryStorage } from "./memory"; export { PersistentStorage } from "./persistent"; -export type { CacheEntry, CacheStorage } from "./types"; diff --git a/packages/app-kit/src/cache/storage/memory.ts b/packages/app-kit/src/cache/storage/memory.ts index 22bf3e883..23f36e08a 100644 --- a/packages/app-kit/src/cache/storage/memory.ts +++ b/packages/app-kit/src/cache/storage/memory.ts @@ -1,6 +1,5 @@ -import type { CacheConfig } from "shared"; +import type { CacheConfig, CacheEntry, CacheStorage } from "shared"; import { inMemoryStorageDefaults } from "./defaults"; -import type { CacheEntry, CacheStorage } from "./types"; /** * In-memory cache storage implementation. Uses a least recently used (LRU) eviction policy diff --git a/packages/app-kit/src/cache/storage/persistent.ts b/packages/app-kit/src/cache/storage/persistent.ts index 8de5d4e99..2e38a8eec 100644 --- a/packages/app-kit/src/cache/storage/persistent.ts +++ b/packages/app-kit/src/cache/storage/persistent.ts @@ -1,8 +1,7 @@ import { createHash } from "node:crypto"; -import type { CacheConfig } from "shared"; +import type { CacheConfig, CacheEntry, CacheStorage } from "shared"; import type { LakebaseConnector } from "../../connectors"; import { lakebaseStorageDefaults } from "./defaults"; -import type { CacheEntry, CacheStorage } from "./types"; /** * Persistent cache storage implementation. Uses a least recently used (LRU) eviction policy diff --git a/packages/app-kit/src/cache/storage/types.ts b/packages/app-kit/src/cache/storage/types.ts deleted file mode 100644 index 45997b923..000000000 --- a/packages/app-kit/src/cache/storage/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** Cache entry interface */ -export interface CacheEntry { - value: T; - expiry: number; -} - -/** Cache storage interface */ -export interface CacheStorage { - /** Get a cached value from the storage */ - get(key: string): Promise | null>; - /** Set a value in the storage */ - set(key: string, entry: CacheEntry): Promise; - /** Delete a value from the storage */ - delete(key: string): Promise; - /** Clear the storage */ - clear(): Promise; - /** Check if a value exists in the storage */ - has(key: string): Promise; - /** Get the size of the storage */ - size(): Promise; - /** Check if the storage is persistent */ - isPersistent(): boolean; - /** Check if the storage is healthy */ - healthCheck(): Promise; - /** Close the storage */ - close(): Promise; -} diff --git a/packages/app-kit/src/cache/tests/cache-manager.test.ts b/packages/app-kit/src/cache/tests/cache-manager.test.ts index e2bc3aae0..7fc45c597 100644 --- a/packages/app-kit/src/cache/tests/cache-manager.test.ts +++ b/packages/app-kit/src/cache/tests/cache-manager.test.ts @@ -1,35 +1,51 @@ +import type { CacheStorage } from "shared"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { CacheManager } from "../../index"; -import type { CacheStorage } from "../storage"; - -// Mock the storage modules -vi.mock("../storage/memory", () => ({ - InMemoryStorage: vi.fn().mockImplementation(() => createMockStorage()), -})); - -vi.mock("../src/storage/persistent", () => ({ - PersistentStorage: vi.fn().mockImplementation(() => { - const storage = createMockStorage(); - storage.isPersistent = vi.fn().mockReturnValue(true); - return storage; - }), -})); // Mock LakebaseConnector -vi.mock("@databricks-apps/connectors", () => ({ +const mockLakebaseHealthCheck = vi.fn(); +vi.mock("@/connectors", () => ({ LakebaseConnector: vi.fn().mockImplementation(() => ({ - healthCheck: vi.fn().mockResolvedValue(true), + healthCheck: mockLakebaseHealthCheck, close: vi.fn().mockResolvedValue(undefined), })), })); +// Mock PersistentStorage +vi.mock("../storage/persistent", () => ({ + PersistentStorage: vi.fn().mockImplementation(() => { + const cache = new Map(); + return { + initialize: vi.fn().mockResolvedValue(undefined), + get: vi + .fn() + .mockImplementation(async (key: string) => cache.get(key) || null), + set: vi + .fn() + .mockImplementation(async (key: string, entry: any) => + cache.set(key, entry), + ), + delete: vi + .fn() + .mockImplementation(async (key: string) => cache.delete(key)), + clear: vi.fn().mockImplementation(async () => cache.clear()), + has: vi.fn().mockImplementation(async (key: string) => cache.has(key)), + size: vi.fn().mockImplementation(async () => cache.size), + isPersistent: vi.fn().mockReturnValue(true), + healthCheck: vi.fn().mockResolvedValue(true), + close: vi.fn().mockResolvedValue(undefined), + cleanupExpired: vi.fn().mockResolvedValue(0), + }; + }), +})); + // Mock WorkspaceClient vi.mock("@databricks/sdk-experimental", () => ({ WorkspaceClient: vi.fn().mockImplementation(() => ({})), })); /** Create a mock storage for testing */ -function createMockStorage(): CacheStorage { +function createMockStorage(persistent = false): CacheStorage { const cache = new Map(); return { @@ -51,18 +67,27 @@ function createMockStorage(): CacheStorage { size: vi.fn().mockImplementation(async () => { return cache.size; }), - isPersistent: vi.fn().mockReturnValue(false), + isPersistent: vi.fn().mockReturnValue(persistent), healthCheck: vi.fn().mockResolvedValue(true), close: vi.fn().mockResolvedValue(undefined), }; } +/** Create a mock storage with healthCheck returning false */ +function createUnhealthyMockStorage(): CacheStorage { + const storage = createMockStorage(); + storage.healthCheck = vi.fn().mockResolvedValue(false); + return storage; +} + describe("CacheManager", () => { // Reset singleton between tests beforeEach(() => { // Access private static fields to reset singleton (CacheManager as any).instance = null; (CacheManager as any).initPromise = null; + // Default: Lakebase unavailable (most tests pass explicit storage) + mockLakebaseHealthCheck.mockResolvedValue(false); }); afterEach(() => { @@ -78,7 +103,7 @@ describe("CacheManager", () => { test("getInstance should create singleton", async () => { const instance1 = await CacheManager.getInstance({ - persistentCache: false, + storage: createMockStorage(), }); const instance2 = await CacheManager.getInstance(); @@ -86,7 +111,7 @@ describe("CacheManager", () => { }); test("getInstanceSync should return instance after initialization", async () => { - await CacheManager.getInstance({ persistentCache: false }); + await CacheManager.getInstance({ storage: createMockStorage() }); const instance = CacheManager.getInstanceSync(); @@ -96,7 +121,9 @@ describe("CacheManager", () => { describe("generateKey", () => { test("should generate consistent hash for same inputs", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const key1 = cache.generateKey(["users", 123], "user1"); const key2 = cache.generateKey(["users", 123], "user1"); @@ -105,7 +132,9 @@ describe("CacheManager", () => { }); test("should generate different hash for different inputs", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const key1 = cache.generateKey(["users", 123], "user1"); const key2 = cache.generateKey(["users", 456], "user1"); @@ -117,7 +146,9 @@ describe("CacheManager", () => { }); test("should handle objects in key parts", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const key1 = cache.generateKey([{ filter: "active" }], "user1"); const key2 = cache.generateKey([{ filter: "active" }], "user1"); @@ -130,7 +161,9 @@ describe("CacheManager", () => { describe("get/set operations", () => { test("should return null for non-existent key", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const result = await cache.get("non-existent"); @@ -138,7 +171,9 @@ describe("CacheManager", () => { }); test("should set and get value", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await cache.set("test-key", { data: "test-value" }); const result = await cache.get("test-key"); @@ -147,7 +182,9 @@ describe("CacheManager", () => { }); test("should respect TTL expiry", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); // Set with very short TTL await cache.set("test-key", "value", { ttl: 0.001 }); // 1ms @@ -163,7 +200,9 @@ describe("CacheManager", () => { describe("delete operation", () => { test("should delete existing key", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await cache.set("test-key", "value"); await cache.delete("test-key"); @@ -175,7 +214,9 @@ describe("CacheManager", () => { describe("has operation", () => { test("should return true for existing key", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await cache.set("test-key", "value"); @@ -184,14 +225,18 @@ describe("CacheManager", () => { }); test("should return false for non-existent key", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const exists = await cache.has("non-existent"); expect(exists).toBe(false); }); test("should return false for expired key", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await cache.set("test-key", "value", { ttl: 0.001 }); await new Promise((resolve) => setTimeout(resolve, 10)); @@ -203,7 +248,9 @@ describe("CacheManager", () => { describe("clear operation", () => { test("should clear all entries", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await cache.set("key1", "value1"); await cache.set("key2", "value2"); @@ -217,7 +264,9 @@ describe("CacheManager", () => { describe("getOrExecute", () => { test("should execute function on cache miss", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const fn = vi.fn().mockResolvedValue("result"); const result = await cache.getOrExecute(["key"], fn, "user1"); @@ -227,7 +276,9 @@ describe("CacheManager", () => { }); test("should return cached value on cache hit", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const fn = vi.fn().mockResolvedValue("new-result"); // First call - populates cache @@ -241,7 +292,9 @@ describe("CacheManager", () => { }); test("should deduplicate concurrent requests", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); let callCount = 0; const fn = vi.fn().mockImplementation(async () => { callCount++; @@ -266,7 +319,9 @@ describe("CacheManager", () => { }); test("should use different cache keys for different users", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await cache.getOrExecute(["key"], async () => "user1-data", "user1"); await cache.getOrExecute(["key"], async () => "user2-data", "user2"); @@ -291,7 +346,7 @@ describe("CacheManager", () => { test("should bypass cache when disabled", async () => { const cache = await CacheManager.getInstance({ enabled: false, - persistentCache: false, + storage: createMockStorage(), }); const fn = vi.fn().mockResolvedValue("result"); @@ -306,7 +361,7 @@ describe("CacheManager", () => { test("should return null for get when disabled", async () => { const cache = await CacheManager.getInstance({ enabled: false, - persistentCache: false, + storage: createMockStorage(), }); await cache.set("test-key", "value"); @@ -318,7 +373,7 @@ describe("CacheManager", () => { test("should return false for has when disabled", async () => { const cache = await CacheManager.getInstance({ enabled: false, - persistentCache: false, + storage: createMockStorage(), }); await cache.set("test-key", "value"); @@ -330,7 +385,9 @@ describe("CacheManager", () => { describe("storage health", () => { test("should check storage health", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const isHealthy = await cache.isStorageHealthy(); @@ -340,7 +397,9 @@ describe("CacheManager", () => { describe("close", () => { test("should close storage", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); await expect(cache.close()).resolves.not.toThrow(); }); @@ -349,7 +408,7 @@ describe("CacheManager", () => { describe("maybeCleanup", () => { test("should not trigger cleanup for non-persistent storage", async () => { const cache = await CacheManager.getInstance({ - persistentCache: false, + storage: createMockStorage(false), cleanupProbability: 1, // 100% probability }); @@ -365,7 +424,7 @@ describe("CacheManager", () => { test("should respect MIN_CLEANUP_INTERVAL_MS", async () => { const cache = await CacheManager.getInstance({ - persistentCache: false, + storage: createMockStorage(), cleanupProbability: 1, }); @@ -386,7 +445,7 @@ describe("CacheManager", () => { test("should trigger cleanup when probability allows and interval passed", async () => { const cache = await CacheManager.getInstance({ - persistentCache: false, + storage: createMockStorage(), cleanupProbability: 1, // 100% probability }); @@ -407,7 +466,7 @@ describe("CacheManager", () => { test("should not trigger cleanup when already in progress", async () => { const cache = await CacheManager.getInstance({ - persistentCache: false, + storage: createMockStorage(), cleanupProbability: 1, }); @@ -427,7 +486,7 @@ describe("CacheManager", () => { test("should handle cleanup errors gracefully", async () => { const cache = await CacheManager.getInstance({ - persistentCache: false, + storage: createMockStorage(), cleanupProbability: 1, }); @@ -453,7 +512,9 @@ describe("CacheManager", () => { describe("getOrExecute error handling", () => { test("should propagate errors from executed function", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const error = new Error("Execution failed"); const fn = vi.fn().mockRejectedValue(error); @@ -463,7 +524,9 @@ describe("CacheManager", () => { }); test("should remove in-flight request on error", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const error = new Error("Execution failed"); const fn = vi.fn().mockRejectedValue(error); @@ -479,7 +542,9 @@ describe("CacheManager", () => { }); test("should allow retry after error", async () => { - const cache = await CacheManager.getInstance({ persistentCache: false }); + const cache = await CacheManager.getInstance({ + storage: createMockStorage(), + }); const fn = vi .fn() .mockRejectedValueOnce(new Error("First attempt failed")) @@ -495,17 +560,16 @@ describe("CacheManager", () => { }); describe("strictPersistence mode", () => { - test("should disable cache when strictPersistence is true and storage unavailable", async () => { + test("should disable cache when strictPersistence is true and storage unhealthy", async () => { // Reset singleton (CacheManager as any).instance = null; (CacheManager as any).initPromise = null; const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - // Use persistentCache: true but env vars are not set, so it will fail - // and with strictPersistence: true, cache should be disabled + // Pass an unhealthy storage with strictPersistence: true const cache = await CacheManager.getInstance({ - persistentCache: true, + storage: createUnhealthyMockStorage(), strictPersistence: true, }); @@ -526,18 +590,17 @@ describe("CacheManager", () => { }); }); - describe("persistent storage fallback", () => { - test("should fallback to in-memory when persistent storage unavailable", async () => { + describe("storage fallback", () => { + test("should fallback to in-memory when provided storage is unhealthy", async () => { // Reset singleton (CacheManager as any).instance = null; (CacheManager as any).initPromise = null; const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - // Use persistentCache: true but env vars are not set, so it will fail - // and fallback to in-memory + // Pass an unhealthy storage, should fallback to in-memory const cache = await CacheManager.getInstance({ - persistentCache: true, + storage: createUnhealthyMockStorage(), strictPersistence: false, }); @@ -546,7 +609,7 @@ describe("CacheManager", () => { expect.stringContaining("[Cache]"), ); - // Cache should still work (in-memory) + // Cache should still work (in-memory fallback) await cache.set("test-key", "value"); const result = await cache.get("test-key"); expect(result).toBe("value"); @@ -554,7 +617,7 @@ describe("CacheManager", () => { consoleSpy.mockRestore(); }); - test("should log error message when persistent storage fails", async () => { + test("should log warning when provided storage health check fails", async () => { // Reset singleton (CacheManager as any).instance = null; (CacheManager as any).initPromise = null; @@ -562,13 +625,133 @@ describe("CacheManager", () => { const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); await CacheManager.getInstance({ - persistentCache: true, + storage: createUnhealthyMockStorage(), strictPersistence: false, }); - // Should have logged about unavailable storage + // Should have logged about storage health check failing + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("[Cache] Provided storage health check failed"), + ); + + consoleSpy.mockRestore(); + }); + }); + + describe("lakebase default storage", () => { + test("should use Lakebase when no storage provided and Lakebase is available", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + // Make Lakebase healthy + mockLakebaseHealthCheck.mockResolvedValue(true); + + const cache = await CacheManager.getInstance({}); + + // Storage should be persistent (Lakebase) + const storage = (cache as any).storage; + expect(storage.isPersistent()).toBe(true); + }); + + test("should fallback to in-memory when Lakebase is unavailable", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + // Lakebase unhealthy (default in beforeEach) + mockLakebaseHealthCheck.mockResolvedValue(false); + + const cache = await CacheManager.getInstance({}); + + // Should log fallback message + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("[Cache] Falling back to in-memory cache"), + ); + + // Cache should work (in-memory fallback) + await cache.set("test-key", "value"); + const result = await cache.get("test-key"); + expect(result).toBe("value"); + + // Storage should not be persistent + const storage = (cache as any).storage; + expect(storage.isPersistent()).toBe(false); + + consoleSpy.mockRestore(); + }); + + test("should disable cache when Lakebase unavailable and strictPersistence is true", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + // Lakebase unhealthy + mockLakebaseHealthCheck.mockResolvedValue(false); + + const cache = await CacheManager.getInstance({ + strictPersistence: true, + }); + + // Should have logged about strictPersistence + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining( + "strictPersistence enabled but lakebase unavailable", + ), + ); + + // Cache should be disabled + const fn = vi.fn().mockResolvedValue("result"); + await cache.getOrExecute(["key"], fn, "user1"); + await cache.getOrExecute(["key"], fn, "user1"); + + // Function called twice because cache is disabled + expect(fn).toHaveBeenCalledTimes(2); + + consoleSpy.mockRestore(); + }); + + test("should log warning when Lakebase health check fails", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + // Lakebase unhealthy + mockLakebaseHealthCheck.mockResolvedValue(false); + + await CacheManager.getInstance({}); + + // Should have logged about Lakebase health check + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining("[Cache] Lakebase health check failed"), + ); + + consoleSpy.mockRestore(); + }); + + test("should log warning when Lakebase throws an error", async () => { + // Reset singleton + (CacheManager as any).instance = null; + (CacheManager as any).initPromise = null; + + const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + // Lakebase throws + mockLakebaseHealthCheck.mockRejectedValue( + new Error("Connection refused"), + ); + + await CacheManager.getInstance({}); + + // Should have logged about Lakebase being unavailable expect(consoleSpy).toHaveBeenCalledWith( - expect.stringContaining("[Cache] Persistent storage unavailable"), + expect.stringContaining("[Cache] Lakebase unavailable"), ); consoleSpy.mockRestore(); diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index b324bb8f5..6bc7be000 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -1,5 +1,37 @@ import type { TelemetryOptions } from "./plugin"; +/** Cache entry interface */ +export interface CacheEntry { + value: T; + expiry: number; +} + +/** + * Cache storage interface for custom implementations + * - InMemoryStorage + * - PersistentStorage (Lakebase) + */ +export interface CacheStorage { + /** Get a cached value from the storage */ + get(key: string): Promise | null>; + /** Set a value in the storage */ + set(key: string, entry: CacheEntry): Promise; + /** Delete a value from the storage */ + delete(key: string): Promise; + /** Clear the storage */ + clear(): Promise; + /** Check if a value exists in the storage */ + has(key: string): Promise; + /** Get the size of the storage */ + size(): Promise; + /** Check if the storage is persistent */ + isPersistent(): boolean; + /** Check if the storage is healthy */ + healthCheck(): Promise; + /** Close the storage */ + close(): Promise; +} + /** Configuration for caching */ export interface CacheConfig { /** Whether caching is enabled */ @@ -12,8 +44,8 @@ export interface CacheConfig { maxSize?: number; /** Cache key */ cacheKey?: (string | number | object)[]; - /** Whether to use persistent cache */ - persistentCache?: boolean; + /** Cache Storage provider instance */ + storage?: CacheStorage; /** Whether to enforce strict persistence */ strictPersistence?: boolean; /** Telemetry configuration */ From f1ca1c6ff1e57bdc906424f90e58d305591f37b8 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 10 Dec 2025 11:06:05 +0000 Subject: [PATCH 14/18] chore: extract theme selector to another PR --- apps/dev-playground/client/index.html | 25 --- .../src/assets/databricks-logo-white.svg | 21 --- .../client/src/assets/databricks-logo.svg | 21 --- .../components/analytics/databricks-logo.tsx | 148 ++++++++---------- .../client/src/components/theme-selector.tsx | 140 ----------------- .../client/src/routes/__root.tsx | 12 +- .../app-kit-ui/src/react/styles/globals.css | 46 +----- 7 files changed, 79 insertions(+), 334 deletions(-) delete mode 100644 apps/dev-playground/client/src/assets/databricks-logo-white.svg delete mode 100644 apps/dev-playground/client/src/assets/databricks-logo.svg delete mode 100644 apps/dev-playground/client/src/components/theme-selector.tsx diff --git a/apps/dev-playground/client/index.html b/apps/dev-playground/client/index.html index 149833aef..a20a7a0a9 100644 --- a/apps/dev-playground/client/index.html +++ b/apps/dev-playground/client/index.html @@ -7,31 +7,6 @@
- diff --git a/apps/dev-playground/client/src/assets/databricks-logo-white.svg b/apps/dev-playground/client/src/assets/databricks-logo-white.svg deleted file mode 100644 index 5b67a675d..000000000 --- a/apps/dev-playground/client/src/assets/databricks-logo-white.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/apps/dev-playground/client/src/assets/databricks-logo.svg b/apps/dev-playground/client/src/assets/databricks-logo.svg deleted file mode 100644 index efea6391d..000000000 --- a/apps/dev-playground/client/src/assets/databricks-logo.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx b/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx index 64cce6f32..69bf83855 100644 --- a/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx +++ b/apps/dev-playground/client/src/components/analytics/databricks-logo.tsx @@ -1,84 +1,74 @@ -import { useEffect, useState } from "react"; -import databricksLogo from "@/assets/databricks-logo.svg"; -import databricksLogoWhite from "@/assets/databricks-logo-white.svg"; - -function useDarkMode() { - const [isDark, setIsDark] = useState(() => { - if (typeof window === "undefined") return false; - const root = document.documentElement; - // Check if dark class is explicitly set - if (root.classList.contains("dark")) return true; - if (root.classList.contains("light")) return false; - // Fallback to system preference - return window.matchMedia("(prefers-color-scheme: dark)").matches; - }); - - useEffect(() => { - if (typeof window === "undefined") return; - - const root = document.documentElement; - - const checkTheme = () => { - if (root.classList.contains("dark")) { - setIsDark(true); - } else if (root.classList.contains("light")) { - setIsDark(false); - } else { - // No explicit class, use system preference - setIsDark(window.matchMedia("(prefers-color-scheme: dark)").matches); - } - }; - - // Check initial theme - checkTheme(); - - // Observe changes to the classList - const observer = new MutationObserver(checkTheme); - observer.observe(root, { - attributes: true, - attributeFilter: ["class"], - }); - - // Also listen to system theme changes - const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); - const handleMediaChange = () => { - // Only update if no explicit class is set - if ( - !root.classList.contains("dark") && - !root.classList.contains("light") - ) { - setIsDark(mediaQuery.matches); - } - }; - - if (mediaQuery.addEventListener) { - mediaQuery.addEventListener("change", handleMediaChange); - return () => { - observer.disconnect(); - mediaQuery.removeEventListener("change", handleMediaChange); - }; - } else { - mediaQuery.addListener(handleMediaChange); - return () => { - observer.disconnect(); - mediaQuery.removeListener(handleMediaChange); - }; - } - }, []); - - return isDark; -} +import { useId } from "react"; export function DatabricksLogo() { - const isDark = useDarkMode(); - const logoSrc = isDark ? databricksLogoWhite : databricksLogo; - + const titleId = useId(); + const clipPathId = useId(); return ( - Databricks + + Databricks + + + + + + + + + + + + + + + + + + + + ); } diff --git a/apps/dev-playground/client/src/components/theme-selector.tsx b/apps/dev-playground/client/src/components/theme-selector.tsx deleted file mode 100644 index 9cc96e0c8..000000000 --- a/apps/dev-playground/client/src/components/theme-selector.tsx +++ /dev/null @@ -1,140 +0,0 @@ -import { useEffect, useState } from "react"; -import { MoonIcon, SunIcon, MonitorIcon } from "lucide-react"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "@databricks/app-kit-ui/react"; -import { Button } from "@databricks/app-kit-ui/react"; - -type Theme = "light" | "dark" | "system"; - -const THEME_STORAGE_KEY = "app-kit-playground-theme"; - -function getSystemTheme(): "light" | "dark" { - if (typeof window === "undefined") return "light"; - return window.matchMedia("(prefers-color-scheme: dark)").matches - ? "dark" - : "light"; -} - -function getStoredTheme(): Theme { - if (typeof window === "undefined") return "system"; - const stored = localStorage.getItem(THEME_STORAGE_KEY); - return (stored as Theme) || "system"; -} - -function applyTheme(theme: Theme) { - if (typeof window === "undefined") return; - - const root = document.documentElement; - root.classList.remove("light", "dark"); - - if (theme === "system") { - const systemTheme = getSystemTheme(); - root.classList.add(systemTheme); - } else { - root.classList.add(theme); - } -} - -export function ThemeSelector() { - const [theme, setTheme] = useState(() => getStoredTheme()); - const [mounted, setMounted] = useState(false); - const [systemTheme, setSystemTheme] = useState<"light" | "dark">(() => - getSystemTheme(), - ); - - useEffect(() => { - setMounted(true); - applyTheme(theme); - }, [theme]); - - useEffect(() => { - // Listen for system theme changes - const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); - const handleChange = (e: MediaQueryListEvent | MediaQueryList) => { - const isDark = e.matches; - setSystemTheme(isDark ? "dark" : "light"); - // Apply theme if current theme is "system" - if (theme === "system") { - applyTheme("system"); - } - }; - - // Set initial system theme - handleChange(mediaQuery); - - if (mediaQuery.addEventListener) { - mediaQuery.addEventListener("change", handleChange); - return () => mediaQuery.removeEventListener("change", handleChange); - } else { - mediaQuery.addListener(handleChange); - return () => mediaQuery.removeListener(handleChange); - } - }, [theme]); - - const handleThemeChange = (newTheme: Theme) => { - setTheme(newTheme); - localStorage.setItem(THEME_STORAGE_KEY, newTheme); - applyTheme(newTheme); - }; - - // Get current effective theme for icon display - const effectiveTheme = theme === "system" ? systemTheme : theme; - - if (!mounted) { - // Return a placeholder to avoid hydration mismatch - return ( - - ); - } - - return ( - - - - - - handleThemeChange("light")} - className="cursor-pointer" - > - - Light - {theme === "light" && ✓} - - handleThemeChange("dark")} - className="cursor-pointer" - > - - Dark - {theme === "dark" && ✓} - - handleThemeChange("system")} - className="cursor-pointer" - > - - System - {theme === "system" && ✓} - - - - ); -} diff --git a/apps/dev-playground/client/src/routes/__root.tsx b/apps/dev-playground/client/src/routes/__root.tsx index 14f417d3e..c4baee29b 100644 --- a/apps/dev-playground/client/src/routes/__root.tsx +++ b/apps/dev-playground/client/src/routes/__root.tsx @@ -7,7 +7,6 @@ import { } from "@tanstack/react-router"; import { ErrorComponent } from "@/components/error-component"; import { Button, TooltipProvider } from "@databricks/app-kit-ui/react"; -import { ThemeSelector } from "@/components/theme-selector"; export const Route = createRootRoute({ component: RootComponent, @@ -31,7 +30,7 @@ function RootComponent() { App Kit Playground -
+
+
diff --git a/packages/app-kit-ui/src/react/styles/globals.css b/packages/app-kit-ui/src/react/styles/globals.css index 79148aed9..9650fcc95 100644 --- a/packages/app-kit-ui/src/react/styles/globals.css +++ b/packages/app-kit-ui/src/react/styles/globals.css @@ -42,49 +42,8 @@ --sidebar-ring: oklch(0.705 0.015 286.067); } -/* Dark theme via class (takes precedence over media query) */ -.dark { - --background: oklch(0.141 0.005 285.823); - --foreground: oklch(0.985 0 0); - --card: oklch(0.21 0.006 285.885); - --card-foreground: oklch(0.985 0 0); - --popover: oklch(0.21 0.006 285.885); - --popover-foreground: oklch(0.985 0 0); - --primary: oklch(0.92 0.004 286.32); - --primary-foreground: oklch(0.21 0.006 285.885); - --secondary: oklch(0.274 0.006 286.033); - --secondary-foreground: oklch(0.985 0 0); - --muted: oklch(0.274 0.006 286.033); - --muted-foreground: oklch(0.705 0.015 286.067); - --accent: oklch(0.274 0.006 286.033); - --accent-foreground: oklch(0.985 0 0); - --destructive: oklch(0.704 0.191 22.216); - --destructive-foreground: oklch(0.985 0 0); - --success: oklch(0.67 0.12 167); - --success-foreground: oklch(1 0 0); - --warning: oklch(0.83 0.165 85); - --warning-foreground: oklch(0.199 0.027 238.732); - --border: oklch(1 0 0 / 10%); - --input: oklch(1 0 0 / 15%); - --ring: oklch(0.552 0.016 285.938); - --chart-1: oklch(0.488 0.243 264.376); - --chart-2: oklch(0.696 0.17 162.48); - --chart-3: oklch(0.769 0.188 70.08); - --chart-4: oklch(0.627 0.265 303.9); - --chart-5: oklch(0.645 0.246 16.439); - --sidebar: oklch(0.21 0.006 285.885); - --sidebar-foreground: oklch(0.985 0 0); - --sidebar-primary: oklch(0.488 0.243 264.376); - --sidebar-primary-foreground: oklch(0.985 0 0); - --sidebar-accent: oklch(0.274 0.006 286.033); - --sidebar-accent-foreground: oklch(0.985 0 0); - --sidebar-border: oklch(1 0 0 / 10%); - --sidebar-ring: oklch(0.552 0.016 285.938); -} - -/* Dark theme via media query (fallback when no class is set) */ @media (prefers-color-scheme: dark) { - :root:not(.light) { + :root { --background: oklch(0.141 0.005 285.823); --foreground: oklch(0.985 0 0); --card: oklch(0.21 0.006 285.885); @@ -258,11 +217,8 @@ *, *::before, *::after { - /* biome-ignore lint/complexity/noImportantStyles: !important is intentional for accessibility - forces reduced motion regardless of specificity */ animation-duration: 0.01ms !important; - /* biome-ignore lint/complexity/noImportantStyles: !important is intentional for accessibility - forces reduced motion regardless of specificity */ animation-iteration-count: 1 !important; - /* biome-ignore lint/complexity/noImportantStyles: !important is intentional for accessibility - forces reduced motion regardless of specificity */ transition-duration: 0.01ms !important; } } From 8fd64edcc52db834441c7d60ec03e246561fafbc Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 10 Dec 2025 11:10:56 +0000 Subject: [PATCH 15/18] fix(lakebase): add missing span attr on auth retry --- packages/app-kit/src/connectors/lakebase/client.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index 617bc9a71..95d8f1296 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -117,7 +117,10 @@ export class LakebaseConnector { span.addEvent("auth_error_retry"); await this.rotateCredentials(); const newPool = await this.getPool(); - return await newPool.query(sql, params); + const result = await newPool.query(sql, params); + span.setAttribute("db.rows_affected", result.rowCount ?? 0); + span.setStatus({ code: SpanStatusCode.OK }); + return result; } // retry on transient errors, but only once From 25b120365a1ef8a74df12dcdb6a00721aab7ebb3 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 10 Dec 2025 12:16:45 +0000 Subject: [PATCH 16/18] refactor(lakebase): use token TTL from API response instead of config --- .../app-kit/src/connectors/lakebase/client.ts | 38 +++++++++++-------- .../src/connectors/lakebase/defaults.ts | 1 - .../app-kit/src/connectors/lakebase/types.ts | 3 -- .../src/connectors/tests/lakebase.test.ts | 37 ++++++++++-------- 4 files changed, 44 insertions(+), 35 deletions(-) diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index 95d8f1296..912ae22d0 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -74,9 +74,6 @@ export class LakebaseConnector { if (this.config.maxPoolSize < 1) { throw new Error("maxPoolSize must be at least 1"); } - if (this.config.credentialTTLMs < 60_000) { - throw new Error("credentialTTLMs must be at least 60 seconds"); - } } /** @@ -337,15 +334,15 @@ export class LakebaseConnector { // fetch new credentials const username = await this.fetchUsername(); - const password = await this.fetchPassword(); + const { token, expiresAt } = await this.fetchPassword(); this.credentials = { username, - password, - expiresAt: now + this.config.credentialTTLMs, + password: token, + expiresAt, }; - return { username, password }; + return { username, password: token }; } /** Rotate credentials and recreate pool */ @@ -376,7 +373,7 @@ export class LakebaseConnector { } /** Fetch password (OAuth token) from Databricks */ - private async fetchPassword(): Promise { + private async fetchPassword(): Promise<{ token: string; expiresAt: number }> { const workspaceClient = this.getWorkspaceClient(); const config = new Config({ host: workspaceClient.config.host }); const apiClient = new ApiClient(config); @@ -396,13 +393,15 @@ export class LakebaseConnector { }, }); - if (!this.hasToken(credentials)) { + if (!this.validateCredentials(credentials)) { throw new Error( `Failed to generate credentials for instance: ${this.connectionConfig.appName}`, ); } - return credentials.token; + const expiresAt = new Date(credentials.expiration_time).getTime(); + + return { token: credentials.token, expiresAt }; } /** Check if error is auth failure */ @@ -415,6 +414,7 @@ export class LakebaseConnector { ); } + /** Check if error is transient */ private isTransientError(error: unknown): boolean { if (typeof error !== "object" || error === null || !("code" in error)) { return false; @@ -434,12 +434,20 @@ export class LakebaseConnector { } /** Type guard for credentials */ - private hasToken(value: unknown): value is { token: string } { + private validateCredentials( + value: unknown, + ): value is { token: string; expiration_time: string } { + if (typeof value !== "object" || value === null) { + return false; + } + + const credentials = value as { token: string; expiration_time: string }; return ( - typeof value === "object" && - value !== null && - "token" in value && - typeof (value as any).token === "string" + "token" in credentials && + typeof credentials.token === "string" && + "expiration_time" in credentials && + typeof credentials.expiration_time === "string" && + new Date(credentials.expiration_time).getTime() > Date.now() ); } diff --git a/packages/app-kit/src/connectors/lakebase/defaults.ts b/packages/app-kit/src/connectors/lakebase/defaults.ts index 67d3688f2..c02533475 100644 --- a/packages/app-kit/src/connectors/lakebase/defaults.ts +++ b/packages/app-kit/src/connectors/lakebase/defaults.ts @@ -7,5 +7,4 @@ export const lakebaseDefaults: LakebaseConfig = { maxPoolSize: 10, idleTimeoutMs: 30_000, connectionTimeoutMs: 10_000, - credentialTTLMs: 25 * 60 * 1000, // 25 minutes }; diff --git a/packages/app-kit/src/connectors/lakebase/types.ts b/packages/app-kit/src/connectors/lakebase/types.ts index b65f88201..a8591b505 100644 --- a/packages/app-kit/src/connectors/lakebase/types.ts +++ b/packages/app-kit/src/connectors/lakebase/types.ts @@ -33,9 +33,6 @@ export interface LakebaseConfig { /** Connection timeout (milliseconds) */ connectionTimeoutMs: number; - /** How long credentials are valid (milliseconds) */ - credentialTTLMs: number; - /** Telemetry configuration */ telemetry?: TelemetryOptions; diff --git a/packages/app-kit/src/connectors/tests/lakebase.test.ts b/packages/app-kit/src/connectors/tests/lakebase.test.ts index 059169ea1..22322ac5a 100644 --- a/packages/app-kit/src/connectors/tests/lakebase.test.ts +++ b/packages/app-kit/src/connectors/tests/lakebase.test.ts @@ -75,16 +75,6 @@ describe("LakebaseConnector", () => { ).toThrow("maxPoolSize must be at least 1"); }); - test("should throw error when credentialTTLMs is less than 60 seconds", () => { - expect( - () => - new LakebaseConnector({ - credentialTTLMs: 30_000, - workspaceClient: {} as any, - }), - ).toThrow("credentialTTLMs must be at least 60 seconds"); - }); - test("should create connector with valid config", () => { const connector = new LakebaseConnector({ workspaceClient: {} as any, @@ -148,7 +138,10 @@ describe("LakebaseConnector", () => { // Setup default mocks mockMe.mockResolvedValue({ userName: "test-user@example.com" }); - mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockRequest.mockResolvedValue({ + token: "test-oauth-token", + expiration_time: new Date(Date.now() + 3600000).toISOString(), // 1 hour from now + }); mockQuery.mockResolvedValue({ rows: [{ result: 1 }] }); connector = new LakebaseConnector({ @@ -247,7 +240,10 @@ describe("LakebaseConnector", () => { mockRequest = (sdk as any).__mockRequest; mockMe.mockResolvedValue({ userName: "test-user@example.com" }); - mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockRequest.mockResolvedValue({ + token: "test-oauth-token", + expiration_time: new Date(Date.now() + 3600000).toISOString(), + }); const mockClient = { query: vi.fn().mockResolvedValue({ rows: [] }), @@ -305,7 +301,10 @@ describe("LakebaseConnector", () => { mockRequest = (sdk as any).__mockRequest; mockMe.mockResolvedValue({ userName: "test-user@example.com" }); - mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockRequest.mockResolvedValue({ + token: "test-oauth-token", + expiration_time: new Date(Date.now() + 3600000).toISOString(), + }); connector = new LakebaseConnector({ workspaceClient: { @@ -357,7 +356,10 @@ describe("LakebaseConnector", () => { mockRequest = (sdk as any).__mockRequest; mockMe.mockResolvedValue({ userName: "test-user@example.com" }); - mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockRequest.mockResolvedValue({ + token: "test-oauth-token", + expiration_time: new Date(Date.now() + 3600000).toISOString(), + }); mockQuery.mockResolvedValue({ rows: [{ result: 1 }] }); mockEnd.mockResolvedValue(undefined); @@ -418,7 +420,7 @@ describe("LakebaseConnector", () => { test("should throw when token cannot be fetched", async () => { mockMe.mockResolvedValue({ userName: "test-user@example.com" }); - mockRequest.mockResolvedValue({ error: "unauthorized" }); + mockRequest.mockResolvedValue({ error: "unauthorized" }); // missing token and expiration_time const connector = new LakebaseConnector({ workspaceClient: { @@ -448,7 +450,10 @@ describe("LakebaseConnector", () => { mockRequest = (sdk as any).__mockRequest; mockMe.mockResolvedValue({ userName: "test-user@example.com" }); - mockRequest.mockResolvedValue({ token: "test-oauth-token" }); + mockRequest.mockResolvedValue({ + token: "test-oauth-token", + expiration_time: new Date(Date.now() + 3600000).toISOString(), + }); connector = new LakebaseConnector({ workspaceClient: { From 706e766a163dbe16ec62b1c83af85b7a6147228d Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 10 Dec 2025 12:27:09 +0000 Subject: [PATCH 17/18] refactor(lakebase): add automatic BEGIN/COMMIT/ROLLBACK to transaction method --- .../app-kit/src/connectors/lakebase/client.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/app-kit/src/connectors/lakebase/client.ts b/packages/app-kit/src/connectors/lakebase/client.ts index 912ae22d0..34e57c261 100644 --- a/packages/app-kit/src/connectors/lakebase/client.ts +++ b/packages/app-kit/src/connectors/lakebase/client.ts @@ -144,13 +144,14 @@ export class LakebaseConnector { /** * Execute a transaction * + * COMMIT and ROLLBACK are automatically managed by the transaction function. + * + * @param callback - Callback function to execute within the transaction context * @example * ```typescript * await connector.transaction(async (client) => { - * await client.query('BEGIN'); * await client.query('INSERT INTO accounts (name) VALUES ($1)', ['Alice']); * await client.query('INSERT INTO logs (action) VALUES ($1)', ['Created Alice']); - * await client.query('COMMIT'); * }); * ``` */ @@ -171,10 +172,15 @@ export class LakebaseConnector { const pool = await this.getPool(); const client = await pool.connect(); try { + await client.query("BEGIN"); const result = await callback(client); + await client.query("COMMIT"); span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (error) { + try { + await client.query("ROLLBACK"); + } catch {} // retry on auth failure if (this.isAuthError(error)) { span.addEvent("auth_error_retry"); @@ -183,7 +189,16 @@ export class LakebaseConnector { const newPool = await this.getPool(); const retryClient = await newPool.connect(); try { - return await callback(retryClient); + await client.query("BEGIN"); + const result = await callback(retryClient); + await client.query("COMMIT"); + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } catch (retryError) { + try { + await retryClient.query("ROLLBACK"); + } catch {} + throw retryError; } finally { retryClient.release(); } From 4ca6d8571c8eb4910ac5d9282ebf2de494ad8cb4 Mon Sep 17 00:00:00 2001 From: Ditadi Date: Wed, 10 Dec 2025 14:25:52 +0000 Subject: [PATCH 18/18] refactor(lakebase): add probabilistic eviction check to reduce set query load --- .../app-kit/src/cache/storage/defaults.ts | 2 ++ .../app-kit/src/cache/storage/persistent.ts | 13 ++++++++--- .../src/cache/tests/persistent.test.ts | 23 +++++++++++++------ packages/shared/src/cache.ts | 3 +++ 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/app-kit/src/cache/storage/defaults.ts b/packages/app-kit/src/cache/storage/defaults.ts index ffc72c4d6..2db682a84 100644 --- a/packages/app-kit/src/cache/storage/defaults.ts +++ b/packages/app-kit/src/cache/storage/defaults.ts @@ -16,4 +16,6 @@ export const lakebaseStorageDefaults = { maxSize: 1000, /** Number of entries to evict when cache is full */ evictionBatchSize: 100, + /** Probability (0-1) of checking total bytes on each write operation */ + evictionCheckProbability: 0.1, }; diff --git a/packages/app-kit/src/cache/storage/persistent.ts b/packages/app-kit/src/cache/storage/persistent.ts index 2e38a8eec..9309a8bad 100644 --- a/packages/app-kit/src/cache/storage/persistent.ts +++ b/packages/app-kit/src/cache/storage/persistent.ts @@ -23,6 +23,7 @@ export class PersistentStorage implements CacheStorage { private readonly maxBytes: number; private readonly maxEntryBytes: number; private readonly evictionBatchSize: number; + private readonly evictionCheckProbability: number; private initialized: boolean; constructor(config: CacheConfig, connector: LakebaseConnector) { @@ -31,6 +32,9 @@ export class PersistentStorage implements CacheStorage { this.maxEntryBytes = config.maxEntryBytes ?? lakebaseStorageDefaults.maxEntryBytes; this.evictionBatchSize = lakebaseStorageDefaults.evictionBatchSize; + this.evictionCheckProbability = + config.evictionCheckProbability ?? + lakebaseStorageDefaults.evictionCheckProbability; this.tableName = lakebaseStorageDefaults.tableName; // hardcoded, safe for now this.initialized = false; } @@ -105,9 +109,12 @@ export class PersistentStorage implements CacheStorage { ); } - const totalBytes = await this.totalBytes(); - if (totalBytes + byteSize > this.maxBytes) { - await this.evictBySize(byteSize); + // probabilistic eviction check + if (Math.random() < this.evictionCheckProbability) { + const totalBytes = await this.totalBytes(); + if (totalBytes + byteSize > this.maxBytes) { + await this.evictBySize(byteSize); + } } await this.connector.query( diff --git a/packages/app-kit/src/cache/tests/persistent.test.ts b/packages/app-kit/src/cache/tests/persistent.test.ts index 7bdc105d1..2623e25b4 100644 --- a/packages/app-kit/src/cache/tests/persistent.test.ts +++ b/packages/app-kit/src/cache/tests/persistent.test.ts @@ -132,10 +132,9 @@ describe("PersistentStorage", () => { }); test("should insert new entry", async () => { - // totalBytes() returns 0 - mockConnector.query.mockResolvedValueOnce({ - rows: [{ total: "0" }], - }); + // Mock Math.random to skip eviction check (>= evictionCheckProbability) + const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.5); + // INSERT succeeds mockConnector.query.mockResolvedValueOnce({ rows: [] }); @@ -154,9 +153,14 @@ describe("PersistentStorage", () => { expect.any(Number), // expiry ]), ); + + randomSpy.mockRestore(); }); test("should evict when maxBytes exceeded", async () => { + // Mock Math.random to ensure eviction check runs (< evictionCheckProbability) + const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.05); + // totalBytes() returns maxBytes (triggers eviction) mockConnector.query.mockResolvedValueOnce({ rows: [{ total: String(1024 * 1024) }], // 1MB (at limit) @@ -180,12 +184,15 @@ describe("PersistentStorage", () => { expect.stringContaining("DELETE FROM"), expect.any(Array), ); + + randomSpy.mockRestore(); }); test("should serialize value to Buffer", async () => { - mockConnector.query.mockResolvedValueOnce({ - rows: [{ total: "0" }], - }); + // Mock Math.random to skip eviction check (>= evictionCheckProbability) + const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.5); + + // INSERT succeeds mockConnector.query.mockResolvedValueOnce({ rows: [] }); const value = { nested: { array: [1, 2, 3] } }; @@ -202,6 +209,8 @@ describe("PersistentStorage", () => { const valueBuffer = insertCall?.[1]?.[2] as Buffer; expect(valueBuffer).toBeInstanceOf(Buffer); expect(valueBuffer.toString("utf-8")).toBe(JSON.stringify(value)); + + randomSpy.mockRestore(); }); }); diff --git a/packages/shared/src/cache.ts b/packages/shared/src/cache.ts index 6bc7be000..485b568bd 100644 --- a/packages/shared/src/cache.ts +++ b/packages/shared/src/cache.ts @@ -54,6 +54,9 @@ export interface CacheConfig { /** Probability (0-1) of triggering cleanup on each get operation */ cleanupProbability?: number; + /** Probability (0-1) of checking total bytes on each write operation */ + evictionCheckProbability?: number; + /** Maximum number of bytes per entry in the cache */ maxEntryBytes?: number;