From b2e525331ac8bd4c270617a61c509809251fec2f Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 20:38:53 -0800 Subject: [PATCH 1/9] refactor: move app type to server class as member --- packages/api/src/server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 7b460400f2..e694376d49 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -8,10 +8,12 @@ import logger from './utils/logger'; import redisClient from './utils/redis'; export default class Server { + protected readonly appType = config.APP_TYPE; + protected httpServer!: http.Server; private async createServer() { - switch (config.APP_TYPE) { + switch (this.appType) { case 'api': return http.createServer( // eslint-disable-next-line n/no-unsupported-features/es-syntax From 90e9ffe1bbe500d57493d1aa997624b9d36d07f0 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 20:41:51 -0800 Subject: [PATCH 2/9] ci: add aggregator service: --- docker-compose.ci.yml | 27 +++++++++++++++++++++++++++ packages/api/src/fixtures.ts | 19 ++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 64363956b3..29c61b02c4 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -30,6 +30,32 @@ services: - 6379:6379 networks: - internal + aggregator: + container_name: hdx-ci-aggregator + build: + context: . + dockerfile: ./packages/api/Dockerfile + target: dev + ports: + - 8001:8001 + environment: + APP_TYPE: 'aggregator' + CLICKHOUSE_HOST: http://ch_server:8123 + CLICKHOUSE_PASSWORD: aggregator + CLICKHOUSE_USER: aggregator + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} + MONGO_URI: 'mongodb://db:29999/hyperdx-test' + NODE_ENV: ci + PORT: 8001 + REDIS_URL: redis://redis:6379 + volumes: + - ./packages/api/src:/app/src + networks: + - internal + depends_on: + - ch_server + - db + - redis api: build: context: . @@ -56,6 +82,7 @@ services: networks: - internal depends_on: + - aggregator - ch_server - db - redis diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index fd74401042..1dee11a891 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -86,7 +86,24 @@ class MockServer extends Server { } } -export const getServer = () => new MockServer(); +class MockAPIServer extends MockServer { + protected readonly appType = 'api'; +} + +class MockAggregatorServer extends MockServer { + protected readonly appType = 'aggregator'; +} + +export const getServer = (appType: 'api' | 'aggregator' = 'api') => { + switch (appType) { + case 'api': + return new MockAPIServer(); + case 'aggregator': + return new MockAggregatorServer(); + default: + throw new Error(`Invalid APP_TYPE: ${appType}`); + } +}; export const getAgent = (server: MockServer) => request.agent(server.getHttpServer()); From eb8d235b85324749368dde8e88e7c299a711a27a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 21:39:09 -0800 Subject: [PATCH 3/9] test: ingest logs --- .../routers/aggregator/__tests__/root.test.ts | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 packages/api/src/routers/aggregator/__tests__/root.test.ts diff --git a/packages/api/src/routers/aggregator/__tests__/root.test.ts b/packages/api/src/routers/aggregator/__tests__/root.test.ts new file mode 100644 index 0000000000..0912cf023c --- /dev/null +++ b/packages/api/src/routers/aggregator/__tests__/root.test.ts @@ -0,0 +1,79 @@ +import _ from 'lodash'; + +import { + clearClickhouseTables, + clearDBCollections, + closeDB, + getAgent, + getServer, +} from '@/fixtures'; +import * as clickhouse from '@/clickhouse'; +import { createTeam } from '@/controllers/team'; + +describe('aggregator root router', () => { + const server = getServer('aggregator'); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await clearDBCollections(); + await clearClickhouseTables(); + }); + + afterAll(async () => { + await server.closeHttpServer(); + await closeDB(); + }); + + it('GET /health', async () => { + const agent = await getAgent(server); + await agent.get('/health').expect(200); + }); + + it('POST / -> should return 400 if no logs', async () => { + const agent = await getAgent(server); + await agent.post('/').send({}).expect(400); + }); + + it('POST / -> should aggregate logs', async () => { + const team = await createTeam({ name: 'test-team' }); + const agent = await getAgent(server); + await agent.post('/').send([ + { + b: { + _hdx_body: 'Initializing ClickHouse...', + level: 'info', + message: 'Initializing ClickHouse...', + }, + h: '509a8b2dea19', + hdx_platform: 'nodejs', + hdx_token: team.apiKey, + hdx_token_hash: '2f4da895de6a20c100c28daaa5c07c51', + path: '/', + r: { level: 'info', message: 'Initializing ClickHouse...' }, + s_id: null, + sn: 0, + st: 'info', + sv: 'hdx-oss-dev-api', + t_id: null, + ts: 1704517334214000000, + tso: 1704517336156579600, + }, + ]); + + // wait for data to be committed to clickhouse + await new Promise(resolve => setTimeout(resolve, 500)); + + const resp = await clickhouse.client.query({ + query: `SELECT * FROM default.${clickhouse.TableName.LogStream}`, + format: 'JSON', + }); + const result: any = await resp.json(); + expect(result.data.length).toBe(1); + expect(result.data[0]._service).toBe('hdx-oss-dev-api'); + expect(result.data[0]._platform).toBe('nodejs'); + expect(result.data[0].severity_text).toBe('info'); + }); +}); From b61ce86f317f602572ed7ae9af0bed1c639be92a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 21:42:14 -0800 Subject: [PATCH 4/9] docs: comment --- packages/api/src/routers/aggregator/__tests__/root.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/aggregator/__tests__/root.test.ts b/packages/api/src/routers/aggregator/__tests__/root.test.ts index 0912cf023c..4c739e6088 100644 --- a/packages/api/src/routers/aggregator/__tests__/root.test.ts +++ b/packages/api/src/routers/aggregator/__tests__/root.test.ts @@ -37,7 +37,7 @@ describe('aggregator root router', () => { await agent.post('/').send({}).expect(400); }); - it('POST / -> should aggregate logs', async () => { + it('POST / -> should ingest logs', async () => { const team = await createTeam({ name: 'test-team' }); const agent = await getAgent(server); await agent.post('/').send([ @@ -76,4 +76,6 @@ describe('aggregator root router', () => { expect(result.data[0]._platform).toBe('nodejs'); expect(result.data[0].severity_text).toBe('info'); }); + + // TODO: test metrics }); From 2a24db04870b09a7c7a2ca742a0deaa445192e80 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 21:44:11 -0800 Subject: [PATCH 5/9] style: error msg --- packages/api/src/fixtures.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 1dee11a891..fb38b1136e 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -101,7 +101,7 @@ export const getServer = (appType: 'api' | 'aggregator' = 'api') => { case 'aggregator': return new MockAggregatorServer(); default: - throw new Error(`Invalid APP_TYPE: ${appType}`); + throw new Error(`Invalid app type: ${appType}`); } }; From 84a807bd116ca940ba5ee3138b1ad7f0fd99ac40 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 21:46:05 -0800 Subject: [PATCH 6/9] style: use sleep method --- packages/api/src/routers/aggregator/__tests__/root.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/api/src/routers/aggregator/__tests__/root.test.ts b/packages/api/src/routers/aggregator/__tests__/root.test.ts index 4c739e6088..5c2582af79 100644 --- a/packages/api/src/routers/aggregator/__tests__/root.test.ts +++ b/packages/api/src/routers/aggregator/__tests__/root.test.ts @@ -1,5 +1,7 @@ import _ from 'lodash'; +import * as clickhouse from '@/clickhouse'; +import { createTeam } from '@/controllers/team'; import { clearClickhouseTables, clearDBCollections, @@ -7,8 +9,7 @@ import { getAgent, getServer, } from '@/fixtures'; -import * as clickhouse from '@/clickhouse'; -import { createTeam } from '@/controllers/team'; +import { sleep } from '@/utils/common'; describe('aggregator root router', () => { const server = getServer('aggregator'); @@ -64,7 +65,7 @@ describe('aggregator root router', () => { ]); // wait for data to be committed to clickhouse - await new Promise(resolve => setTimeout(resolve, 500)); + await sleep(500); const resp = await clickhouse.client.query({ query: `SELECT * FROM default.${clickhouse.TableName.LogStream}`, From e96e875dfc08c303f23938872fc5a14c0d449b8b Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 21:47:11 -0800 Subject: [PATCH 7/9] docs: add changeset --- .changeset/mean-balloons-shake.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/mean-balloons-shake.md diff --git a/.changeset/mean-balloons-shake.md b/.changeset/mean-balloons-shake.md new file mode 100644 index 0000000000..58b4d85dc1 --- /dev/null +++ b/.changeset/mean-balloons-shake.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +ci: setup aggregator int tests From 32476648d21b4aca9652396265667c6bb3e9f351 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 22:03:36 -0800 Subject: [PATCH 8/9] style: test with snapshot --- .../routers/aggregator/__tests__/root.test.ts | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/packages/api/src/routers/aggregator/__tests__/root.test.ts b/packages/api/src/routers/aggregator/__tests__/root.test.ts index 5c2582af79..21a0a3f195 100644 --- a/packages/api/src/routers/aggregator/__tests__/root.test.ts +++ b/packages/api/src/routers/aggregator/__tests__/root.test.ts @@ -73,9 +73,42 @@ describe('aggregator root router', () => { }); const result: any = await resp.json(); expect(result.data.length).toBe(1); - expect(result.data[0]._service).toBe('hdx-oss-dev-api'); - expect(result.data[0]._platform).toBe('nodejs'); - expect(result.data[0].severity_text).toBe('info'); + expect(result.data.map((row: any) => _.omit(row, ['id', '_created_at']))) + .toMatchInlineSnapshot(` +Array [ + Object { + "_host": "509a8b2dea19", + "_namespace": "", + "_platform": "nodejs", + "_service": "hdx-oss-dev-api", + "_source": "{\\"level\\":\\"info\\",\\"message\\":\\"Initializing ClickHouse...\\"}", + "bool.names": Array [], + "bool.values": Array [], + "end_timestamp": "1970-01-01T00:00:00.000000000Z", + "number.names": Array [], + "number.values": Array [], + "observed_timestamp": "2024-01-06T05:02:16.156579600Z", + "parent_span_id": "", + "severity_number": 0, + "severity_text": "info", + "span_id": "", + "span_name": "", + "string.names": Array [ + "_hdx_body", + "level", + "message", + ], + "string.values": Array [ + "Initializing ClickHouse...", + "info", + "Initializing ClickHouse...", + ], + "timestamp": "2024-01-06T05:02:14.214000000Z", + "trace_id": "", + "type": "log", + }, +] +`); }); // TODO: test metrics From 4bccaa549ede8a02cf6fe7e32f8cf551bbed5755 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 22:21:33 -0800 Subject: [PATCH 9/9] style: remove service --- docker-compose.ci.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 29c61b02c4..64363956b3 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -30,32 +30,6 @@ services: - 6379:6379 networks: - internal - aggregator: - container_name: hdx-ci-aggregator - build: - context: . - dockerfile: ./packages/api/Dockerfile - target: dev - ports: - - 8001:8001 - environment: - APP_TYPE: 'aggregator' - CLICKHOUSE_HOST: http://ch_server:8123 - CLICKHOUSE_PASSWORD: aggregator - CLICKHOUSE_USER: aggregator - HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} - MONGO_URI: 'mongodb://db:29999/hyperdx-test' - NODE_ENV: ci - PORT: 8001 - REDIS_URL: redis://redis:6379 - volumes: - - ./packages/api/src:/app/src - networks: - - internal - depends_on: - - ch_server - - db - - redis api: build: context: . @@ -82,7 +56,6 @@ services: networks: - internal depends_on: - - aggregator - ch_server - db - redis