diff --git a/.changeset/odd-geckos-worry.md b/.changeset/odd-geckos-worry.md new file mode 100644 index 0000000000..559389fa57 --- /dev/null +++ b/.changeset/odd-geckos-worry.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': patch +--- + +Add experimental sql obfuscator diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 91c0b9ddb2..0537242361 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -75,6 +75,7 @@ services: context: ./docker/otel-collector target: dev environment: + HYPERDX_API_KEY: ${HYPERDX_API_KEY} HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml @@ -99,12 +100,25 @@ services: ports: - 8001:8001 environment: + CLICKHOUSE_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} + EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋' + HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1 + HDX_NODE_BETA_MODE: 1 + HDX_NODE_CONSOLE_CAPTURE: 1 + SENTRY_DSN: ${SENTRY_DSN} + INGESTOR_API_URL: 'http://ingestor:8002' + MINER_API_URL: 'http://miner:5123' + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + OTEL_SERVICE_NAME: 'hdx-oss-dev-aggregator' APP_TYPE: 'aggregator' CLICKHOUSE_HOST: http://ch-server:8123 CLICKHOUSE_PASSWORD: aggregator CLICKHOUSE_USER: aggregator FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS) HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} + # uncomment to allow aggregator logs + # disabled by default due to possible infinite loop (aggregator aggregating aggregator logs) + # HYPERDX_API_KEY: ${HYPERDX_API_KEY} MONGO_URI: 'mongodb://db:27017/hyperdx' NODE_ENV: development PORT: 8001 diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 9c455275fe..0518983177 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -26,3 +26,4 @@ export const PORT = Number.parseInt(env.PORT as string); export const REDIS_URL = env.REDIS_URL as string; export const SERVER_URL = env.SERVER_URL as string; export const USAGE_STATS_ENABLED = env.USAGE_STATS_ENABLED !== 'false'; +export const OBFUSCATE_SQL = env.OBFUSCATE_SQL !== 'false'; diff --git a/packages/api/src/gobin/LICENSE b/packages/api/src/gobin/LICENSE new file mode 100644 index 0000000000..fec91553f1 --- /dev/null +++ b/packages/api/src/gobin/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Datadog, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/packages/api/src/gobin/sql_obfuscator_arm64 b/packages/api/src/gobin/sql_obfuscator_arm64 new file mode 100755 index 0000000000..57f6add691 Binary files /dev/null and b/packages/api/src/gobin/sql_obfuscator_arm64 differ diff --git a/packages/api/src/gobin/sql_obfuscator_x64 b/packages/api/src/gobin/sql_obfuscator_x64 new file mode 100755 index 0000000000..73f5c67038 Binary files /dev/null and b/packages/api/src/gobin/sql_obfuscator_x64 differ diff --git a/packages/api/src/gobin/src/go.mod b/packages/api/src/gobin/src/go.mod new file mode 100644 index 0000000000..e6bd64d95f --- /dev/null +++ b/packages/api/src/gobin/src/go.mod @@ -0,0 +1,5 @@ +module sqlObfuscator + +go 1.21.5 + +require github.com/DataDog/go-sqllexer v0.0.10 diff --git a/packages/api/src/gobin/src/go.sum b/packages/api/src/gobin/src/go.sum new file mode 100644 index 0000000000..74ba2dd78d --- /dev/null +++ b/packages/api/src/gobin/src/go.sum @@ -0,0 +1,10 @@ +github.com/DataDog/go-sqllexer v0.0.10 h1:u07DuRfdlPPmOX/dclb1gcn/zaqWxUiURRRVenKILxc= +github.com/DataDog/go-sqllexer v0.0.10/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/api/src/gobin/src/sql_obfuscator.go b/packages/api/src/gobin/src/sql_obfuscator.go new file mode 100644 index 0000000000..4d3e263455 --- /dev/null +++ b/packages/api/src/gobin/src/sql_obfuscator.go @@ -0,0 +1,50 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/DataDog/go-sqllexer" +) + +// This is a simple command line tool that reads multiple newline-separated SQL queries from stdin +// and normalizes and obfuscates them, then prints them one at a time (newline separated) to stdout. +// Example: +// $ echo "SELECT * FROM foo as foo_table limit 1; SELECT * FROM /* sql comment */ bar where name = 'bob';" | go run sql_obfuscator.go +// SELECT * FROM foo limit ?; SELECT * FROM bar where name = ? +func main() { + reader := bufio.NewReader(os.Stdin) + for { + query, err := reader.ReadString('\n') + if err != nil { + // write to stderr + fmt.Fprintf(os.Stderr, "error: %s\n", err) + os.Exit(1) + } + stripped_string := strings.Trim(query, " \t\r\n") + if stripped_string == "" { + // skip empty entries + continue + } + + normalizer := sqllexer.NewNormalizer( + sqllexer.WithCollectComments(false), + sqllexer.WithCollectCommands(false), + sqllexer.WithCollectTables(false), + sqllexer.WithKeepSQLAlias(false), + ) + + normalized, _, err := normalizer.Normalize(query) + if err != nil { + // write to stderr + fmt.Fprintf(os.Stderr, "error: %s\n", err) + os.Exit(1) + } + + obfuscator := sqllexer.NewObfuscator() + obfuscated := obfuscator.Obfuscate(normalized) + fmt.Printf("%s\n", obfuscated) + } +} diff --git a/packages/api/src/routers/aggregator/root.ts b/packages/api/src/routers/aggregator/root.ts index 3f41f6712e..f4dfac6c0f 100644 --- a/packages/api/src/routers/aggregator/root.ts +++ b/packages/api/src/routers/aggregator/root.ts @@ -29,7 +29,7 @@ const bulkInsert = async ( switch (hdxTelemetry) { case 'metric': await bulkInsertTeamMetricStream( - vectorMetricParser.parse(data as VectorMetric[]), + await vectorMetricParser.parse(data as VectorMetric[]), ); break; default: { @@ -46,12 +46,12 @@ const bulkInsert = async ( bulkInsertTeamLogStream( team.logStreamTableVersion, team._id.toString(), - vectorLogParser.parse(logs), + await vectorLogParser.parse(logs), ), ]; if (rrwebEvents.length > 0) { promises.push( - bulkInsertRrwebEvents(vectorRrwebParser.parse(rrwebEvents)), + bulkInsertRrwebEvents(await vectorRrwebParser.parse(rrwebEvents)), ); } await Promise.all(promises); diff --git a/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap b/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap index 1add8294ac..6d068b0495 100644 --- a/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap +++ b/packages/api/src/utils/__tests__/__snapshots__/logParser.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`logParser mapObjectToKeyValuePairs 1`] = ` +exports[`logParser mapObjectToKeyValuePairs obeys basic serialization 1`] = ` Object { "bool.names": Array [ "foo2", @@ -31,7 +31,7 @@ Object { } `; -exports[`logParser mapObjectToKeyValuePairs 2`] = ` +exports[`logParser mapObjectToKeyValuePairs obeys basic serialization 2`] = ` Object { "bool.names": Array [], "bool.values": Array [], diff --git a/packages/api/src/utils/__tests__/logParser.test.ts b/packages/api/src/utils/__tests__/logParser.test.ts index a2153a9e7e..7762121841 100644 --- a/packages/api/src/utils/__tests__/logParser.test.ts +++ b/packages/api/src/utils/__tests__/logParser.test.ts @@ -53,69 +53,90 @@ describe('logParser', () => { ]); }); - it('mapObjectToKeyValuePairs', () => { - expect(mapObjectToKeyValuePairs(null as any)).toEqual({ - 'bool.names': [], - 'bool.values': [], - 'number.names': [], - 'number.values': [], - 'string.names': [], - 'string.values': [], - }); + describe('mapObjectToKeyValuePairs', () => { + it('obeys basic serialization', async () => { + expect(await mapObjectToKeyValuePairs(null as any)).toEqual({ + 'bool.names': [], + 'bool.values': [], + 'number.names': [], + 'number.values': [], + 'string.names': [], + 'string.values': [], + }); - expect(mapObjectToKeyValuePairs({})).toEqual({ - 'bool.names': [], - 'bool.values': [], - 'number.names': [], - 'number.values': [], - 'string.names': [], - 'string.values': [], - }); + expect(await mapObjectToKeyValuePairs({})).toEqual({ + 'bool.names': [], + 'bool.values': [], + 'number.names': [], + 'number.values': [], + 'string.names': [], + 'string.values': [], + }); - expect( - mapObjectToKeyValuePairs({ foo: '123', foo1: 123, foo2: false }), - ).toEqual({ - 'bool.names': ['foo2'], - 'bool.values': [0], - 'number.names': ['foo1'], - 'number.values': [123], - 'string.names': ['foo'], - 'string.values': ['123'], - }); + expect( + await mapObjectToKeyValuePairs({ foo: '123', foo1: 123, foo2: false }), + ).toEqual({ + 'bool.names': ['foo2'], + 'bool.values': [0], + 'number.names': ['foo1'], + 'number.values': [123], + 'string.names': ['foo'], + 'string.values': ['123'], + }); - expect( - mapObjectToKeyValuePairs({ - foo: '123', - foo1: 123, - foo2: false, - nested: { foo: 'bar' }, - good: { - burrito: { - is: true, - }, - }, - array1: [456], - array2: [ - 'foo1', - { - foo2: 'bar2', + expect( + await mapObjectToKeyValuePairs({ + foo: '123', + foo1: 123, + foo2: false, + nested: { foo: 'bar' }, + good: { + burrito: { + is: true, + }, }, - [ + array1: [456], + array2: [ + 'foo1', { - foo3: 'bar3', + foo2: 'bar2', }, + [ + { + foo3: 'bar3', + }, + ], ], - ], - }), - ).toMatchSnapshot(); + }), + ).toMatchSnapshot(); - const testObject = {}; - for (let i = 0; i < 2000; i++) { - testObject[`foo${i}`] = i; - } - const result = mapObjectToKeyValuePairs(testObject); - expect(result['number.names'].length).toEqual(1024); - expect(result['number.values'].length).toEqual(1024); - expect(result).toMatchSnapshot(); + const testObject = {}; + for (let i = 0; i < 2000; i++) { + testObject[`foo${i}`] = i; + } + const result = await mapObjectToKeyValuePairs(testObject); + expect(result['number.names'].length).toEqual(1024); + expect(result['number.values'].length).toEqual(1024); + expect(result).toMatchSnapshot(); + }); + + it('adds obfuscated sql to output', async () => { + const original = { + db: { + statement: `SELECT * from users where name = 'Carl' limit 1 order by name asc;\n`, + }, + }; + expect(await mapObjectToKeyValuePairs(original)).toEqual({ + 'bool.names': [], + 'bool.values': [], + 'number.names': [], + 'number.values': [], + 'string.names': ['db.statement', 'db.sql.normalized'], + 'string.values': [ + original.db.statement, + `SELECT * from users where name = ? limit ? order by name asc\n`, + ], + }); + }); }); }); diff --git a/packages/api/src/utils/__tests__/sqlObfuscator.test.ts b/packages/api/src/utils/__tests__/sqlObfuscator.test.ts new file mode 100644 index 0000000000..fd2c6ed149 --- /dev/null +++ b/packages/api/src/utils/__tests__/sqlObfuscator.test.ts @@ -0,0 +1,53 @@ +import * as childProcess from 'child_process'; + +import { getChild, sqlObfuscator } from '../sqlObfuscator'; + +describe('logParser', () => { + it('obfuscates a basic query', async () => { + const n = 1; + const start = Date.now(); + for (let i = 0; i < n; i++) { + expect( + await sqlObfuscator( + `SELECT * from users where name = 'Carl' limit 1 order by name asc;\n`, + ), + ).toEqual( + `SELECT * from users where name = ? limit ? order by name asc`.replace( + /(\r\n|\n|\r)/gm, + ' ', + ) + '\n', + ); + } + const end = Date.now(); + //console.log(`Took ${(end - start) / n}ms`); + }); + + it('handles multiple statements in one line', async () => { + expect( + await sqlObfuscator( + `SELECT * from users as should_be_removed where name = 'Carl' limit 1 order by name asc; SELECT * from users /* standard comment, really */ where name = 'Carl' limit 1 order by name asc;\n`, + ), + ).toEqual( + `SELECT * from users where name = ? limit ? order by name asc; SELECT * from users where name = ? limit ? order by name asc`.replace( + /(\r\n|\n|\r)/gm, + ' ', + ) + '\n', + ); + }); + + it('handles if the subprocess dies', async () => { + const subprocess = getChild(); + subprocess.kill(); + expect( + await sqlObfuscator( + `SELECT * from users where name = 'Carl' limit 1 order by name asc;\n`, + ), + ).toEqual( + `SELECT * from users where name = ? limit ? order by name asc`.replace( + /(\r\n|\n|\r)/gm, + ' ', + ) + '\n', + ); + expect(subprocess).not.toBe(getChild()); + }); +}); diff --git a/packages/api/src/utils/logParser.ts b/packages/api/src/utils/logParser.ts index 9041cfd054..2198dc1eda 100644 --- a/packages/api/src/utils/logParser.ts +++ b/packages/api/src/utils/logParser.ts @@ -1,6 +1,10 @@ import _ from 'lodash'; +import * as config from '@/config'; + import { tryJSONStringify } from './common'; +import logger from './logger'; +import { sqlObfuscator } from './sqlObfuscator'; export type JSONBlob = Record; @@ -104,10 +108,10 @@ export function* traverseJson( } const MAX_KEY_VALUE_PAIRS_LENGTH = 1024; -export const mapObjectToKeyValuePairs = ( +export const mapObjectToKeyValuePairs = async ( blob: JSONBlob, maxArrayLength = MAX_KEY_VALUE_PAIRS_LENGTH, -): KeyValuePairs => { +): Promise => { const output: KeyValuePairs = { 'bool.names': [], 'bool.values': [], @@ -170,6 +174,20 @@ export const mapObjectToKeyValuePairs = ( } } + if (config.OBFUSCATE_SQL && output['string.names'].includes('db.statement')) { + const index = output['string.names'].indexOf('db.statement'); + let obfuscated = ''; + try { + obfuscated = await sqlObfuscator(output['string.values'][index]); + } catch (e) { + logger.error(`Normalizer error: ${e}`); + } + if (obfuscated.length > 0 && obfuscated.trim().length > 0) { + output['string.names'].push('db.sql.normalized'); + output['string.values'].push(obfuscated); + } + } + return output; }; @@ -233,13 +251,13 @@ abstract class ParsingInterface { abstract _parse( log: T, ...args: any[] - ): LogStreamModel | MetricModel | RrwebEventModel; + ): Promise; - parse(logs: T[], ...args: any[]) { + async parse(logs: T[], ...args: any[]) { const parsedLogs: any[] = []; for (const log of logs) { try { - parsedLogs.push(this._parse(log, ...args)); + parsedLogs.push(await this._parse(log, ...args)); } catch (e) { // continue if parser fails to parse single log console.warn(e); @@ -259,9 +277,9 @@ class VectorLogParser extends ParsingInterface { return LogType.Log; } - _parse(log: VectorLog): LogStreamModel { + async _parse(log: VectorLog): Promise { return { - ...mapObjectToKeyValuePairs(log.b), + ...(await mapObjectToKeyValuePairs(log.b)), _platform: log.hdx_platform, _service: log.sv, _source: log.r, @@ -283,7 +301,7 @@ class VectorLogParser extends ParsingInterface { } class VectorMetricParser extends ParsingInterface { - _parse(metric: VectorMetric): MetricModel { + async _parse(metric: VectorMetric): Promise { return { _string_attributes: metric.b, data_type: metric.dt, @@ -298,9 +316,9 @@ class VectorMetricParser extends ParsingInterface { } class VectorRrwebParser extends ParsingInterface { - _parse(log: VectorLog): RrwebEventModel { + async _parse(log: VectorLog): Promise { return { - ...mapObjectToKeyValuePairs(log.b), + ...(await mapObjectToKeyValuePairs(log.b)), _service: log.sv, _source: log.r, timestamp: log.ts, diff --git a/packages/api/src/utils/sqlObfuscator.ts b/packages/api/src/utils/sqlObfuscator.ts new file mode 100644 index 0000000000..597157ec4b --- /dev/null +++ b/packages/api/src/utils/sqlObfuscator.ts @@ -0,0 +1,96 @@ +import type { ChildProcess } from 'child_process'; +import { spawn } from 'child_process'; + +let subprocess: ChildProcess | undefined; + +// exported only for testing +export const getChild = () => { + if (subprocess && subprocess?.killed === false && subprocess?.connected) { + return subprocess; + } + + const arch = process.arch; + + // can be updated if new arch is built + if (arch !== 'x64' && arch !== 'arm64') { + throw new Error(`Unsupported architecture: ${arch}`); + } + + subprocess = spawn(`src/gobin/sql_obfuscator_${arch}`, [], { + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + }); + + if (!subprocess) { + throw new Error(`Could not spawn child process`); + } + + process.on('SIGINT', () => { + if (subprocess && subprocess?.killed === false) { + subprocess.kill('SIGINT'); + } + }); + + return subprocess; +}; + +const removeListeners = (subprocess: ChildProcess | undefined) => { + if (!subprocess) { + return; + } + subprocess.removeAllListeners(); + if (subprocess.stdout) { + subprocess.stdout.removeAllListeners(); + } + if (subprocess.stderr) { + subprocess.stderr.removeAllListeners(); + } +}; + +export const sqlObfuscator = async (sql: string): Promise => { + if (!sql || sql.length === 0 || sql.trim().length === 0) { + return ''; + } + subprocess = getChild(); + if (!subprocess || subprocess.stdin === null) { + throw new Error(`Could not spawn child process`); + } + + const strippedSql = sql.replace(/(\r\n|\n|\r)/gm, ' '); + subprocess.stdin.write(`${strippedSql}\n`); + + let output = ''; + + return new Promise((resolve, reject) => { + if ( + !subprocess || + subprocess.stdin === null || + subprocess.stdout === null || + subprocess.stderr === null + ) { + throw new Error(`Could not spawn child process`); + } + const errorOutput = (data: any) => { + removeListeners(subprocess); + reject(data.toString()); + }; + const dataRecieved = (data: any) => { + const str = data.toString(); + output += str; + if (str[str.length - 1] === '\n') { + removeListeners(subprocess); + resolve(output); + } + }; + const handleExit = (code: number | null, signal: string | null) => { + if (code !== 0) { + errorOutput(`Child process exited with code ${code}`); + } else { + errorOutput(`Child process exited with signal ${signal}`); + } + }; + subprocess.stdout.on('data', dataRecieved); + subprocess.stderr.on('data', errorOutput); + subprocess.on('error', errorOutput); + subprocess.on('exit', handleExit); + }); +};