feat: added prefix option to redis

.github/workflows/pull_request.yaml

@@ -77,6 +77,16 @@ jobs:
  --health-timeout 5s
  --health-retries 5
 
+ redis:
+ image: redis
+ ports:
+ - 6379:6379
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+
  steps:
  - name: Check out Git repository
  uses: actions/checkout@v4

README.md

@@ -41,8 +41,9 @@ Almost all of our routes consists of the same common query parameters: `geo_cont
 | disable_fuzziness | Boolean | false | Fuzziness is on by default. If you want exact match, you may set `disable_fuzziness: true`. |
 > [!IMPORTANT]
 > We also have Feedback API. Each request is sent back with x-req-id which is the identifier of the request. We kindly ask our users to provide us with a request to Feedback API which contains x-api-key and clicked response. It enables us to research the request and response to be more accurate.
-> Feedback API source code is built in a different repository at <TODO: ADD LINK TO FEEDBACK API REPO>.
+> Feedback API source code is built in a different repository at [feedback-api](https://github.com/MapColonies/feedback-api).
 > Geocoding API inserts the request and response to Redis before the response is sent.
+Speaking of Redis, in the redis config you can add a `prefix` flag, if you'd like to add keys with prefixes to redis.
 
 ## Installation
 Setup Elasticsearch and S3 provider (For local environment, Minio as a personal recommendation).

config/test.json

@@ -35,7 +35,7 @@
  "fileName": "table.json"
  },
  "redis": {
- "host": "localhost",
+ "host": "redis",
  "port": 6379,
  "username": "",
  "password": "",
@@ -45,7 +45,8 @@
  "key": "",
  "cert": ""
  },
- "dbIndex": 1
+ "dbIndex": 1,
+ "ttl": 300
  }
  },
  "application": {

package.json

@@ -41,7 +41,7 @@
  "@map-colonies/js-logger": "^2.0.0",
  "@map-colonies/openapi-express-viewer": "^4.0.0",
  "@map-colonies/read-pkg": "^1.0.0",
- "@map-colonies/schemas": "^1.12.2",
+ "@map-colonies/schemas": "^1.14.0",
  "@map-colonies/telemetry": "^10.0.1",
  "@opentelemetry/api": "^1.9.0",
  "@smithy/node-http-handler": "^3.1.4",

src/common/middlewares/feedbackApi.middleware.ts

@@ -21,6 +21,7 @@ export class FeedbackApiMiddlewareManager {
  const reqId = res.getHeader(XApi.REQUEST);
  const redisClient = this.redis;
  const logger = this.logger;
+ const prefix = this.config.get(redisConfigPath).prefix;
 
  const drSite = this.config.get(siteConfig);
 
@@ -36,10 +37,11 @@ export class FeedbackApiMiddlewareManager {
  const { ttl: redisTtl } = this.config.get(redisConfigPath);
 
  const originalJson = res.json;
+ const fullRequestId = prefix !== undefined ? `${prefix}:${reqId as string}` : (reqId as string);
  const logJson = function (this: Response, body: JSON): Response {
  geocodingResponseDetails.response = body;
  redisClient
- .setEx(reqId as string, redisTtl ?? defaultRedisTtl, JSON.stringify(geocodingResponseDetails))
+ .setEx(fullRequestId, redisTtl ?? defaultRedisTtl, JSON.stringify(geocodingResponseDetails))
  .then(() => {
  logger.info({ msg: `response ${reqId?.toString() ?? ''} saved to redis` });
  })

tests/integration/control/item/helpers/index.ts

@@ -0,0 +1,20 @@
+import jsLogger from '@map-colonies/js-logger';
+import { trace } from '@opentelemetry/api';
+import { SERVICES } from '@src/common/constants';
+import { S3_REPOSITORY_SYMBOL } from '@src/common/s3/s3Repository';
+import { cronLoadTileLatLonDataSymbol } from '@src/latLon/DAL/latLonDAL';
+import { GetBaseRegisterOptions } from '@tests/integration/helpers/types';
+
+export const getBaseRegisterOptions: GetBaseRegisterOptions = (options = []) => {
+ return {
+ override: [
+ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
+ { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
+ { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
+ { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
+ { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
+ ...options,
+ ],
+ useChild: true,
+ };
+};

tests/integration/control/item/item.spec.ts

@@ -1,35 +1,26 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import jsLogger from '@map-colonies/js-logger';
-import { trace } from '@opentelemetry/api';
 import { CleanupRegistry } from '@map-colonies/cleanup-registry';
 import { DependencyContainer } from 'tsyringe';
 import httpStatusCodes from 'http-status-codes';
+import { IConfig } from 'config';
+import { RedisClient } from '@src/common/redis';
 import { getApp } from '../../../../src/app';
-import { SERVICES } from '../../../../src/common/constants';
+import { redisConfigPath, SERVICES } from '../../../../src/common/constants';
 import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController';
 import { Item } from '../../../../src/control/item/models/item';
 import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces';
-import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL';
-import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository';
 import { expectedResponse } from '../utils';
 import { ITEM_1234, ITEM_1235, ITEM_1236 } from '../../../mockObjects/items';
 import { ItemRequestSender } from './helpers/requestSender';
+import { getBaseRegisterOptions } from './helpers';
 
 describe('/search/control/items', function () {
  let requestSender: ItemRequestSender;
  let depContainer: DependencyContainer;
 
  beforeEach(async function () {
- const [app, container] = await getApp({
- override: [
- { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
- { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
- { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
- { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
- { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
- ],
- useChild: true,
- });
+ const [app, container] = await getApp(getBaseRegisterOptions());
+
  depContainer = container;
  requestSender = new ItemRequestSender(app);
  });
@@ -245,7 +236,47 @@ describe('/search/control/items', function () {
  expectedResponse(requestParams, [ITEM_1234, ITEM_1235], expect)
  );
  });
+
+ describe('Redis uses prefix key', () => {
+ it('should return 200 status code and add key to Redis with prefix', async function () {
+ const realConfig = depContainer.resolve<IConfig>(SERVICES.CONFIG);
+ const prefix = 'test-prefix-item';
+
+ const configWithPrefix: IConfig = {
+ ...realConfig,
+ get<T>(key: string): T {
+ if (key === redisConfigPath) {
+ const realRedisConfig = realConfig.get<RedisClient>(redisConfigPath);
+ return { ...realRedisConfig, prefix } as T;
+ }
+ return realConfig.get<T>(key);
+ },
+ };
+
+ const mockRegisterOptions = getBaseRegisterOptions([
+ {
+ token: SERVICES.CONFIG,
+ provider: { useValue: configWithPrefix },
+ },
+ ]);
+
+ const [mockApp, localContainer] = await getApp(mockRegisterOptions);
+ const localRequestSender = new ItemRequestSender(mockApp);
+
+ const redisConnection = localContainer.resolve<RedisClient>(SERVICES.REDIS);
+
+ const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false };
+ const response = await localRequestSender.getItems(requestParams);
+
+ const keys = await redisConnection.keys(prefix + '*');
+ expect(keys.length).toBeGreaterThanOrEqual(1);
+ expect(response.status).toBe(httpStatusCodes.OK);
+
+ await localContainer.dispose();
+ });
+ });
  });
+
  describe('Bad Path', function () {
  // All requests with status code of 400
  it("should return 400 status code and error message when item's command_name", async function () {

tests/integration/control/route/helpers/index.ts

@@ -0,0 +1,20 @@
+import jsLogger from '@map-colonies/js-logger';
+import { trace } from '@opentelemetry/api';
+import { SERVICES } from '@src/common/constants';
+import { S3_REPOSITORY_SYMBOL } from '@src/common/s3/s3Repository';
+import { cronLoadTileLatLonDataSymbol } from '@src/latLon/DAL/latLonDAL';
+import { GetBaseRegisterOptions } from '@tests/integration/helpers/types';
+
+export const getBaseRegisterOptions: GetBaseRegisterOptions = (options = []) => {
+ return {
+ override: [
+ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
+ { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
+ { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
+ { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
+ { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
+ ...options,
+ ],
+ useChild: true,
+ };
+};

tests/integration/control/route/route.spec.ts

@@ -1,16 +1,14 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import jsLogger from '@map-colonies/js-logger';
-import { trace } from '@opentelemetry/api';
 import httpStatusCodes from 'http-status-codes';
 import { DependencyContainer } from 'tsyringe';
 import { CleanupRegistry } from '@map-colonies/cleanup-registry';
+import { IConfig } from 'config';
+import { RedisClient } from '@src/common/redis';
 import { getApp } from '../../../../src/app';
-import { SERVICES } from '../../../../src/common/constants';
+import { redisConfigPath, SERVICES } from '../../../../src/common/constants';
 import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController';
 import { Route } from '../../../../src/control/route/models/route';
 import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces';
-import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository';
-import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL';
 import { expectedResponse } from '../utils';
 import {
  ROUTE_VIA_CAMILLUCCIA_A,
@@ -19,22 +17,14 @@ import {
  CONTROL_POINT_OLIMPIADE_112,
 } from '../../../mockObjects/routes';
 import { RouteRequestSender } from './helpers/requestSender';
+import { getBaseRegisterOptions } from './helpers';
 
 describe('/search/control/route', function () {
  let requestSender: RouteRequestSender;
  let depContainer: DependencyContainer;
 
  beforeEach(async function () {
- const [app, container] = await getApp({
- override: [
- { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
- { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
- { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
- { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
- { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
- ],
- useChild: true,
- });
+ const [app, container] = await getApp(getBaseRegisterOptions());
 
  depContainer = container;
  requestSender = new RouteRequestSender(app);
@@ -382,7 +372,47 @@ describe('/search/control/route', function () {
  bbox: null,
  });
  });
+
+ describe('Redis uses prefix key', () => {
+ it('should return 200 status code and add key to Redis with prefix', async function () {
+ const realConfig = depContainer.resolve<IConfig>(SERVICES.CONFIG);
+ const prefix = 'test-prefix-route';
+
+ const configWithPrefix: IConfig = {
+ ...realConfig,
+ get<T>(key: string): T {
+ if (key === redisConfigPath) {
+ const realRedisConfig = realConfig.get<RedisClient>(redisConfigPath);
+ return { ...realRedisConfig, prefix } as T;
+ }
+ return realConfig.get<T>(key);
+ },
+ };
+
+ const mockRegisterOptions = getBaseRegisterOptions([
+ {
+ token: SERVICES.CONFIG,
+ provider: { useValue: configWithPrefix },
+ },
+ ]);
+
+ const [mockApp, localContainer] = await getApp(mockRegisterOptions);
+ const localRequestSender = new RouteRequestSender(mockApp);
+
+ const redisConnection = localContainer.resolve<RedisClient>(SERVICES.REDIS);
+
+ const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false };
+ const response = await localRequestSender.getRoutes(requestParams);
+
+ const keys = await redisConnection.keys(prefix + '*');
+ expect(keys.length).toBeGreaterThanOrEqual(1);
+ expect(response.status).toBe(httpStatusCodes.OK);
+
+ await localContainer.dispose();
+ });
+ });
  });
+
  describe('Bad Path', function () {
  // All requests with status code of 400
  it('should return 400 status code and error message when empty object is passed', async function () {

tests/integration/control/tile/helpers/index.ts

@@ -0,0 +1,20 @@
+import jsLogger from '@map-colonies/js-logger';
+import { trace } from '@opentelemetry/api';
+import { SERVICES } from '@src/common/constants';
+import { S3_REPOSITORY_SYMBOL } from '@src/common/s3/s3Repository';
+import { cronLoadTileLatLonDataSymbol } from '@src/latLon/DAL/latLonDAL';
+import { GetBaseRegisterOptions } from '@tests/integration/helpers/types';
+
+export const getBaseRegisterOptions: GetBaseRegisterOptions = (options = []) => {
+ return {
+ override: [
+ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } },
+ { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } },
+ { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } },
+ { token: SERVICES.S3_CLIENT, provider: { useValue: {} } },
+ { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } },
+ ...options,
+ ],
+ useChild: true,
+ };
+};