From 0fd0032fce770074bf4d678a5b0f3b4d0026e156 Mon Sep 17 00:00:00 2001 From: Rhys Date: Thu, 1 Jul 2021 13:11:02 +0100 Subject: [PATCH 01/12] feat(search-index): add search-index export command --- package-lock.json | 54 +- package.json | 3 +- src/__snapshots__/cli.spec.ts.snap | 1 + src/commands/hub/steps/settings-clone-step.ts | 2 +- src/commands/search-index.ts | 16 + .../__snapshots__/export.spec.ts.snap | 186 ++++ src/commands/search-index/export.spec.ts | 823 ++++++++++++++++++ src/commands/search-index/export.ts | 258 ++++++ src/commands/search-index/import.ts | 32 + 9 files changed, 1325 insertions(+), 50 deletions(-) create mode 100644 src/commands/search-index.ts create mode 100644 src/commands/search-index/__snapshots__/export.spec.ts.snap create mode 100644 src/commands/search-index/export.spec.ts create mode 100644 src/commands/search-index/export.ts create mode 100644 src/commands/search-index/import.ts diff --git a/package-lock.json b/package-lock.json index 85a1d293..872865fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,13 +12,14 @@ "ajv": "^6.12.3", "axios": "^0.21.1", "chalk": "^2.4.2", - "dc-management-sdk-js": "^1.13.0", + "dc-management-sdk-js": "github:rs-amp/dc-management-sdk-js#feature/search-index-webhook", "lodash": "^4.17.21", "node-fetch": "^2.6.1", "promise-retry": "^2.0.1", "rimraf": "^3.0.0", "sanitize-filename": "^1.6.3", "table": "^5.4.6", + "url-template": "^2.0.8", "yargs": "^14.0.0" }, "bin": { @@ -4391,8 +4392,8 @@ }, "node_modules/dc-management-sdk-js": { "version": "1.13.0", - "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.13.0.tgz", - "integrity": "sha512-E97UYNvDqLQ80SvxV1T73/1k6Qb43+kV043QJIiB5QgYIiyRIleBOIX5NCzZzzb65Ti0D7WOvSqYHoVM8lQ4Ag==", + "resolved": "git+ssh://git@github.com/rs-amp/dc-management-sdk-js.git#9dd4a827bc00401daae8c7cdd2a30d0a143a1654", + "license": "Apache-2.0", "dependencies": { "axios": "^0.21.1", "url-template": "^2.0.8" @@ -4401,33 +4402,6 @@ "node": ">=10" } }, - "node_modules/dc-management-sdk-js/node_modules/axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", - "dependencies": { - "follow-redirects": "^1.10.0" - } - }, - "node_modules/dc-management-sdk-js/node_modules/follow-redirects": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", - "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -15739,27 +15713,11 @@ "dev": true }, "dc-management-sdk-js": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.13.0.tgz", - "integrity": "sha512-E97UYNvDqLQ80SvxV1T73/1k6Qb43+kV043QJIiB5QgYIiyRIleBOIX5NCzZzzb65Ti0D7WOvSqYHoVM8lQ4Ag==", + "version": "git+ssh://git@github.com/rs-amp/dc-management-sdk-js.git#9dd4a827bc00401daae8c7cdd2a30d0a143a1654", + "from": "dc-management-sdk-js@github:rs-amp/dc-management-sdk-js#feature/search-index-webhook", "requires": { "axios": "^0.21.1", "url-template": "^2.0.8" - }, - "dependencies": { - "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", - "requires": { - "follow-redirects": "^1.10.0" - } - }, - "follow-redirects": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", - "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==" - } } }, "debug": { diff --git a/package.json b/package.json index 0a58fbd6..9cafa96a 100644 --- a/package.json +++ b/package.json @@ -115,13 +115,14 @@ "ajv": "^6.12.3", "axios": "^0.21.1", "chalk": "^2.4.2", - "dc-management-sdk-js": "^1.13.0", + "dc-management-sdk-js": "github:rs-amp/dc-management-sdk-js#feature/search-index-webhook", "lodash": "^4.17.21", "node-fetch": "^2.6.1", "promise-retry": "^2.0.1", "rimraf": "^3.0.0", "sanitize-filename": "^1.6.3", "table": "^5.4.6", + "url-template": "^2.0.8", "yargs": "^14.0.0" }, "engines": { diff --git a/src/__snapshots__/cli.spec.ts.snap b/src/__snapshots__/cli.spec.ts.snap index 02465bd7..70e5ee40 100644 --- a/src/__snapshots__/cli.spec.ts.snap +++ b/src/__snapshots__/cli.spec.ts.snap @@ -12,6 +12,7 @@ Commands: dc-cli event Event dc-cli extension Extension dc-cli hub Hub + dc-cli search-index Search Index dc-cli settings Settings Options: diff --git a/src/commands/hub/steps/settings-clone-step.ts b/src/commands/hub/steps/settings-clone-step.ts index c04af375..d63abf2f 100644 --- a/src/commands/hub/steps/settings-clone-step.ts +++ b/src/commands/hub/steps/settings-clone-step.ts @@ -58,7 +58,7 @@ export class SettingsCloneStep implements CloneHubStep { await importSettings({ filePath: join(state.path, 'settings', matchingFile), - mapFile: state.argv.mapFile, + //mapFile: state.argv.mapFile, force: state.argv.force, logFile: state.logFile, ...state.to diff --git a/src/commands/search-index.ts b/src/commands/search-index.ts new file mode 100644 index 00000000..ff53211d --- /dev/null +++ b/src/commands/search-index.ts @@ -0,0 +1,16 @@ +import { Argv } from 'yargs'; +import YargsCommandBuilderOptions from '../common/yargs/yargs-command-builder-options'; + +export const command = 'search-index'; + +export const desc = 'Search Index'; + +export const builder = (yargs: Argv): Argv => + yargs + .commandDir('search-index', YargsCommandBuilderOptions) + .demandCommand() + .help(); + +export const handler = (): void => { + /* do nothing */ +}; diff --git a/src/commands/search-index/__snapshots__/export.spec.ts.snap b/src/commands/search-index/__snapshots__/export.spec.ts.snap new file mode 100644 index 00000000..3f99f49a --- /dev/null +++ b/src/commands/search-index/__snapshots__/export.spec.ts.snap @@ -0,0 +1,186 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`search-index export command filterIndicesById should throw an error for ids which do not exist in the list of indices 1`] = ` +"The following ID(s) could not be found: ['index-id-4']. +Nothing was exported, exiting." +`; + +exports[`search-index export command getExports should return a list of indices to export and a list of filenames that will be updated 1`] = ` +Array [ + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir", + Object { + "export-dir/export-filename-1.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir/export-filename-2.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + }, + ], + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "export-dir", + Object { + "export-dir/export-filename-1.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir/export-filename-2.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + }, + ], +] +`; + +exports[`search-index export command getExports should return a list of indices to export and a list of filenames that will be updated 2`] = ` +Array [ + Object { + "filename": "export-dir/export-filename-1.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "status": "CREATED", + }, + Object { + "filename": "export-dir/export-filename-2.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "status": "UPDATED", + }, +] +`; + +exports[`search-index export command getExports should return a list of indices to export and a list of filenames that will be updated 3`] = ` +Array [ + Object { + "filename": "export-dir/export-filename-2.json", + "uri": "index-name-2", + }, +] +`; + +exports[`search-index export command getExports should return a list of indices to export and no filenames that will be updated (first export) 1`] = ` +Array [ + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir", + Object {}, + ], + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "export-dir", + Object {}, + ], +] +`; + +exports[`search-index export command getExports should return a list of indices to export and no filenames that will be updated (first export) 2`] = ` +Array [ + Object { + "filename": "export-dir/export-filename-1.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "status": "CREATED", + }, + Object { + "filename": "export-dir/export-filename-2.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "status": "CREATED", + }, +] +`; + +exports[`search-index export command processIndices should output a message if no indices to export from hub 1`] = ` +Array [ + Array [ + "No search indices to export from this hub, exiting. +", + ], +] +`; diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts new file mode 100644 index 00000000..adc0bfc8 --- /dev/null +++ b/src/commands/search-index/export.spec.ts @@ -0,0 +1,823 @@ +import * as exportModule from './export'; +import * as directoryUtils from '../../common/import/directory-utils'; +import { + builder, + command, + filterIndicesById, + getIndexExports, + getExportRecordForIndex, + handler, + LOG_FILENAME, + processIndices, + EnrichedSearchIndex, + EnrichedAssignedContentType +} from './export'; +import Yargs from 'yargs/yargs'; +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { SearchIndex, SearchIndexSettings, Webhook } from 'dc-management-sdk-js'; +import MockPage from '../../common/dc-management-sdk-js/mock-page'; +import * as exportServiceModule from '../../services/export.service'; +import { table } from 'table'; +import chalk from 'chalk'; +import { loadJsonFromDirectory } from '../../services/import.service'; +import { FileLog } from '../../common/file-log'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { validateNoDuplicateIndexNames } from './import'; +import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; +import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; + +jest.mock('../../services/dynamic-content-client-factory'); +jest.mock('./import'); +jest.mock('../../services/import.service'); +jest.mock('../../common/import/directory-utils'); +jest.mock('table'); +jest.mock('../../common/log-helpers'); + +describe('search-index export command', (): void => { + afterEach((): void => { + jest.restoreAllMocks(); + }); + + afterAll(() => { + jest.resetModules(); + }); + + it('should implement an export command', () => { + expect(command).toEqual('export '); + }); + + describe('builder tests', () => { + it('should configure yargs', () => { + const argv = Yargs(process.argv.slice(2)); + const spyPositional = jest.spyOn(argv, 'positional').mockReturnThis(); + const spyOption = jest.spyOn(argv, 'option').mockReturnThis(); + + builder(argv); + + expect(spyPositional).toHaveBeenCalledWith('dir', { + describe: 'Output directory for the exported Search Index definitions', + type: 'string' + }); + expect(spyOption).toHaveBeenCalledWith('id', { + type: 'string', + describe: + 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indices for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indices at the same time.', + requiresArg: true + }); + expect(spyOption).toHaveBeenCalledWith('f', { + type: 'boolean', + boolean: true, + describe: 'Overwrite search indices without asking.' + }); + expect(spyOption).toHaveBeenCalledWith('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); + }); + }); + + describe('enrichIndex tests', () => { + it('should request settings, types, keys and webhooks to enrich the given indices', async (): Promise => { + const index = new SearchIndex({ + name: 'account.suffix-1', + suffix: 'suffix-1', + label: 'Index 1', + type: 'STAGING' + }); + + const settings = new SearchIndexSettings({ + example: 'setting' + }); + + const key = new SearchIndexKey({ + id: 'key-id', + key: 'example-key' + }); + + const assignedContentTypes = [ + new AssignedContentType({ contentTypeUri: 'http://1' }), + new AssignedContentType({ contentTypeUri: 'http://2' }) + ]; + + const enrichedContentTypes: EnrichedAssignedContentType[] = []; + + assignedContentTypes.forEach((type, index) => { + type.related.webhook = jest.fn().mockResolvedValue(new Webhook({ id: 'webhook-' + index })); + type.related.activeContentWebhook = jest.fn().mockResolvedValue(new Webhook({ id: 'webhook-active-' + index })); + type.related.archivedContentWebhook = jest + .fn() + .mockResolvedValue(new Webhook({ id: 'webhook-archive-' + index })); + + enrichedContentTypes[index] = new EnrichedAssignedContentType({ + contentTypeUri: type.contentTypeUri, + webhook: 'webhook-' + index, + activeContentWebhook: 'webhook-active-' + index, + archivedContentWebhook: 'webhook-archive-' + index + }); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (index as any).related = { + settings: { + get: jest.fn().mockResolvedValue(settings) + }, + assignedContentTypes: { + list: jest.fn().mockResolvedValue(new MockPage(AssignedContentType, assignedContentTypes)) + }, + keys: { + get: jest.fn().mockResolvedValue(key) + } + }; + + const expectedEnriched = new EnrichedSearchIndex({ + ...index.toJSON(), + settings: settings, + keys: key, + assignedContentTypes: enrichedContentTypes + }); + + const enriched = await exportModule.enrichIndex(index); + + expect(index.related.settings.get).toHaveBeenCalledTimes(1); + expect(index.related.assignedContentTypes.list).toHaveBeenCalledTimes(1); + expect(index.related.keys.get).toHaveBeenCalledTimes(1); + + for (const type of assignedContentTypes) { + expect(type.related.webhook).toHaveBeenCalledTimes(1); + expect(type.related.activeContentWebhook).toHaveBeenCalledTimes(1); + expect(type.related.archivedContentWebhook).toHaveBeenCalledTimes(1); + } + + expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); + }); + }); + + describe('getExports', () => { + let getExportRecordForIndexSpy: jest.SpyInstance; + + const indicesToExport = [ + new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + assignedContentTypes: [ + new AssignedContentType({ + id: 'assigned-type-1' + }) + ] + }), + new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + assignedContentTypes: [ + new AssignedContentType({ + id: 'assigned-type-2' + }) + ] + }) + ]; + + const exportedIndices = { + 'export-dir/export-filename-1.json': indicesToExport[0], + 'export-dir/export-filename-2.json': indicesToExport[1] + }; + + beforeEach(() => { + getExportRecordForIndexSpy = jest.spyOn(exportModule, 'getExportRecordForIndex'); + }); + + it('should return a list of indices to export and no filenames that will be updated (first export)', () => { + getExportRecordForIndexSpy + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-1.json', + status: 'CREATED', + index: indicesToExport[0] + }) + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-2.json', + status: 'CREATED', + index: indicesToExport[1] + }); + + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, indicesToExport); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); + expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); + expect(allExports).toMatchSnapshot(); + expect(updatedExportsMap).toEqual([]); + }); + + it('should return a list of indices to export and a list of filenames that will be updated', () => { + getExportRecordForIndexSpy + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-1.json', + status: 'CREATED', + index: indicesToExport[0] + }) + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: indicesToExport[1] + }); + + const [allExports, updatedExportsMap] = getIndexExports('export-dir', exportedIndices, indicesToExport); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); + expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); + expect(allExports).toMatchSnapshot(); + expect(updatedExportsMap).toMatchSnapshot(); + }); + + it('should not return a list of indices to export or a list of filenames that will be updated', () => { + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, []); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(0); + expect(allExports).toEqual([]); + expect(updatedExportsMap).toEqual([]); + }); + + it('should skip any that are missing a name', () => { + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, [ + new EnrichedSearchIndex({ + label: 'index 1' + }) + ]); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(0); + expect(allExports).toEqual([]); + expect(updatedExportsMap).toEqual([]); + }); + }); + + describe('getExportRecordForIndex', () => { + const extraResources = { + settings: new SearchIndexSettings({ + setting: 'test' + }), + keys: new SearchIndexKey({ + key: 'test' + }), + assignedContentTypes: [ + new AssignedContentType({ + id: 'assigned-type-1' + }) + ] + }; + + it('should create export for any newly exported index', async () => { + const exportedIndices = { + 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + ...extraResources + }), + 'export-dir/export-filename-2.json': new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + ...extraResources + }) + }; + const newIndexToExport = new EnrichedSearchIndex({ + name: 'index-name-3', + label: 'index 3', + ...extraResources + }); + + jest.spyOn(exportServiceModule, 'uniqueFilenamePath').mockReturnValueOnce('export-dir/export-filename-3.json'); + + const existingIndices = Object.keys(exportedIndices); + + const result = getExportRecordForIndex(newIndexToExport, 'export-dir', exportedIndices); + + expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledWith( + 'export-dir', + newIndexToExport.name, + 'json', + existingIndices + ); + expect(result).toEqual({ + filename: 'export-dir/export-filename-3.json', + status: 'CREATED', + index: newIndexToExport + }); + }); + + it('should update export for any index with different content', async () => { + const exportedIndices = { + 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + ...extraResources + }), + 'export-dir/export-filename-2.json': new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + ...extraResources + }) + }; + const updatedIndexToExport = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2 - mutated label', + ...extraResources + }); + + jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); + + const result = getExportRecordForIndex(updatedIndexToExport, 'export-dir', exportedIndices); + + expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); + expect(result).toEqual({ + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: updatedIndexToExport + }); + }); + + it('should not update export for any index with same content', async () => { + const exportedIndices = { + 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + ...extraResources + }), + 'export-dir/export-filename-2.json': new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + ...extraResources + }) + }; + const unchangedIndexToExport = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2', + ...extraResources + }); + + jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); + + const result = getExportRecordForIndex(unchangedIndexToExport, 'export-dir', exportedIndices); + + expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); + expect(result).toEqual({ + filename: 'export-dir/export-filename-2.json', + status: 'UP-TO-DATE', + index: unchangedIndexToExport + }); + }); + }); + + describe('filterIndicesById', () => { + const listToFilter = [ + new EnrichedSearchIndex({ + id: 'index-id-1', + label: 'index 1' + }), + new EnrichedSearchIndex({ + id: 'index-id-2', + label: 'index 2' + }), + new EnrichedSearchIndex({ + id: 'index-id-3', + label: 'index 3' + }) + ]; + + it('should return the indices matching the given uris', async () => { + const result = filterIndicesById(listToFilter, ['index-id-1', 'index-id-3']); + expect(result).toEqual(expect.arrayContaining([listToFilter[0], listToFilter[2]])); + }); + + it('should return all the indices because there are no URIs to filter', async () => { + const result = filterIndicesById(listToFilter, []); + expect(result).toEqual(listToFilter); + }); + + it('should throw an error for ids which do not exist in the list of indices', async () => { + expect(() => + filterIndicesById(listToFilter, ['index-id-1', 'index-id-4', 'index-id-3']) + ).toThrowErrorMatchingSnapshot(); + }); + }); + + describe('processIndices', () => { + let mockEnsureDirectory: jest.Mock; + let mockTable: jest.Mock; + let stdoutSpy: jest.SpyInstance; + + const indicesToProcess = [ + new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1', + label: 'index 1', + status: 'ACTIVE' + }), + new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2', + status: 'ACTIVE' + }), + new EnrichedSearchIndex({ + id: 'index-id-3', + name: 'index-name-3', + label: 'index 3', + status: 'ACTIVE' + }) + ]; + + const exportedIndices = [ + { + name: 'index-name-1', + label: 'index 1', + status: 'ACTIVE' + }, + { + name: 'index-name-2', + label: 'index 2', + status: 'ACTIVE' + }, + { + name: 'index-name-3', + label: 'index 3', + status: 'ACTIVE' + } + ]; + + beforeEach(() => { + mockEnsureDirectory = directoryUtils.ensureDirectoryExists as jest.Mock; + mockTable = table as jest.Mock; + mockTable.mockImplementation(jest.requireActual('table').table); + jest.spyOn(exportServiceModule, 'writeJsonToFile').mockImplementation(); + stdoutSpy = jest.spyOn(process.stdout, 'write'); + stdoutSpy.mockImplementation(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should output export files for the given indices if nothing previously exported', async () => { + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'CREATED', + index: indicesToProcess[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'CREATED', + index: indicesToProcess[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'CREATED', + index: indicesToProcess[2] + } + ], + [] + ]); + + const previouslyExportedIndices = {}; + await processIndices('export-dir', previouslyExportedIndices, indicesToProcess, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndices, + indicesToProcess + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); + + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(3); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 1, + 'export-dir/export-filename-1.json', + expect.objectContaining(exportedIndices[0]) + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 2, + 'export-dir/export-filename-2.json', + expect.objectContaining(exportedIndices[1]) + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 3, + 'export-dir/export-filename-3.json', + expect.objectContaining(exportedIndices[2]) + ); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], + ['export-dir/export-filename-1.json', indicesToProcess[0].name, 'CREATED'], + ['export-dir/export-filename-2.json', indicesToProcess[1].name, 'CREATED'], + ['export-dir/export-filename-3.json', indicesToProcess[2].name, 'CREATED'] + ], + streamTableOptions + ); + }); + + it('should output a message if no indices to export from hub', async () => { + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([[], []]); + + const previouslyExportedIndices = {}; + + await processIndices('export-dir', previouslyExportedIndices, [], new FileLog(), false); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(0); + expect(stdoutSpy.mock.calls).toMatchSnapshot(); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(0); + expect(mockTable).toHaveBeenCalledTimes(0); + }); + + it('should not output any export files if a previous export exists and the index is unchanged', async () => { + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'UP-TO-DATE', + index: indicesToProcess[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'UP-TO-DATE', + index: indicesToProcess[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'UP-TO-DATE', + index: indicesToProcess[2] + } + ], + [] + ]); + + const previouslyExportedIndices = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) + }; + await processIndices('export-dir', previouslyExportedIndices, indicesToProcess, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndices, + indicesToProcess + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(0); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], + ['export-dir/export-filename-1.json', indicesToProcess[0].name, 'UP-TO-DATE'], + ['export-dir/export-filename-2.json', indicesToProcess[1].name, 'UP-TO-DATE'], + ['export-dir/export-filename-3.json', indicesToProcess[2].name, 'UP-TO-DATE'] + ], + streamTableOptions + ); + }); + + it('should update the existing export file for a changed index', async () => { + const mutatedIndices = [...indicesToProcess]; + mutatedIndices[1] = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2 - mutated label', + status: 'ACTIVE' + }); + + jest.spyOn(exportServiceModule, 'promptToOverwriteExports').mockResolvedValueOnce(true); + + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'UP-TO-DATE', + index: mutatedIndices[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: mutatedIndices[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'UP-TO-DATE', + index: mutatedIndices[2] + } + ], + [ + { + filename: 'export-dir/export-filename-2.json', + uri: mutatedIndices[1].id as string + } + ] + ]); + + const previouslyExportedIndices = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) + }; + + await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndices, + mutatedIndices + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(1); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], + ['export-dir/export-filename-1.json', indicesToProcess[0].name, 'UP-TO-DATE'], + ['export-dir/export-filename-2.json', indicesToProcess[1].name, 'UPDATED'], + ['export-dir/export-filename-3.json', indicesToProcess[2].name, 'UP-TO-DATE'] + ], + streamTableOptions + ); + }); + + it('should not update anything if the user says "n" to the overwrite prompt', async () => { + const mutatedIndices = [...indicesToProcess]; + mutatedIndices[1] = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2 - mutated label', + status: 'ACTIVE' + }); + + jest.spyOn(exportServiceModule, 'promptToOverwriteExports').mockResolvedValueOnce(false); + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'UP-TO-DATE', + index: mutatedIndices[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: mutatedIndices[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'UP-TO-DATE', + index: mutatedIndices[2] + } + ], + [ + { + filename: 'export-dir/export-filename-2.json', + uri: mutatedIndices[1].id as string + } + ] + ]); + + const previouslyExportedIndices = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) + }; + + await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndices, + mutatedIndices + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(0); + expect(mockTable).toHaveBeenCalledTimes(0); + }); + }); + + describe('handler tests', () => { + const yargArgs = { + $0: 'test', + _: ['test'] + }; + const config = { + clientId: 'client-id', + clientSecret: 'client-id', + hubId: 'hub-id' + }; + + const indicesToExport: EnrichedSearchIndex[] = [ + new EnrichedSearchIndex({ + name: 'account.suffix-1', + suffix: 'suffix-1', + label: 'Index 1', + type: 'STAGING', + settings: {}, + keys: {}, + assignedContentTypes: {} + }), + new EnrichedSearchIndex({ + name: 'account.suffix-2', + suffix: 'suffix-2', + label: 'Index 2', + type: 'STAGING', + settings: {}, + keys: {}, + assignedContentTypes: {} + }) + ]; + + let mockGetHub: jest.Mock; + let mockList: jest.Mock; + + beforeEach(() => { + (loadJsonFromDirectory as jest.Mock).mockReturnValue([]); + + const listResponse = new MockPage(SearchIndex, indicesToExport); + mockList = jest.fn().mockResolvedValue(listResponse); + + mockGetHub = jest.fn().mockResolvedValue({ + related: { + searchIndexes: { + list: mockList + } + } + }); + + (dynamicContentClientFactory as jest.Mock).mockReturnValue({ + hubs: { + get: mockGetHub + } + }); + jest.spyOn(exportModule, 'processIndices').mockResolvedValue(); + }); + + it('should use getDefaultLogPath for LOG_FILENAME with process.platform as default', function() { + LOG_FILENAME(); + + expect(getDefaultLogPath).toHaveBeenCalledWith('search-index', 'export', process.platform); + }); + + it('should export all indices for the current hub if no ids specified', async (): Promise => { + const schemaIdsToExport: string[] | undefined = undefined; + const argv = { ...yargArgs, ...config, dir: 'my-dir', schemaId: schemaIdsToExport, logFile: new FileLog() }; + + const filteredIndicesToExport = [...indicesToExport]; + jest.spyOn(exportModule, 'enrichIndex').mockImplementation(x => Promise.resolve(x as EnrichedSearchIndex)); + jest.spyOn(exportModule, 'filterIndicesById').mockReturnValue(filteredIndicesToExport); + + await handler(argv); + + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalledTimes(1); + expect(mockList).toHaveBeenCalledWith(undefined, undefined, { size: 100 }); + expect(loadJsonFromDirectory).toHaveBeenCalledWith(argv.dir, EnrichedSearchIndex); + expect(validateNoDuplicateIndexNames).toHaveBeenCalled(); + expect(exportModule.filterIndicesById).toHaveBeenCalledWith(indicesToExport, []); + expect(exportModule.processIndices).toHaveBeenCalledWith( + argv.dir, + [], + filteredIndicesToExport, + expect.any(FileLog), + false + ); + }); + + it('should export only selected indices if ids specified', async (): Promise => { + const idsToExport: string[] | undefined = ['index-id-2']; + const argv = { ...yargArgs, ...config, dir: 'my-dir', id: idsToExport, logFile: new FileLog() }; + + const filteredIndicesToExport = [indicesToExport[1]]; + jest.spyOn(exportModule, 'enrichIndex').mockImplementation(x => Promise.resolve(x as EnrichedSearchIndex)); + jest.spyOn(exportModule, 'filterIndicesById').mockReturnValue(filteredIndicesToExport); + + await handler(argv); + + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalled(); + expect(loadJsonFromDirectory).toHaveBeenCalledWith(argv.dir, EnrichedSearchIndex); + expect(validateNoDuplicateIndexNames).toHaveBeenCalled(); + expect(exportModule.filterIndicesById).toHaveBeenCalledWith(indicesToExport, idsToExport); + expect(exportModule.processIndices).toHaveBeenCalledWith( + argv.dir, + [], + filteredIndicesToExport, + expect.any(FileLog), + false + ); + }); + }); +}); diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts new file mode 100644 index 00000000..dd9835fb --- /dev/null +++ b/src/commands/search-index/export.ts @@ -0,0 +1,258 @@ +import chalk from 'chalk'; +import { HalResource, Hub, Page, Pageable, SearchIndex, SearchIndexSettings, Sortable } from 'dc-management-sdk-js'; +import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; +import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; +import { isEqual } from 'lodash'; +import { table } from 'table'; +import { Arguments, Argv } from 'yargs'; +import paginator from '../../common/dc-management-sdk-js/paginator'; +import { FileLog } from '../../common/file-log'; +import { ensureDirectoryExists } from '../../common/import/directory-utils'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { ExportBuilderOptions } from '../../interfaces/export-builder-options.interface'; +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { + ExportResult, + nothingExportedExit, + promptToOverwriteExports, + uniqueFilenamePath, + writeJsonToFile +} from '../../services/export.service'; +import { loadJsonFromDirectory } from '../../services/import.service'; +import { ConfigurationParameters } from '../configure'; +import { validateNoDuplicateIndexNames } from './import'; + +export const command = 'export '; + +export const desc = 'Export Search Indices'; + +export const LOG_FILENAME = (platform: string = process.platform): string => + getDefaultLogPath('search-index', 'export', platform); + +export const builder = (yargs: Argv): void => { + yargs + .positional('dir', { + describe: 'Output directory for the exported Search Index definitions', + type: 'string' + }) + .option('id', { + type: 'string', + describe: + 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indices for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indices at the same time.', + requiresArg: true + }) + .alias('f', 'force') + .option('f', { + type: 'boolean', + boolean: true, + describe: 'Overwrite search indices without asking.' + }) + .option('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); +}; + +const ensureJSON = (resource: HalResource): object => { + return resource.toJSON != null ? resource.toJSON() : resource; +}; + +const equals = (a: EnrichedSearchIndex, b: EnrichedSearchIndex): boolean => + a.label === b.label && + isEqual(a.assignedContentTypes.map(x => ensureJSON(x)), b.assignedContentTypes.map(x => ensureJSON(x))) && + isEqual(ensureJSON(a.keys), ensureJSON(b.keys)) && + isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); + +const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { + return (options?: Pageable & Sortable): Promise> => + hub.related.searchIndexes.list(parentId, projection, options); +}; + +export class EnrichedAssignedContentType extends AssignedContentType { + webhook: string; + activeContentWebhook: string; + archivedContentWebhook: string; +} + +export class EnrichedSearchIndex extends SearchIndex { + settings: SearchIndexSettings; + keys: SearchIndexKey; + assignedContentTypes: EnrichedAssignedContentType[]; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public toJSON(): any { + const result = super.toJSON(); + + result.assignedContentTypes = result.assignedContentTypes.map((type: AssignedContentType) => type.toJSON()); + return result; + } +} + +export const filterIndicesById = ( + listToFilter: EnrichedSearchIndex[], + indexIdList: string[] +): EnrichedSearchIndex[] => { + if (indexIdList.length === 0) { + return listToFilter; + } + + const unmatchedIndexUriList: string[] = indexIdList.filter(id => !listToFilter.some(index => index.id === id)); + if (unmatchedIndexUriList.length > 0) { + throw new Error( + `The following ID(s) could not be found: [${unmatchedIndexUriList + .map(u => `'${u}'`) + .join(', ')}].\nNothing was exported, exiting.` + ); + } + + return listToFilter.filter(index => indexIdList.some(id => index.id === id)); +}; + +export const enrichIndex = async (index: SearchIndex): Promise => { + const enrichedIndex = new EnrichedSearchIndex(index); + + enrichedIndex.settings = await index.related.settings.get(); + const types = await paginator(index.related.assignedContentTypes.list); + enrichedIndex.keys = await index.related.keys.get(); + + const enrichedTypes: EnrichedAssignedContentType[] = []; + + for (const type of types) { + const enriched = new EnrichedAssignedContentType(type); + enriched.webhook = (await type.related.webhook()).id as string; + enriched.activeContentWebhook = (await type.related.activeContentWebhook()).id as string; + enriched.archivedContentWebhook = (await type.related.archivedContentWebhook()).id as string; + + enrichedTypes.push(enriched); + } + + enrichedIndex.assignedContentTypes = enrichedTypes; + + return enrichedIndex; +}; + +interface ExportRecord { + readonly filename: string; + readonly status: ExportResult; + readonly index: EnrichedSearchIndex; +} + +export const getExportRecordForIndex = ( + index: EnrichedSearchIndex, + outputDir: string, + previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex } +): ExportRecord => { + const indexOfExportedIndex = Object.values(previouslyExportedIndices).findIndex(c => c.name === index.name); + + if (indexOfExportedIndex < 0) { + const filename = uniqueFilenamePath(outputDir, index.name, 'json', Object.keys(previouslyExportedIndices)); + + // This filename is now used. + previouslyExportedIndices[filename] = index; + + return { + filename: filename, + status: 'CREATED', + index + }; + } + const filename = Object.keys(previouslyExportedIndices)[indexOfExportedIndex]; + const previouslyExportedIndex = Object.values(previouslyExportedIndices)[indexOfExportedIndex]; + + if (equals(previouslyExportedIndex, index)) { + return { filename, status: 'UP-TO-DATE', index }; + } + + return { + filename, + status: 'UPDATED', + index + }; +}; + +type ExportsMap = { + uri: string; + filename: string; +}; + +export const getIndexExports = ( + outputDir: string, + previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, + indicesBeingExported: EnrichedSearchIndex[] +): [ExportRecord[], ExportsMap[]] => { + const allExports: ExportRecord[] = []; + const updatedExportsMap: ExportsMap[] = []; // uri x filename + for (const index of indicesBeingExported) { + if (!index.name) { + continue; + } + + const exportRecord = getExportRecordForIndex(index, outputDir, previouslyExportedIndices); + + allExports.push(exportRecord); + + if (exportRecord.status === 'UPDATED') { + updatedExportsMap.push({ uri: index.name, filename: exportRecord.filename }); + } + } + return [allExports, updatedExportsMap]; +}; + +export const processIndices = async ( + outputDir: string, + previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, + indicesBeingExported: EnrichedSearchIndex[], + log: FileLog, + force: boolean +): Promise => { + if (indicesBeingExported.length === 0) { + nothingExportedExit(log, 'No search indices to export from this hub, exiting.'); + return; + } + + const [allExports, updatedExportsMap] = getIndexExports(outputDir, previouslyExportedIndices, indicesBeingExported); + if ( + allExports.length === 0 || + (Object.keys(updatedExportsMap).length > 0 && !(force || (await promptToOverwriteExports(updatedExportsMap, log)))) + ) { + nothingExportedExit(log); + return; + } + + await ensureDirectoryExists(outputDir); + + const data: string[][] = []; + + data.push([chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')]); + for (const { filename, status, index } of allExports) { + if (status !== 'UP-TO-DATE') { + delete index.id; // do not export id + writeJsonToFile(filename, index); + } + data.push([filename, index.name as string, status]); + } + + log.appendLine(table(data, streamTableOptions)); +}; + +export const handler = async (argv: Arguments): Promise => { + const { dir, id, logFile, force } = argv; + const client = dynamicContentClientFactory(argv); + const hub = await client.hubs.get(argv.hubId); + const log = logFile.open(); + + const previouslyExportedIndices = loadJsonFromDirectory(dir, EnrichedSearchIndex); + validateNoDuplicateIndexNames(previouslyExportedIndices); + + const storedIndices = await paginator(searchIndexList(hub)); + const enrichedIndices = await Promise.all(storedIndices.map(enrichIndex)); + + const idArray: string[] = id ? (Array.isArray(id) ? id : [id]) : []; + const filteredIndices = filterIndicesById(enrichedIndices, idArray); + await processIndices(dir, previouslyExportedIndices, filteredIndices, log, force || false); + + await log.close(); +}; diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts new file mode 100644 index 00000000..73d8bc44 --- /dev/null +++ b/src/commands/search-index/import.ts @@ -0,0 +1,32 @@ +import { EnrichedSearchIndex } from './export'; + +type IndexName = string; +type IndexFile = string; + +export const validateNoDuplicateIndexNames = (importedIndices: { + [filename: string]: EnrichedSearchIndex; +}): void | never => { + const nameToFilenameMap = new Map(); // map: name x filenames + for (const [filename, index] of Object.entries(importedIndices)) { + if (index.name) { + const otherFilenames: string[] = nameToFilenameMap.get(index.name) || []; + if (filename) { + nameToFilenameMap.set(index.name, [...otherFilenames, filename]); + } + } + } + const uniqueDuplicateNames: [string, IndexFile[]][] = []; + nameToFilenameMap.forEach((filenames, name) => { + if (filenames.length > 1) { + uniqueDuplicateNames.push([name, filenames]); + } + }); + + if (uniqueDuplicateNames.length > 0) { + throw new Error( + `Indices must have unique name values. Duplicate values found:-\n${uniqueDuplicateNames + .map(([name, filenames]) => ` name: '${name}' in files: [${filenames.map(f => `'${f}'`).join(', ')}]`) + .join('\n')}` + ); + } +}; From 5238f7c5bcbe9958d1a6f8064b1a5c03559408a4 Mon Sep 17 00:00:00 2001 From: Rhys Date: Fri, 30 Jul 2021 13:48:59 +0100 Subject: [PATCH 02/12] feat(search-index): add support for replicas, webhook export (wip import) --- .../__snapshots__/import.spec.ts.snap | 27 ++ .../webhook-rewriter.spec.ts.snap | 48 ++ src/commands/search-index/export.spec.ts | 82 +++- src/commands/search-index/export.ts | 137 +++++- src/commands/search-index/import.spec.ts | 432 ++++++++++++++++++ src/commands/search-index/import.ts | 219 ++++++++- .../search-index/webhook-rewriter.spec.ts | 81 ++++ src/commands/search-index/webhook-rewriter.ts | 63 +++ 8 files changed, 1072 insertions(+), 17 deletions(-) create mode 100644 src/commands/search-index/__snapshots__/import.spec.ts.snap create mode 100644 src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap create mode 100644 src/commands/search-index/import.spec.ts create mode 100644 src/commands/search-index/webhook-rewriter.spec.ts create mode 100644 src/commands/search-index/webhook-rewriter.ts diff --git a/src/commands/search-index/__snapshots__/import.spec.ts.snap b/src/commands/search-index/__snapshots__/import.spec.ts.snap new file mode 100644 index 00000000..0ba373ac --- /dev/null +++ b/src/commands/search-index/__snapshots__/import.spec.ts.snap @@ -0,0 +1,27 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`search-index import command doCreate should throw an error when enrichIndex fails 1`] = ` +"Error creating index index-name: + +The update-index action is not available, ensure you have permission to perform this action." +`; + +exports[`search-index import command doCreate should throw an error when index create fails 1`] = ` +"Error creating index index-name: + +Error: Error creating index" +`; + +exports[`search-index import command doCreate should throw an error when index create fails if a string error is returned by the sdk 1`] = ` +"Error creating index index-name: + +The create-index action is not available, ensure you have permission to perform this action." +`; + +exports[`search-index import command handler tests should throw an error when no content found in import directory 1`] = `"No indices found in my-empty-dir"`; + +exports[`search-index import command validateNoDuplicateIndexNames should throw and error when there are duplicate uris 1`] = ` +"Indices must have unique name values. Duplicate values found:- + name: 'index-name-1' in files: ['file-1', 'file-4'] + name: 'index-name-2' in files: ['file-2', 'file-3']" +`; diff --git a/src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap b/src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap new file mode 100644 index 00000000..9d17879a --- /dev/null +++ b/src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap @@ -0,0 +1,48 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`webhook-rewriter tests should escape quotes and backslashes 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"\\"quotedString\\\\\\\\with\\\\\\\\backslash\\"\\" stagingEnvironment=\\"back\\\\\\\\slash\\"}} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should rewrite a simple webhook 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\" stagingEnvironment=\\"staging\\"}} + \\"content\\" : \\"{{~#each (first (pluck contentBlocks \\"content\\") 4)~}}{{~#each this.values~}}{{#if (eq locale \\"en-GB\\")}}{{{truncate value 1000}}}{{/if}}{{~/each~}}{{#unless @last}};{{/unless~}}{{~/each~}}\\" +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should rewrite multiple tags within the same webhook 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\" stagingEnvironment=\\"staging\\"}} + {{example1}} +{{/withDeliveryContentItem}} + \\"key\\": \\"value\\", +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\" stagingEnvironment=\\"staging\\"}} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should rewrite only one argument if the other is not present 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\"}} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should still replace values with unusual whitespace 1`] = ` +"{ +{{#withDeliveryContentItem + contentItemId=payload.id + account=\\"replaced\\" + stagingEnvironment=\\"whitespace\\" }} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts index adc0bfc8..375bf7fa 100644 --- a/src/commands/search-index/export.spec.ts +++ b/src/commands/search-index/export.spec.ts @@ -136,10 +136,11 @@ describe('search-index export command', (): void => { ...index.toJSON(), settings: settings, keys: key, - assignedContentTypes: enrichedContentTypes + assignedContentTypes: enrichedContentTypes, + replicas: [] }); - const enriched = await exportModule.enrichIndex(index); + const enriched = await exportModule.enrichIndex(new Map(), new Map(), index); expect(index.related.settings.get).toHaveBeenCalledTimes(1); expect(index.related.assignedContentTypes.list).toHaveBeenCalledTimes(1); @@ -263,7 +264,8 @@ describe('search-index export command', (): void => { new AssignedContentType({ id: 'assigned-type-1' }) - ] + ], + replicas: [] }; it('should create export for any newly exported index', async () => { @@ -708,6 +710,66 @@ describe('search-index export command', (): void => { }); }); + describe('separateReplicas', () => { + it('it should separate replicas into a mapping', () => { + const indices = [ + new SearchIndex({ + id: 'parent', + label: 'not-replica', + parentId: null + }), + new SearchIndex({ + id: 'child', + label: 'replica', + parentId: 'parent' + }), + new SearchIndex({ + id: 'child2', + label: 'replica2', + parentId: 'parent' + }), + new SearchIndex({ + id: 'not-parent', + label: 'not-replica2', + parentId: null + }) + ]; + + const { storedIndices, allReplicas } = exportModule.separateReplicas(indices); + + expect(storedIndices).toEqual([indices[0], indices[3]]); + expect(allReplicas.size).toEqual(1); + expect(allReplicas.get('parent')).toEqual([indices[1], indices[2]]); + }); + + it('it should return no replicas when none are present', () => { + const indices = [ + new SearchIndex({ + id: 'parent', + label: 'not-replica', + parentId: null + }), + new SearchIndex({ + id: 'not-parent', + label: 'not-replica2', + parentId: null + }) + ]; + + const { storedIndices, allReplicas } = exportModule.separateReplicas(indices); + + expect(storedIndices).toEqual(indices); + expect(allReplicas.size).toEqual(0); + }); + + it('it should return nothing when no indices are provided', () => { + const { storedIndices, allReplicas } = exportModule.separateReplicas([]); + + expect(storedIndices).toEqual([]); + expect(allReplicas.size).toEqual(0); + }); + }); + describe('handler tests', () => { const yargArgs = { $0: 'test', @@ -727,7 +789,8 @@ describe('search-index export command', (): void => { type: 'STAGING', settings: {}, keys: {}, - assignedContentTypes: {} + assignedContentTypes: [], + replicas: [] }), new EnrichedSearchIndex({ name: 'account.suffix-2', @@ -736,7 +799,8 @@ describe('search-index export command', (): void => { type: 'STAGING', settings: {}, keys: {}, - assignedContentTypes: {} + assignedContentTypes: [], + replicas: [] }) ]; @@ -776,7 +840,9 @@ describe('search-index export command', (): void => { const argv = { ...yargArgs, ...config, dir: 'my-dir', schemaId: schemaIdsToExport, logFile: new FileLog() }; const filteredIndicesToExport = [...indicesToExport]; - jest.spyOn(exportModule, 'enrichIndex').mockImplementation(x => Promise.resolve(x as EnrichedSearchIndex)); + jest + .spyOn(exportModule, 'enrichIndex') + .mockImplementation((x, y, z) => Promise.resolve(z as EnrichedSearchIndex)); jest.spyOn(exportModule, 'filterIndicesById').mockReturnValue(filteredIndicesToExport); await handler(argv); @@ -801,7 +867,9 @@ describe('search-index export command', (): void => { const argv = { ...yargArgs, ...config, dir: 'my-dir', id: idsToExport, logFile: new FileLog() }; const filteredIndicesToExport = [indicesToExport[1]]; - jest.spyOn(exportModule, 'enrichIndex').mockImplementation(x => Promise.resolve(x as EnrichedSearchIndex)); + jest + .spyOn(exportModule, 'enrichIndex') + .mockImplementation((x, y, z) => Promise.resolve(z as EnrichedSearchIndex)); jest.spyOn(exportModule, 'filterIndicesById').mockReturnValue(filteredIndicesToExport); await handler(argv); diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index dd9835fb..dcdea19a 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -1,10 +1,19 @@ import chalk from 'chalk'; -import { HalResource, Hub, Page, Pageable, SearchIndex, SearchIndexSettings, Sortable } from 'dc-management-sdk-js'; +import { + HalResource, + Hub, + Page, + Pageable, + SearchIndex, + SearchIndexSettings, + Sortable, + Webhook +} from 'dc-management-sdk-js'; import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; import { isEqual } from 'lodash'; import { table } from 'table'; -import { Arguments, Argv } from 'yargs'; +import { Arguments, Argv, string } from 'yargs'; import paginator from '../../common/dc-management-sdk-js/paginator'; import { FileLog } from '../../common/file-log'; import { ensureDirectoryExists } from '../../common/import/directory-utils'; @@ -22,6 +31,7 @@ import { import { loadJsonFromDirectory } from '../../services/import.service'; import { ConfigurationParameters } from '../configure'; import { validateNoDuplicateIndexNames } from './import'; +import { join } from 'path'; export const command = 'export '; @@ -60,9 +70,16 @@ const ensureJSON = (resource: HalResource): object => { return resource.toJSON != null ? resource.toJSON() : resource; }; -const equals = (a: EnrichedSearchIndex, b: EnrichedSearchIndex): boolean => +const replicaEquals = (a: EnrichedReplica, b: EnrichedReplica): boolean => + a.label === b.label && + isEqual(ensureJSON(a.keys), ensureJSON(b.keys)) && + isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); + +export const equals = (a: EnrichedSearchIndex, b: EnrichedSearchIndex): boolean => a.label === b.label && isEqual(a.assignedContentTypes.map(x => ensureJSON(x)), b.assignedContentTypes.map(x => ensureJSON(x))) && + a.replicas.length == b.replicas.length && + a.replicas.map((x, i) => replicaEquals(x, b.replicas[i])).reduce((a, b) => a && b, true) && isEqual(ensureJSON(a.keys), ensureJSON(b.keys)) && isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); @@ -77,10 +94,16 @@ export class EnrichedAssignedContentType extends AssignedContentType { archivedContentWebhook: string; } +export class EnrichedReplica extends SearchIndex { + settings: SearchIndexSettings; + keys: SearchIndexKey; +} + export class EnrichedSearchIndex extends SearchIndex { settings: SearchIndexSettings; keys: SearchIndexKey; assignedContentTypes: EnrichedAssignedContentType[]; + replicas: EnrichedReplica[]; // eslint-disable-next-line @typescript-eslint/no-explicit-any public toJSON(): any { @@ -111,20 +134,46 @@ export const filterIndicesById = ( return listToFilter.filter(index => indexIdList.some(id => index.id === id)); }; -export const enrichIndex = async (index: SearchIndex): Promise => { +export const registerWebhook = (mapping: Map, webhook: Webhook): string => { + mapping.set(webhook.id as string, webhook); + + return webhook.id as string; +}; + +export const enrichReplica = async (replica: SearchIndex): Promise => { + const enrichedReplica = new EnrichedReplica(replica); + + enrichedReplica.settings = await replica.related.settings.get(); + enrichedReplica.keys = await replica.related.keys.get(); + + return enrichedReplica; +}; + +export const enrichIndex = async ( + webhooks: Map, + allReplicas: Map, + index: SearchIndex +): Promise => { const enrichedIndex = new EnrichedSearchIndex(index); enrichedIndex.settings = await index.related.settings.get(); const types = await paginator(index.related.assignedContentTypes.list); enrichedIndex.keys = await index.related.keys.get(); + const replicas = allReplicas.get(index.id as string); + if (replicas) { + enrichedIndex.replicas = await Promise.all(replicas.map(enrichReplica)); + } else { + enrichedIndex.replicas = []; + } + const enrichedTypes: EnrichedAssignedContentType[] = []; for (const type of types) { const enriched = new EnrichedAssignedContentType(type); - enriched.webhook = (await type.related.webhook()).id as string; - enriched.activeContentWebhook = (await type.related.activeContentWebhook()).id as string; - enriched.archivedContentWebhook = (await type.related.archivedContentWebhook()).id as string; + enriched.webhook = registerWebhook(webhooks, await type.related.webhook()); + enriched.activeContentWebhook = registerWebhook(webhooks, await type.related.activeContentWebhook()); + enriched.archivedContentWebhook = registerWebhook(webhooks, await type.related.archivedContentWebhook()); enrichedTypes.push(enriched); } @@ -238,6 +287,70 @@ export const processIndices = async ( log.appendLine(table(data, streamTableOptions)); }; +const filterWebhooks = ( + webhooks: Map, + filteredIndices: EnrichedSearchIndex[] +): Map => { + const filtered = new Map(); + + for (const index of filteredIndices) { + for (const type of index.assignedContentTypes) { + filtered.set(type.webhook, webhooks.get(type.webhook) as Webhook); + filtered.set(type.activeContentWebhook, webhooks.get(type.activeContentWebhook) as Webhook); + filtered.set(type.archivedContentWebhook, webhooks.get(type.archivedContentWebhook) as Webhook); + } + } + + return filtered; +}; + +const processWebhooks = async ( + outputDir: string, + //previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, + webhooksBeingExported: Webhook[], + log: FileLog, + force: boolean +): Promise => { + const previouslyExportedWebhooks: { [filename: string]: Webhook } = {}; + const base = join(outputDir, 'webhooks'); + await ensureDirectoryExists(base); + + const data: string[][] = []; + + data.push([chalk.bold('File'), chalk.bold('Label'), chalk.bold('Result')]); + for (const webhook of webhooksBeingExported) { + const filename = uniqueFilenamePath(base, webhook.label, 'json', Object.keys(previouslyExportedWebhooks)); + delete webhook.id; // do not export id + writeJsonToFile(filename, webhook); + data.push([filename, webhook.label as string, 'UPDATED']); + } + + log.appendLine(table(data, streamTableOptions)); +}; + +export const separateReplicas = ( + allIndices: SearchIndex[] +): { storedIndices: SearchIndex[]; allReplicas: Map } => { + const storedIndices: SearchIndex[] = []; + const allReplicas = new Map(); + for (const index of allIndices) { + if (index.parentId == null) { + storedIndices.push(index); + } else { + let list = allReplicas.get(index.parentId); + + if (list == null) { + list = []; + allReplicas.set(index.parentId, list); + } + + list.push(index); + } + } + + return { storedIndices, allReplicas }; +}; + export const handler = async (argv: Arguments): Promise => { const { dir, id, logFile, force } = argv; const client = dynamicContentClientFactory(argv); @@ -247,12 +360,18 @@ export const handler = async (argv: Arguments(dir, EnrichedSearchIndex); validateNoDuplicateIndexNames(previouslyExportedIndices); - const storedIndices = await paginator(searchIndexList(hub)); - const enrichedIndices = await Promise.all(storedIndices.map(enrichIndex)); + const allStoredIndices = await paginator(searchIndexList(hub)); + const { storedIndices, allReplicas } = separateReplicas(allStoredIndices); + + const webhooks = new Map(); + const enrichedIndices = await Promise.all(storedIndices.map(index => enrichIndex(webhooks, allReplicas, index))); const idArray: string[] = id ? (Array.isArray(id) ? id : [id]) : []; const filteredIndices = filterIndicesById(enrichedIndices, idArray); await processIndices(dir, previouslyExportedIndices, filteredIndices, log, force || false); + const filteredWebhooks = filterWebhooks(webhooks, filteredIndices); + await processWebhooks(dir, Array.from(filteredWebhooks.values()), log, force || false); + await log.close(); }; diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts new file mode 100644 index 00000000..70331ab4 --- /dev/null +++ b/src/commands/search-index/import.spec.ts @@ -0,0 +1,432 @@ +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { SearchIndex, Hub } from 'dc-management-sdk-js'; +import * as importModule from './import'; +import { + builder, + command, + doCreate, + doUpdate, + handler, + LOG_FILENAME, + processIndices, + storedIndexMapper, + validateNoDuplicateIndexNames +} from './import'; +import Yargs from 'yargs/yargs'; +import { table } from 'table'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { loadJsonFromDirectory, UpdateStatus } from '../../services/import.service'; +import chalk from 'chalk'; +import { FileLog } from '../../common/file-log'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { EnrichedSearchIndex } from './export'; +import MockPage from '../../common/dc-management-sdk-js/mock-page'; + +jest.mock('../../services/dynamic-content-client-factory'); +jest.mock('../../view/data-presenter'); +jest.mock('../../services/import.service'); +jest.mock('fs'); +jest.mock('table'); +jest.mock('../../common/log-helpers'); + +describe('search-index import command', (): void => { + afterEach((): void => { + jest.restoreAllMocks(); + }); + + it('should implement an import command', () => { + expect(command).toEqual('import '); + }); + + describe('builder tests', () => { + it('should configure yargs', () => { + const argv = Yargs(process.argv.slice(2)); + const spyPositional = jest.spyOn(argv, 'positional').mockReturnThis(); + const spyOption = jest.spyOn(argv, 'option').mockReturnThis(); + + builder(argv); + + expect(spyPositional).toHaveBeenCalledWith('dir', { + describe: 'Directory containing Search Indices', + type: 'string' + }); + + expect(spyOption).toHaveBeenCalledWith('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); + }); + }); + + describe('storedIndexMapper', () => { + it('it should map to a stored index', () => { + const importedIndex = new EnrichedSearchIndex({ + name: 'matched-name', + label: 'mutated-label' + }); + const storedIndex = [new SearchIndex({ id: 'stored-id', name: 'matched-name', label: 'label' })]; + const result = storedIndexMapper(importedIndex, storedIndex); + + expect(result).toEqual( + expect.objectContaining({ + id: 'stored-id', + name: 'matched-name', + label: 'mutated-label' + }) + ); + }); + + it('should not map to a stored index', () => { + const importedIndex = new EnrichedSearchIndex({ + name: 'not-matched-name', + label: 'mutated-label' + }); + const storedIndex = [new SearchIndex({ id: 'stored-id', name: 'matched-name', label: 'label' })]; + const result = storedIndexMapper(importedIndex, storedIndex); + + expect(result).toEqual(expect.objectContaining({ name: 'not-matched-name', label: 'mutated-label' })); + }); + }); + + describe('doCreate', () => { + it('should create a index and return report', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + const newIndex = new SearchIndex({ id: 'created-id' }); + const mockCreate = jest.fn().mockResolvedValue(newIndex); + mockHub.related.searchIndexes.create = mockCreate; + jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); + const index = { name: 'index-name', label: 'test-label' }; + const result = await doCreate(mockHub, index as EnrichedSearchIndex, log); + + expect(log.getData('CREATE')).toMatchInlineSnapshot(` + Array [ + "created-id", + ] + `); + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining(index)); + expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index); + expect(result).toEqual(newIndex); + }); + + it('should throw an error when index create fails', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); + const mockCreate = jest.fn().mockImplementation(() => { + throw new Error('Error creating index'); + }); + mockHub.related.searchIndexes.create = mockCreate; + const index = { name: 'index-name', label: 'test-label' }; + + await expect(doCreate(mockHub, index as EnrichedSearchIndex, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(importModule.enrichIndex).not.toHaveBeenCalled(); + expect(log.getData('CREATE')).toEqual([]); + }); + + it('should throw an error when index create fails if a string error is returned by the sdk', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); + const mockCreate = jest + .fn() + .mockRejectedValue( + 'The create-index action is not available, ensure you have permission to perform this action.' + ); + mockHub.related.searchIndexes.create = mockCreate; + const index = { name: 'index-name', label: 'test-label' }; + + await expect(doCreate(mockHub, index as EnrichedSearchIndex, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(importModule.enrichIndex).not.toHaveBeenCalled(); + expect(log.getData('CREATE')).toEqual([]); + }); + + it('should throw an error when enrichIndex fails', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + const newIndex = new SearchIndex({ id: 'created-id' }); + const mockCreate = jest.fn().mockResolvedValue(newIndex); + mockHub.related.searchIndexes.create = mockCreate; + jest + .spyOn(importModule, 'enrichIndex') + .mockRejectedValue( + 'The update-index action is not available, ensure you have permission to perform this action.' + ); + const index = { name: 'index-name', label: 'test-label' }; + + await expect(doCreate(mockHub, index as EnrichedSearchIndex, log)).rejects.toThrowErrorMatchingSnapshot(); + + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining(index)); + expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index); + expect(log.getData('CREATE')).toEqual([]); + }); + }); + + describe('processIndices', () => { + let mockTable: jest.Mock; + + beforeEach(() => { + mockTable = table as jest.Mock; + mockTable.mockImplementation(jest.requireActual('table').table); + }); + + it('should create and update a index', async () => { + const client = (dynamicContentClientFactory as jest.Mock)(); + const hub = new Hub(); + const indicesToProcess = [ + new EnrichedSearchIndex({ + name: 'index-name', + label: 'created', + assignedContentTypes: [] + }), + new EnrichedSearchIndex({ + id: 'updated-id', + name: 'index-name-2', + label: 'updated' + }), + new EnrichedSearchIndex({ + id: 'up-to-date-id', + name: 'index-name-3', + label: 'up-to date' + }) + ]; + + const createdIndex = new EnrichedSearchIndex({ + id: 'created-id', + ...indicesToProcess[0].toJSON() + }); + jest.spyOn(importModule, 'doCreate').mockResolvedValueOnce(createdIndex); + const doUpdateResult1 = { + index: indicesToProcess[1], + updateStatus: UpdateStatus.UPDATED + }; + jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult1); + const doUpdateResult2 = { + index: indicesToProcess[2], + updateStatus: UpdateStatus.SKIPPED + }; + jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult2); + + await processIndices(indicesToProcess, new Map(), client, hub, new FileLog()); + + expect(importModule.doCreate).toHaveBeenCalledWith(hub, indicesToProcess[0], expect.any(FileLog)); + expect(importModule.doUpdate).toHaveBeenCalledWith( + hub, + expect.any(Map), + indicesToProcess[1], + expect.any(FileLog) + ); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('ID'), chalk.bold('Name'), chalk.bold('Result')], + [createdIndex.id, createdIndex.name, 'CREATED'], + [doUpdateResult1.index.id, doUpdateResult1.index.name, 'UPDATED'], + [doUpdateResult2.index.id, doUpdateResult2.index.name, 'UP-TO-DATE'] + ], + streamTableOptions + ); + }); + }); + + describe('validateNoDuplicateIndexNames', function() { + it('should not throw an error when there are no duplicates', () => { + const indicesToProcess = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + name: 'index-name-2' + }) + }; + + expect(() => validateNoDuplicateIndexNames(indicesToProcess)).not.toThrow(); + }); + + it('should throw and error when there are duplicate uris', () => { + const indicesToProcess = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + name: 'index-name-2' + }), + 'file-3': new EnrichedSearchIndex({ + name: 'index-name-2' + }), + 'file-4': new EnrichedSearchIndex({ + name: 'index-name-1' + }) + }; + + expect(() => validateNoDuplicateIndexNames(indicesToProcess)).toThrowErrorMatchingSnapshot(); + }); + }); + + describe('filterIndicesById', function() { + it('should delete indices without a matching id', () => { + const indicesToProcess = { + 'file-1': new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2' + }) + }; + + const expectedResult = { 'file-2': indicesToProcess['file-2'] }; + + importModule.filterIndicesById(['index-id-2'], indicesToProcess); + + expect(indicesToProcess).toEqual(expectedResult); + }); + + it('should remove all indices if no ids are given', () => { + const indicesToProcess = { + 'file-1': new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2' + }) + }; + + importModule.filterIndicesById([], indicesToProcess); + + expect(indicesToProcess).toEqual({}); + }); + }); + + describe('handler tests', () => { + const yargArgs = { + $0: 'test', + _: ['test'] + }; + const config = { + clientId: 'client-id', + clientSecret: 'client-id', + hubId: 'hub-id' + }; + + const mockGetHub = jest.fn(); + const mockList = jest.fn(); + + beforeEach(() => { + (dynamicContentClientFactory as jest.Mock).mockReturnValue({ + hubs: { + get: mockGetHub + } + }); + + mockGetHub.mockResolvedValue({ + id: 'hub-id', + related: { + searchIndexes: { + list: mockList + } + } + }); + + mockList.mockImplementation(() => { + return Promise.resolve( + new MockPage(SearchIndex, [ + new SearchIndex({ + id: 'id', + label: 'label', + name: 'stored-index' + }) + ]) + ); + }); + }); + + it('should use getDefaultLogPath for LOG_FILENAME with process.platform as default', function() { + LOG_FILENAME(); + + expect(getDefaultLogPath).toHaveBeenCalledWith('search-index', 'import', process.platform); + }); + + it('should create a index and update', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; + const fileNamesAndIndicesToImport = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'created' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'content-index-id', + name: 'index-name-2', + label: 'updated' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndicesToImport); + jest + .spyOn(importModule, 'storedIndexMapper') + .mockReturnValueOnce(fileNamesAndIndicesToImport['file-1']) + .mockReturnValueOnce(fileNamesAndIndicesToImport['file-2']); + jest.spyOn(importModule, 'processIndices').mockResolvedValueOnce(); + + await handler(argv); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalled(); + expect(processIndices).toHaveBeenCalledWith( + Object.values(fileNamesAndIndicesToImport), + expect.any(Map), + expect.any(Object), + expect.any(Object), + expect.any(FileLog) + ); + }); + + it('should call filterIndicesById when a list of ids is provided', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; + const fileNamesAndIndicesToImport = { + 'file-1': new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1', + label: 'created' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'updated' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue({ ...fileNamesAndIndicesToImport }); + jest.spyOn(importModule, 'storedIndexMapper').mockReturnValueOnce(fileNamesAndIndicesToImport['file-2']); + jest.spyOn(importModule, 'processIndices').mockResolvedValueOnce(); + + await handler(argv, ['index-id-2']); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(processIndices).toHaveBeenCalledWith( + [fileNamesAndIndicesToImport['file-2']], + expect.any(Map), + expect.any(Object), + expect.any(Object), + expect.any(FileLog) + ); + }); + + it('should throw an error when no content found in import directory', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-empty-dir', logFile: new FileLog() }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue([]); + + await expect(handler(argv)).rejects.toThrowErrorMatchingSnapshot(); + }); + }); +}); diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index 73d8bc44..4e785c27 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -1,4 +1,47 @@ -import { EnrichedSearchIndex } from './export'; +import chalk from 'chalk'; +import { DynamicContent, Hub, Page, Pageable, SearchIndex, SearchIndexSettings, Sortable } from 'dc-management-sdk-js'; +import { table } from 'table'; +import { Arguments, Argv } from 'yargs'; +import paginator from '../../common/dc-management-sdk-js/paginator'; +import { FileLog } from '../../common/file-log'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { ImportBuilderOptions } from '../../interfaces/import-builder-options.interface'; +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { ImportResult, loadJsonFromDirectory, UpdateStatus } from '../../services/import.service'; +import { ConfigurationParameters } from '../configure'; +import { EnrichedSearchIndex, equals, enrichIndex as enrichServerIndex, separateReplicas } from './export'; + +export const command = 'import '; + +export const desc = 'Import Search Index'; + +export const LOG_FILENAME = (platform: string = process.platform): string => + getDefaultLogPath('search-index', 'import', platform); + +export const builder = (yargs: Argv): void => { + yargs.positional('dir', { + describe: 'Directory containing Search Indices', + type: 'string' + }); + + yargs.option('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); +}; + +const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { + return (options?: Pageable & Sortable): Promise> => + hub.related.searchIndexes.list(parentId, projection, options); +}; + +const replicaList = (index: SearchIndex, projection?: string) => { + return (options?: Pageable & Sortable): Promise> => + index.related.replicas.list(projection, options); +}; type IndexName = string; type IndexFile = string; @@ -30,3 +73,177 @@ export const validateNoDuplicateIndexNames = (importedIndices: { ); } }; + +export const filterIndicesById = ( + idFilter: string[], + importedIndices: { + [filename: string]: SearchIndex; + } +): void | never => { + for (const [filename, index] of Object.entries(importedIndices)) { + if (idFilter.indexOf(index.id as string) === -1) { + delete importedIndices[filename]; + } + } +}; + +export const storedIndexMapper = (index: EnrichedSearchIndex, storedIndices: SearchIndex[]): EnrichedSearchIndex => { + const found = storedIndices.find(stored => stored.name === index.name); + const mutatedIndex = found ? { ...index, id: found.id } : index; + + return new EnrichedSearchIndex(mutatedIndex); +}; + +const getIndexProperties = (index: SearchIndex): object => { + return { + label: index.label, + name: index.name, + suffix: index.suffix, + type: index.type + }; +}; + +export const enrichIndex = async (index: SearchIndex, enrichedIndex: EnrichedSearchIndex): Promise => { + await index.related.settings.update(enrichedIndex.settings, false); + + if (enrichedIndex.settings.replicas && enrichedIndex.settings.replicas.length) { + // Replica settings must also be updated. The replicas may have changed, so fetch them again. + + const replicas = await paginator(replicaList(index)); + + for (const importReplica of enrichedIndex.replicas) { + let replica = replicas.find(replica => replica.name === importReplica.name); + + if (replica) { + replica = await replica.related.update(new SearchIndex(getIndexProperties(importReplica))); + + replica.related.settings.update(importReplica.settings, false); + } + } + } + + const types = await paginator(index.related.assignedContentTypes.list); + + // Assign any content types that are not assigned. + + const unassigned = new Set(types); + + for (const assignment of enrichedIndex.assignedContentTypes) { + let existing = types.find(type => type.contentTypeUri === assignment.contentTypeUri); + + if (!existing) { + // Need to create a new assignment + existing = await index.related.assignedContentTypes.create(assignment); + } + + unassigned.delete(existing); + + // Update any webhooks if they differ from the ones being imported, if the flag is provided. + // Does the webhook being referenced in the saved index exist in the import? + + //const webhooks = new Map(); + } + + // Finally, remove any content type assignments that are not present in the imported index. + for (const toRemove of unassigned) { + await toRemove.related.unassign(index.id as string); + } +}; + +export const doCreate = async (hub: Hub, index: EnrichedSearchIndex, log: FileLog): Promise => { + try { + const createdIndex = await hub.related.searchIndexes.create(new SearchIndex(getIndexProperties(index))); + + await enrichIndex(createdIndex, index); + + log.addAction('CREATE', `${createdIndex.id}`); + + return createdIndex; + } catch (err) { + throw new Error(`Error creating index ${index.name}:\n\n${err}`); + } +}; + +export const doUpdate = async ( + hub: Hub, + allReplicas: Map, + index: EnrichedSearchIndex, + log: FileLog +): Promise<{ index: SearchIndex; updateStatus: UpdateStatus }> => { + try { + const retrievedIndex: SearchIndex = await hub.related.searchIndexes.get(index.id || ''); + const webhooks = new Map(); + + if (equals(await enrichServerIndex(webhooks, allReplicas, retrievedIndex), index)) { + return { index: retrievedIndex, updateStatus: UpdateStatus.SKIPPED }; + } + + Object.assign(retrievedIndex, getIndexProperties(index)); + + const updatedIndex = await retrievedIndex.related.update(retrievedIndex); + + await enrichIndex(updatedIndex, index); + + log.addAction('UPDATE', `${retrievedIndex.id}`); + + return { index: updatedIndex, updateStatus: UpdateStatus.UPDATED }; + } catch (err) { + throw new Error(`Error updating index ${index.name}: ${err.message}`); + } +}; + +export const processIndices = async ( + indicesToProcess: EnrichedSearchIndex[], + allReplicas: Map, + client: DynamicContent, + hub: Hub, + log: FileLog +): Promise => { + const data: string[][] = []; + + data.push([chalk.bold('ID'), chalk.bold('Name'), chalk.bold('Result')]); + for (const schema of indicesToProcess) { + let status: ImportResult; + let index: SearchIndex; + if (schema.id) { + const result = await doUpdate(hub, allReplicas, schema, log); + index = result.index; + status = result.updateStatus === UpdateStatus.SKIPPED ? 'UP-TO-DATE' : 'UPDATED'; + } else { + index = await doCreate(hub, schema, log); + status = 'CREATED'; + } + data.push([index.id || '', index.name as string, status]); + } + + log.appendLine(table(data, streamTableOptions)); +}; + +export const handler = async ( + argv: Arguments, + idFilter?: string[] +): Promise => { + const { dir, logFile } = argv; + const client = dynamicContentClientFactory(argv); + const hub = await client.hubs.get(argv.hubId); + const log = logFile.open(); + const indices = loadJsonFromDirectory(dir, EnrichedSearchIndex); + if (Object.keys(indices).length === 0) { + throw new Error(`No indices found in ${dir}`); + } + + validateNoDuplicateIndexNames(indices); + + if (idFilter) { + filterIndicesById(idFilter, indices); + } + + const allStoredIndices = await paginator(searchIndexList(hub)); + const { storedIndices, allReplicas } = separateReplicas(allStoredIndices); + + const indicesToProcess = Object.values(indices).map(index => storedIndexMapper(index, storedIndices)); + + await processIndices(indicesToProcess, allReplicas, client, hub, log); + + await log.close(); +}; diff --git a/src/commands/search-index/webhook-rewriter.spec.ts b/src/commands/search-index/webhook-rewriter.spec.ts new file mode 100644 index 00000000..57b60b9a --- /dev/null +++ b/src/commands/search-index/webhook-rewriter.spec.ts @@ -0,0 +1,81 @@ +import { rewriteDeliveryContentItem } from './webhook-rewriter'; + +describe('webhook-rewriter tests', function() { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should not rewrite an empty string', async () => { + expect(rewriteDeliveryContentItem('', 'account', 'staging')).toEqual(''); + }); + + it('should not rewrite a simple string', async () => { + expect(rewriteDeliveryContentItem('basic', 'account', 'staging')).toEqual('basic'); + }); + + it('should not rewrite any unrelated tag', async () => { + expect(rewriteDeliveryContentItem('{{#unrelated}}{{/unrelated}}', 'account', 'staging')).toEqual( + '{{#unrelated}}{{/unrelated}}' + ); + }); + + it('should rewrite a simple webhook', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplaced" stagingEnvironment="old"}} + "content" : "{{~#each (first (pluck contentBlocks "content") 4)~}}{{~#each this.values~}}{{#if (eq locale "en-GB")}}{{{truncate value 1000}}}{{/if}}{{~/each~}}{{#unless @last}};{{/unless~}}{{~/each~}}" +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', 'staging')).toMatchSnapshot(); + }); + + it('should rewrite multiple tags within the same webhook', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplaced" stagingEnvironment="old"}} + {{example1}} +{{/withDeliveryContentItem}} + "key": "value", +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplaced2" stagingEnvironment="old2"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', 'staging')).toMatchSnapshot(); + }); + + it('should rewrite only one argument if the other is not present', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplacednostaging"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', "doesn't appear")).toMatchSnapshot(); + }); + + it('should not rewrite a tag with no matching arguments', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', "doesn't appear")).toEqual(example); + }); + + it('should escape quotes and backslashes', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="1" stagingEnvironment="2"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, '"quotedString\\with\\backslash"', 'back\\slash')).toMatchSnapshot(); + }); + it('should still replace values with unusual whitespace', async () => { + const example = `{ +{{#withDeliveryContentItem + contentItemId=payload.id + account="1" + stagingEnvironment="another"\t }} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'replaced', 'whitespace')).toMatchSnapshot(); + }); +}); diff --git a/src/commands/search-index/webhook-rewriter.ts b/src/commands/search-index/webhook-rewriter.ts new file mode 100644 index 00000000..ed567c33 --- /dev/null +++ b/src/commands/search-index/webhook-rewriter.ts @@ -0,0 +1,63 @@ +function matchAll(regex: RegExp, string: string): RegExpExecArray[] { + const result: RegExpExecArray[] = []; + let match: RegExpExecArray | null; + + while ((match = regex.exec(string)) != null) { + result.push(match); + } + + return result; +} + +export function rewriteDeliveryContentItem(webhookBody: string, account: string, stagingEnvironment: string): string { + // Current limitations - cannot handle key/value pairs where the value contains space, or a }} within quotes. + // These should not affect the two variables being replaced anyways. + // First, locate the withDeliveryContentItem opening tags. + + const tagRegexG = /\{\{\#withDeliveryContentItem(?\s+)(?.+\=.+(\s+.+\=.+)*\s*)\}\}/g; + const keyValueRegex = /(?.+?)\=(?.+?)\s+/g; + + const tagMatches = matchAll(tagRegexG, webhookBody); + + let replaceOffset = 0; + + for (const tag of tagMatches) { + const tGroups = tag.groups as { [key: string]: string }; + + const matchIndex = tag.index; + const body = tGroups.body + ' '; + const bodyIndex = matchIndex + '{{#withDeliveryContentItem'.length + tGroups.whitespace.length; + + const keyValueMatches = matchAll(keyValueRegex, body); + + for (const pair of keyValueMatches) { + const pGroups = pair.groups as { [key: string]: string }; + + const key = pGroups.key; + const keyIndex = bodyIndex + pair.index + replaceOffset; + + const value = pGroups.value; + const valueIndex = keyIndex + key.length + 1; + + let replaceValue: string | null = null; + + switch (key) { + case 'account': + replaceValue = account; + break; + case 'stagingEnvironment': + replaceValue = stagingEnvironment; + break; + } + + if (replaceValue != null) { + replaceValue = `"${replaceValue.replace(/\\/g, '\\\\').replace(/"/g, '"')}"`; + webhookBody = webhookBody.substr(0, valueIndex) + replaceValue + webhookBody.substr(valueIndex + value.length); + + replaceOffset += replaceValue.length - value.length; + } + } + } + + return webhookBody; +} From 38c45a2c5284a0e1d247b5a5a087359d7c3ed904 Mon Sep 17 00:00:00 2001 From: Rhys Date: Fri, 30 Jul 2021 14:19:23 +0100 Subject: [PATCH 03/12] fix(search-index): ignore keys when checking equality for import --- src/commands/search-index/export.ts | 12 ++++++------ src/commands/search-index/import.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index dcdea19a..2828f0f6 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -13,7 +13,7 @@ import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/A import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; import { isEqual } from 'lodash'; import { table } from 'table'; -import { Arguments, Argv, string } from 'yargs'; +import { Arguments, Argv } from 'yargs'; import paginator from '../../common/dc-management-sdk-js/paginator'; import { FileLog } from '../../common/file-log'; import { ensureDirectoryExists } from '../../common/import/directory-utils'; @@ -70,17 +70,17 @@ const ensureJSON = (resource: HalResource): object => { return resource.toJSON != null ? resource.toJSON() : resource; }; -const replicaEquals = (a: EnrichedReplica, b: EnrichedReplica): boolean => +const replicaEquals = (a: EnrichedReplica, b: EnrichedReplica, keys: boolean): boolean => a.label === b.label && - isEqual(ensureJSON(a.keys), ensureJSON(b.keys)) && + (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); -export const equals = (a: EnrichedSearchIndex, b: EnrichedSearchIndex): boolean => +export const equals = (a: EnrichedSearchIndex, b: EnrichedSearchIndex, keys = true): boolean => a.label === b.label && isEqual(a.assignedContentTypes.map(x => ensureJSON(x)), b.assignedContentTypes.map(x => ensureJSON(x))) && a.replicas.length == b.replicas.length && - a.replicas.map((x, i) => replicaEquals(x, b.replicas[i])).reduce((a, b) => a && b, true) && - isEqual(ensureJSON(a.keys), ensureJSON(b.keys)) && + a.replicas.map((x, i) => replicaEquals(x, b.replicas[i], keys)).reduce((a, b) => a && b, true) && + (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index 4e785c27..fe618d53 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -174,7 +174,7 @@ export const doUpdate = async ( const retrievedIndex: SearchIndex = await hub.related.searchIndexes.get(index.id || ''); const webhooks = new Map(); - if (equals(await enrichServerIndex(webhooks, allReplicas, retrievedIndex), index)) { + if (equals(await enrichServerIndex(webhooks, allReplicas, retrievedIndex), index, false)) { return { index: retrievedIndex, updateStatus: UpdateStatus.SKIPPED }; } From 1a7d71754320717408f3acca7b4d2677795c5a7b Mon Sep 17 00:00:00 2001 From: Rhys Date: Tue, 10 Aug 2021 09:17:36 +0100 Subject: [PATCH 04/12] feat(search-index): add exporting and importing of webhooks, use with --webhooks argument on import --- .../__snapshots__/export.spec.ts.snap | 18 +- .../__snapshots__/import.spec.ts.snap | 6 + src/commands/search-index/export.spec.ts | 408 +++++++++++++++++- src/commands/search-index/export.ts | 123 +++++- src/commands/search-index/import.spec.ts | 399 ++++++++++++++++- src/commands/search-index/import.ts | 105 ++++- src/commands/search-index/webhook-rewriter.ts | 10 +- .../import-index-builder-options.interface.ts | 5 + 8 files changed, 989 insertions(+), 85 deletions(-) create mode 100644 src/interfaces/import-index-builder-options.interface.ts diff --git a/src/commands/search-index/__snapshots__/export.spec.ts.snap b/src/commands/search-index/__snapshots__/export.spec.ts.snap index 3f99f49a..36e3e4e4 100644 --- a/src/commands/search-index/__snapshots__/export.spec.ts.snap +++ b/src/commands/search-index/__snapshots__/export.spec.ts.snap @@ -5,7 +5,7 @@ exports[`search-index export command filterIndicesById should throw an error for Nothing was exported, exiting." `; -exports[`search-index export command getExports should return a list of indices to export and a list of filenames that will be updated 1`] = ` +exports[`search-index export command getIndexExports should return a list of indices to export and a list of filenames that will be updated 1`] = ` Array [ Array [ Object { @@ -38,6 +38,8 @@ Array [ "name": "index-name-2", }, }, + Map {}, + Map {}, ], Array [ Object { @@ -70,11 +72,13 @@ Array [ "name": "index-name-2", }, }, + Map {}, + Map {}, ], ] `; -exports[`search-index export command getExports should return a list of indices to export and a list of filenames that will be updated 2`] = ` +exports[`search-index export command getIndexExports should return a list of indices to export and a list of filenames that will be updated 2`] = ` Array [ Object { "filename": "export-dir/export-filename-1.json", @@ -105,7 +109,7 @@ Array [ ] `; -exports[`search-index export command getExports should return a list of indices to export and a list of filenames that will be updated 3`] = ` +exports[`search-index export command getIndexExports should return a list of indices to export and a list of filenames that will be updated 3`] = ` Array [ Object { "filename": "export-dir/export-filename-2.json", @@ -114,7 +118,7 @@ Array [ ] `; -exports[`search-index export command getExports should return a list of indices to export and no filenames that will be updated (first export) 1`] = ` +exports[`search-index export command getIndexExports should return a list of indices to export and no filenames that will be updated (first export) 1`] = ` Array [ Array [ Object { @@ -128,6 +132,8 @@ Array [ }, "export-dir", Object {}, + Map {}, + Map {}, ], Array [ Object { @@ -141,11 +147,13 @@ Array [ }, "export-dir", Object {}, + Map {}, + Map {}, ], ] `; -exports[`search-index export command getExports should return a list of indices to export and no filenames that will be updated (first export) 2`] = ` +exports[`search-index export command getIndexExports should return a list of indices to export and no filenames that will be updated (first export) 2`] = ` Array [ Object { "filename": "export-dir/export-filename-1.json", diff --git a/src/commands/search-index/__snapshots__/import.spec.ts.snap b/src/commands/search-index/__snapshots__/import.spec.ts.snap index 0ba373ac..8a7b9d4e 100644 --- a/src/commands/search-index/__snapshots__/import.spec.ts.snap +++ b/src/commands/search-index/__snapshots__/import.spec.ts.snap @@ -18,6 +18,12 @@ exports[`search-index import command doCreate should throw an error when index c The create-index action is not available, ensure you have permission to perform this action." `; +exports[`search-index import command doUpdate should throw an error when unable to get index during update 1`] = `"Error updating index matched-name: Error retrieving index"`; + +exports[`search-index import command doUpdate should throw an error when unable to update index during update 1`] = `"Error updating index not-matched-name: Error saving index"`; + +exports[`search-index import command doUpdate should throw an error when unable to update index during update if a string error is returned by sdk 1`] = `"Error updating index not-matched-name: undefined"`; + exports[`search-index import command handler tests should throw an error when no content found in import directory 1`] = `"No indices found in my-empty-dir"`; exports[`search-index import command validateNoDuplicateIndexNames should throw and error when there are duplicate uris 1`] = ` diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts index 375bf7fa..2bb014cd 100644 --- a/src/commands/search-index/export.spec.ts +++ b/src/commands/search-index/export.spec.ts @@ -10,7 +10,13 @@ import { LOG_FILENAME, processIndices, EnrichedSearchIndex, - EnrichedAssignedContentType + EnrichedAssignedContentType, + webhookEquals, + replicaEquals, + EnrichedReplica, + getExportedWebhooks, + processWebhooks, + filterWebhooks } from './export'; import Yargs from 'yargs/yargs'; import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; @@ -79,6 +85,164 @@ describe('search-index export command', (): void => { }); }); + describe('webhookEquals', () => { + it('should match undefined webhooks, and return false if only one argument is undefined', async () => { + expect(webhookEquals(undefined, undefined)).toBeTruthy(); + expect(webhookEquals(new Webhook(), undefined)).toBeFalsy(); + expect(webhookEquals(undefined, new Webhook())).toBeFalsy(); + }); + + it('should compare webhooks on all parameters', async () => { + const exampleWebhook = { + method: 'GET', + secret: 'bananabread', + label: 'webhook', + active: true, + customPayload: { type: 'example', value: 'text' }, + events: ['event1'], + filters: [{ type: 'equal', arguments: [{ jsonPath: 'path' }] }], + handlers: ['handler1'], + headers: [{ key: 'key', value: 'value' }] + }; + + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, unrelatedProperty: true })) + ).toBeTruthy(); + + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, method: 'POST' })) + ).toBeFalsy(); + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, secret: 'applepie' })) + ).toBeFalsy(); + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, label: 'webhook2' })) + ).toBeFalsy(); + expect(webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, active: false }))).toBeFalsy(); + expect( + webhookEquals( + new Webhook(exampleWebhook), + new Webhook({ ...exampleWebhook, customPayload: { type: 'example', value: 'text2' } }) + ) + ).toBeFalsy(); + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, events: ['event1', 'event2'] })) + ).toBeFalsy(); + expect( + webhookEquals( + new Webhook(exampleWebhook), + new Webhook({ ...exampleWebhook, filters: [{ type: 'in', arguments: [{ jsonPath: 'path' }] }] }) + ) + ).toBeFalsy(); + expect(webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, handlers: [] }))).toBeFalsy(); + expect( + webhookEquals( + new Webhook(exampleWebhook), + new Webhook({ ...exampleWebhook, headers: [{ key: 'key', value: 'value' }, { key: 'key', value: 'value2' }] }) + ) + ).toBeFalsy(); + }); + }); + + describe('replicaEquals', () => { + it('should compare replicas on all settings and label', async () => { + const exampleReplica = { + label: 'replicaIndex', + settings: { example: 'object', example2: 'a' } + }; + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, unrelatedProperty: true }), + false + ) + ).toBeTruthy(); + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, label: 'different' }), + false + ) + ).toBeFalsy(); + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, settings: { example: 'object', example2: 'b' } }), + false + ) + ).toBeFalsy(); + }); + + it('should only compare keys when keys argument is true', async () => { + const exampleReplica = { + label: 'replicaIndex', + keys: { key: 'expected' }, + settings: { example: 'object', example2: 'a' } + }; + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, keys: { key: 'unexpected' } }), + false + ) + ).toBeTruthy(); + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, keys: { key: 'unexpected' } }), + true + ) + ).toBeFalsy(); + }); + }); + + describe('enrichReplica tests', () => { + it('should request settings and keys to enrich the given indices', async (): Promise => { + const index = new SearchIndex({ + name: 'account.suffix-1', + suffix: 'suffix-1', + label: 'Index 1', + type: 'STAGING' + }); + + const settings = new SearchIndexSettings({ + example: 'setting' + }); + + const key = new SearchIndexKey({ + id: 'key-id', + key: 'example-key' + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (index as any).related = { + settings: { + get: jest.fn().mockResolvedValue(settings) + }, + keys: { + get: jest.fn().mockResolvedValue(key) + } + }; + + const expectedEnriched = new EnrichedReplica({ + ...index.toJSON(), + settings: settings, + keys: key + }); + + const enriched = await exportModule.enrichReplica(index); + + expect(index.related.settings.get).toHaveBeenCalledTimes(1); + expect(index.related.keys.get).toHaveBeenCalledTimes(1); + + expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); + }); + }); + describe('enrichIndex tests', () => { it('should request settings, types, keys and webhooks to enrich the given indices', async (): Promise => { const index = new SearchIndex({ @@ -156,7 +320,51 @@ describe('search-index export command', (): void => { }); }); - describe('getExports', () => { + describe('getExportedWebhooks', () => { + it('should create an id to webhook map from the webhooks loaded from the given directory', () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1' + }), + 'directory/webhooks/webhook2.json': new Webhook({ + id: 'id2', + label: 'webhook2' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + + const result = getExportedWebhooks('directory'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(2); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(result.get('id2')).toEqual(webhooks['directory/webhooks/webhook2.json']); + }); + + it('should ignore webhooks without an id', () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1' + }), + 'directory/webhooks/webhook2.json': new Webhook({ + label: 'webhook2' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + + const result = getExportedWebhooks('directory'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(1); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + }); + }); + + describe('getIndexExports', () => { let getExportRecordForIndexSpy: jest.SpyInstance; const indicesToExport = [ @@ -190,6 +398,9 @@ describe('search-index export command', (): void => { }); it('should return a list of indices to export and no filenames that will be updated (first export)', () => { + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + getExportRecordForIndexSpy .mockReturnValueOnce({ filename: 'export-dir/export-filename-1.json', @@ -202,7 +413,7 @@ describe('search-index export command', (): void => { index: indicesToExport[1] }); - const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, indicesToExport); + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, indicesToExport, new Map()); expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); @@ -211,6 +422,9 @@ describe('search-index export command', (): void => { }); it('should return a list of indices to export and a list of filenames that will be updated', () => { + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + getExportRecordForIndexSpy .mockReturnValueOnce({ filename: 'export-dir/export-filename-1.json', @@ -223,7 +437,12 @@ describe('search-index export command', (): void => { index: indicesToExport[1] }); - const [allExports, updatedExportsMap] = getIndexExports('export-dir', exportedIndices, indicesToExport); + const [allExports, updatedExportsMap] = getIndexExports( + 'export-dir', + exportedIndices, + indicesToExport, + new Map() + ); expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); @@ -232,7 +451,9 @@ describe('search-index export command', (): void => { }); it('should not return a list of indices to export or a list of filenames that will be updated', () => { - const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, []); + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, [], new Map()); expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(0); expect(allExports).toEqual([]); @@ -240,11 +461,19 @@ describe('search-index export command', (): void => { }); it('should skip any that are missing a name', () => { - const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, [ - new EnrichedSearchIndex({ - label: 'index 1' - }) - ]); + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + + const [allExports, updatedExportsMap] = getIndexExports( + 'export-dir', + {}, + [ + new EnrichedSearchIndex({ + label: 'index 1' + }) + ], + new Map() + ); expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(0); expect(allExports).toEqual([]); @@ -291,7 +520,7 @@ describe('search-index export command', (): void => { const existingIndices = Object.keys(exportedIndices); - const result = getExportRecordForIndex(newIndexToExport, 'export-dir', exportedIndices); + const result = getExportRecordForIndex(newIndexToExport, 'export-dir', exportedIndices, new Map(), new Map()); expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledWith( 'export-dir', @@ -328,7 +557,7 @@ describe('search-index export command', (): void => { jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); - const result = getExportRecordForIndex(updatedIndexToExport, 'export-dir', exportedIndices); + const result = getExportRecordForIndex(updatedIndexToExport, 'export-dir', exportedIndices, new Map(), new Map()); expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); expect(result).toEqual({ @@ -360,7 +589,13 @@ describe('search-index export command', (): void => { jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); - const result = getExportRecordForIndex(unchangedIndexToExport, 'export-dir', exportedIndices); + const result = getExportRecordForIndex( + unchangedIndexToExport, + 'export-dir', + exportedIndices, + new Map(), + new Map() + ); expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); expect(result).toEqual({ @@ -484,13 +719,15 @@ describe('search-index export command', (): void => { ]); const previouslyExportedIndices = {}; - await processIndices('export-dir', previouslyExportedIndices, indicesToProcess, new FileLog(), false); + const webhooks = new Map(); + await processIndices('export-dir', previouslyExportedIndices, indicesToProcess, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', previouslyExportedIndices, - indicesToProcess + indicesToProcess, + webhooks ); expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); @@ -530,7 +767,7 @@ describe('search-index export command', (): void => { const previouslyExportedIndices = {}; - await processIndices('export-dir', previouslyExportedIndices, [], new FileLog(), false); + await processIndices('export-dir', previouslyExportedIndices, [], new Map(), new FileLog(), false); expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(0); @@ -564,13 +801,16 @@ describe('search-index export command', (): void => { const previouslyExportedIndices = { 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) }; - await processIndices('export-dir', previouslyExportedIndices, indicesToProcess, new FileLog(), false); + const webhooks = new Map(); + const indices = [...indicesToProcess]; + await processIndices('export-dir', previouslyExportedIndices, indices, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', previouslyExportedIndices, - indicesToProcess + indices, + webhooks ); expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); @@ -629,14 +869,16 @@ describe('search-index export command', (): void => { const previouslyExportedIndices = { 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) }; + const webhooks = new Map(); - await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, new FileLog(), false); + await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', previouslyExportedIndices, - mutatedIndices + mutatedIndices, + webhooks ); expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); @@ -694,14 +936,16 @@ describe('search-index export command', (): void => { const previouslyExportedIndices = { 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) }; + const webhooks = new Map(); - await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, new FileLog(), false); + await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', previouslyExportedIndices, - mutatedIndices + mutatedIndices, + webhooks ); expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); @@ -710,6 +954,124 @@ describe('search-index export command', (): void => { }); }); + describe('filterWebhooks', () => { + it('should return a webhook mapping consisting of only webhooks in the given search indices', () => { + const webhooks = new Map([ + ['id1', new Webhook({ id: 'id1', label: 'webhook1' })], + ['id2', new Webhook({ id: 'id2', label: 'webhook2' })], + ['id3', new Webhook({ id: 'id3', label: 'webhook3' })], + ['id4', new Webhook({ id: 'id4', label: 'webhook4' })] + ]); + + const indices = [ + new EnrichedSearchIndex({ + assignedContentTypes: [ + new EnrichedAssignedContentType({ + webhook: 'id1', + activeContentWebhook: 'id2', + archivedContentWebhook: 'id4' + }) + ] + }), + new EnrichedSearchIndex({ + assignedContentTypes: [ + new EnrichedAssignedContentType({ + webhook: 'id4', + activeContentWebhook: 'id1', + archivedContentWebhook: 'id2' + }) + ] + }) + ]; + + const result = filterWebhooks(webhooks, indices); + + expect(result.size).toEqual(3); + expect(result.get('id1')).toEqual(webhooks.get('id1')); + expect(result.get('id2')).toEqual(webhooks.get('id2')); + expect(result.get('id4')).toEqual(webhooks.get('id4')); + }); + + it('should filter all webhooks if no indices are provided', () => { + const webhooks = new Map([ + ['id1', new Webhook({ id: 'id1', label: 'webhook1' })], + ['id2', new Webhook({ id: 'id2', label: 'webhook2' })], + ['id3', new Webhook({ id: 'id3', label: 'webhook3' })], + ['id4', new Webhook({ id: 'id4', label: 'webhook4' })] + ]); + + const indices: EnrichedSearchIndex[] = []; + const result = filterWebhooks(webhooks, indices); + + expect(result.size).toEqual(0); + }); + }); + + describe('processWebhooks', () => { + let mockEnsureDirectory: jest.Mock; + let mockTable: jest.Mock; + let stdoutSpy: jest.SpyInstance; + + beforeEach(() => { + mockEnsureDirectory = directoryUtils.ensureDirectoryExists as jest.Mock; + mockTable = table as jest.Mock; + mockTable.mockImplementation(jest.requireActual('table').table); + jest.spyOn(exportServiceModule, 'writeJsonToFile').mockImplementation(); + stdoutSpy = jest.spyOn(process.stdout, 'write'); + stdoutSpy.mockImplementation(); + }); + + it('should export webhooks, printing a table of all exported files', async () => { + const webhooks = [ + new Webhook({ + id: 'id1', + label: 'webhook1' + }), + new Webhook({ + id: 'id2', + label: 'webhook2' + }), + new Webhook({ + id: 'id3', + label: 'webhook2' + }) + ]; + + await processWebhooks('export-dir', webhooks, new FileLog(), false); + + expect(mockEnsureDirectory).toHaveBeenCalledWith('export-dir/webhooks'); + + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(3); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 1, + 'export-dir/webhooks/webhook1.json', + webhooks[0] + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 2, + 'export-dir/webhooks/webhook2.json', + webhooks[1] + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 3, + 'export-dir/webhooks/webhook2-1.json', + webhooks[2] + ); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Label'), chalk.bold('Result')], + ['export-dir/webhooks/webhook1.json', webhooks[0].label, 'UPDATED'], + ['export-dir/webhooks/webhook2.json', webhooks[1].label, 'UPDATED'], + ['export-dir/webhooks/webhook2-1.json', webhooks[2].label, 'UPDATED'] + ], + streamTableOptions + ); + }); + }); + describe('separateReplicas', () => { it('it should separate replicas into a mapping', () => { const indices = [ @@ -857,6 +1219,7 @@ describe('search-index export command', (): void => { argv.dir, [], filteredIndicesToExport, + expect.any(Map), expect.any(FileLog), false ); @@ -883,6 +1246,7 @@ describe('search-index export command', (): void => { argv.dir, [], filteredIndicesToExport, + expect.any(Map), expect.any(FileLog), false ); diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index 2828f0f6..f1040803 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -19,7 +19,6 @@ import { FileLog } from '../../common/file-log'; import { ensureDirectoryExists } from '../../common/import/directory-utils'; import { createLog, getDefaultLogPath } from '../../common/log-helpers'; import { streamTableOptions } from '../../common/table/table.consts'; -import { ExportBuilderOptions } from '../../interfaces/export-builder-options.interface'; import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; import { ExportResult, @@ -32,6 +31,7 @@ import { loadJsonFromDirectory } from '../../services/import.service'; import { ConfigurationParameters } from '../configure'; import { validateNoDuplicateIndexNames } from './import'; import { join } from 'path'; +import { ExportBuilderOptions } from '../../interfaces/export-builder-options.interface'; export const command = 'export '; @@ -70,14 +70,57 @@ const ensureJSON = (resource: HalResource): object => { return resource.toJSON != null ? resource.toJSON() : resource; }; -const replicaEquals = (a: EnrichedReplica, b: EnrichedReplica, keys: boolean): boolean => +export const webhookEquals = (a?: Webhook, b?: Webhook): boolean => { + if (a === undefined) { + return b === undefined; + } else if (b === undefined) { + return false; + } + + return ( + a.method === b.method && + a.secret === b.secret && + a.label === b.label && + a.active === b.active && + isEqual(a.customPayload, b.customPayload) && + isEqual(a.events, b.events) && + isEqual(a.filters, b.filters) && + isEqual(a.handlers, b.handlers) && + isEqual(a.headers, b.headers) + ); +}; + +export const replicaEquals = (a: EnrichedReplica, b: EnrichedReplica, keys: boolean): boolean => a.label === b.label && (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); -export const equals = (a: EnrichedSearchIndex, b: EnrichedSearchIndex, keys = true): boolean => +export const assignedContentTypeEquals = ( + a: EnrichedAssignedContentType, + b: EnrichedAssignedContentType, + aWebhooks?: Map, + bWebhooks?: Map +): boolean => { + return ( + !(aWebhooks && bWebhooks) || + (webhookEquals(aWebhooks.get(a.webhook), bWebhooks.get(b.webhook)) && + webhookEquals(aWebhooks.get(a.activeContentWebhook), bWebhooks.get(b.activeContentWebhook)) && + webhookEquals(aWebhooks.get(a.archivedContentWebhook), bWebhooks.get(b.archivedContentWebhook))) + ); +}; + +export const equals = ( + a: EnrichedSearchIndex, + b: EnrichedSearchIndex, + keys = true, + aWebhooks?: Map, + bWebhooks?: Map +): boolean => a.label === b.label && - isEqual(a.assignedContentTypes.map(x => ensureJSON(x)), b.assignedContentTypes.map(x => ensureJSON(x))) && + //isEqual(a.assignedContentTypes.map(x => ensureJSON(x)), b.assignedContentTypes.map(x => ensureJSON(x))) && + a.assignedContentTypes + .map((x, i) => assignedContentTypeEquals(x, b.assignedContentTypes[i], aWebhooks, bWebhooks)) + .reduce((a, b) => a && b, true) && a.replicas.length == b.replicas.length && a.replicas.map((x, i) => replicaEquals(x, b.replicas[i], keys)).reduce((a, b) => a && b, true) && (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && @@ -114,10 +157,7 @@ export class EnrichedSearchIndex extends SearchIndex { } } -export const filterIndicesById = ( - listToFilter: EnrichedSearchIndex[], - indexIdList: string[] -): EnrichedSearchIndex[] => { +export const filterIndicesById = (listToFilter: SearchIndex[], indexIdList: string[]): SearchIndex[] => { if (indexIdList.length === 0) { return listToFilter; } @@ -192,7 +232,9 @@ interface ExportRecord { export const getExportRecordForIndex = ( index: EnrichedSearchIndex, outputDir: string, - previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex } + previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, + previouslyExportedWebhooks: Map, + webhooksBeingExported: Map ): ExportRecord => { const indexOfExportedIndex = Object.values(previouslyExportedIndices).findIndex(c => c.name === index.name); @@ -211,7 +253,7 @@ export const getExportRecordForIndex = ( const filename = Object.keys(previouslyExportedIndices)[indexOfExportedIndex]; const previouslyExportedIndex = Object.values(previouslyExportedIndices)[indexOfExportedIndex]; - if (equals(previouslyExportedIndex, index)) { + if (equals(previouslyExportedIndex, index, true, previouslyExportedWebhooks, webhooksBeingExported)) { return { filename, status: 'UP-TO-DATE', index }; } @@ -227,19 +269,43 @@ type ExportsMap = { filename: string; }; +export const getExportedWebhooks = (outputDir: string): Map => { + const exportedWebhooks = loadJsonFromDirectory(join(outputDir, 'webhooks'), Webhook); + const webhookList = Object.values(exportedWebhooks); + + const webhooks = new Map(); + + for (const webhook of webhookList) { + if (webhook.id) { + webhooks.set(webhook.id, webhook); + } + } + + return webhooks; +}; + export const getIndexExports = ( outputDir: string, previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, - indicesBeingExported: EnrichedSearchIndex[] + indicesBeingExported: EnrichedSearchIndex[], + webhooksBeingExported: Map ): [ExportRecord[], ExportsMap[]] => { const allExports: ExportRecord[] = []; const updatedExportsMap: ExportsMap[] = []; // uri x filename + const previouslyExportedWebhooks = getExportedWebhooks(outputDir); + for (const index of indicesBeingExported) { if (!index.name) { continue; } - const exportRecord = getExportRecordForIndex(index, outputDir, previouslyExportedIndices); + const exportRecord = getExportRecordForIndex( + index, + outputDir, + previouslyExportedIndices, + previouslyExportedWebhooks, + webhooksBeingExported + ); allExports.push(exportRecord); @@ -254,6 +320,7 @@ export const processIndices = async ( outputDir: string, previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, indicesBeingExported: EnrichedSearchIndex[], + webhooksBeingExported: Map, log: FileLog, force: boolean ): Promise => { @@ -262,7 +329,12 @@ export const processIndices = async ( return; } - const [allExports, updatedExportsMap] = getIndexExports(outputDir, previouslyExportedIndices, indicesBeingExported); + const [allExports, updatedExportsMap] = getIndexExports( + outputDir, + previouslyExportedIndices, + indicesBeingExported, + webhooksBeingExported + ); if ( allExports.length === 0 || (Object.keys(updatedExportsMap).length > 0 && !(force || (await promptToOverwriteExports(updatedExportsMap, log)))) @@ -280,6 +352,8 @@ export const processIndices = async ( if (status !== 'UP-TO-DATE') { delete index.id; // do not export id writeJsonToFile(filename, index); + } else { + indicesBeingExported.splice(indicesBeingExported.indexOf(index), 1); } data.push([filename, index.name as string, status]); } @@ -287,7 +361,7 @@ export const processIndices = async ( log.appendLine(table(data, streamTableOptions)); }; -const filterWebhooks = ( +export const filterWebhooks = ( webhooks: Map, filteredIndices: EnrichedSearchIndex[] ): Map => { @@ -304,13 +378,19 @@ const filterWebhooks = ( return filtered; }; -const processWebhooks = async ( +export const processWebhooks = async ( outputDir: string, //previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, webhooksBeingExported: Webhook[], log: FileLog, force: boolean ): Promise => { + if (webhooksBeingExported.length === 0) { + return; + } + + log.appendLine('Exporting Webhooks...'); + const previouslyExportedWebhooks: { [filename: string]: Webhook } = {}; const base = join(outputDir, 'webhooks'); await ensureDirectoryExists(base); @@ -320,7 +400,7 @@ const processWebhooks = async ( data.push([chalk.bold('File'), chalk.bold('Label'), chalk.bold('Result')]); for (const webhook of webhooksBeingExported) { const filename = uniqueFilenamePath(base, webhook.label, 'json', Object.keys(previouslyExportedWebhooks)); - delete webhook.id; // do not export id + previouslyExportedWebhooks[filename] = webhook; writeJsonToFile(filename, webhook); data.push([filename, webhook.label as string, 'UPDATED']); } @@ -363,14 +443,15 @@ export const handler = async (argv: Arguments(); - const enrichedIndices = await Promise.all(storedIndices.map(index => enrichIndex(webhooks, allReplicas, index))); + const enrichedIndices = await Promise.all(filteredIndices.map(index => enrichIndex(webhooks, allReplicas, index))); - const idArray: string[] = id ? (Array.isArray(id) ? id : [id]) : []; - const filteredIndices = filterIndicesById(enrichedIndices, idArray); - await processIndices(dir, previouslyExportedIndices, filteredIndices, log, force || false); + await processIndices(dir, previouslyExportedIndices, enrichedIndices, webhooks, log, force || false); - const filteredWebhooks = filterWebhooks(webhooks, filteredIndices); + const filteredWebhooks = filterWebhooks(webhooks, enrichedIndices); await processWebhooks(dir, Array.from(filteredWebhooks.values()), log, force || false); await log.close(); diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts index 70331ab4..4e745f6d 100644 --- a/src/commands/search-index/import.spec.ts +++ b/src/commands/search-index/import.spec.ts @@ -1,12 +1,15 @@ import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; -import { SearchIndex, Hub } from 'dc-management-sdk-js'; +import { SearchIndex, Hub, Webhook } from 'dc-management-sdk-js'; +import * as exportModule from './export'; import * as importModule from './import'; +import * as webhookRewriter from './webhook-rewriter'; import { builder, command, doCreate, doUpdate, handler, + loadAndRewriteWebhooks, LOG_FILENAME, processIndices, storedIndexMapper, @@ -90,8 +93,59 @@ describe('search-index import command', (): void => { }); }); + describe('replicaList', () => { + it('should return a function that lists the replicas for the given index, with the given projection', async () => { + const replicas = new MockPage(SearchIndex, [new SearchIndex({ label: 'webhook1' })]); + const index = new SearchIndex({ label: 'webhook1' }); + index.related.replicas.list = jest.fn().mockResolvedValue(replicas); + + const resultFn = await importModule.replicaList(index, 'projection'); + + const noOptionsResult = await resultFn(); + expect(noOptionsResult).toEqual(replicas); + expect(index.related.replicas.list).toHaveBeenCalledWith('projection', undefined); + + const optionsResult = await resultFn({ sort: 'sort' }); + expect(optionsResult).toEqual(replicas); + expect(index.related.replicas.list).toHaveBeenCalledWith('projection', { sort: 'sort' }); + }); + + it('should return a function that lists the replicas for the given index, with no projection', async () => { + const replicas = new MockPage(SearchIndex, [new SearchIndex({ label: 'webhook1' })]); + const index = new SearchIndex({ label: 'webhook1' }); + index.related.replicas.list = jest.fn().mockResolvedValue(replicas); + + const resultFn = await importModule.replicaList(index); + + const result = await resultFn(); + expect(result).toEqual(replicas); + expect(index.related.replicas.list).toHaveBeenCalledWith(undefined, undefined); + }); + }); + + describe('updateWebhookIfDifferent', () => { + it('should update the webhook if newWebhook is defined', async () => { + const webhook = new Webhook({ label: 'webhook1' }); + webhook.related.update = jest.fn().mockResolvedValue(webhook); + const newWebhook = new Webhook({ label: 'webhook2' }); + + await importModule.updateWebhookIfDifferent(webhook, newWebhook); + + expect(webhook.related.update).toHaveBeenCalledWith(newWebhook); + }); + + it('should not update the webhook if newWebhook is undefined', async () => { + const webhook = new Webhook({ label: 'webhook1' }); + webhook.related.update = jest.fn().mockResolvedValue(webhook); + + await importModule.updateWebhookIfDifferent(webhook, undefined); + + expect(webhook.related.update).not.toHaveBeenCalled(); + }); + }); + describe('doCreate', () => { - it('should create a index and return report', async () => { + it('should create an index and return report', async () => { const mockHub = new Hub(); const log = new FileLog(); const newIndex = new SearchIndex({ id: 'created-id' }); @@ -99,7 +153,8 @@ describe('search-index import command', (): void => { mockHub.related.searchIndexes.create = mockCreate; jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); const index = { name: 'index-name', label: 'test-label' }; - const result = await doCreate(mockHub, index as EnrichedSearchIndex, log); + const webhooks = new Map(); + const result = await doCreate(mockHub, index as EnrichedSearchIndex, webhooks, log); expect(log.getData('CREATE')).toMatchInlineSnapshot(` Array [ @@ -107,7 +162,7 @@ describe('search-index import command', (): void => { ] `); expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining(index)); - expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index); + expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index, webhooks); expect(result).toEqual(newIndex); }); @@ -121,7 +176,9 @@ describe('search-index import command', (): void => { mockHub.related.searchIndexes.create = mockCreate; const index = { name: 'index-name', label: 'test-label' }; - await expect(doCreate(mockHub, index as EnrichedSearchIndex, log)).rejects.toThrowErrorMatchingSnapshot(); + await expect( + doCreate(mockHub, index as EnrichedSearchIndex, new Map(), log) + ).rejects.toThrowErrorMatchingSnapshot(); expect(importModule.enrichIndex).not.toHaveBeenCalled(); expect(log.getData('CREATE')).toEqual([]); }); @@ -138,7 +195,9 @@ describe('search-index import command', (): void => { mockHub.related.searchIndexes.create = mockCreate; const index = { name: 'index-name', label: 'test-label' }; - await expect(doCreate(mockHub, index as EnrichedSearchIndex, log)).rejects.toThrowErrorMatchingSnapshot(); + await expect( + doCreate(mockHub, index as EnrichedSearchIndex, new Map(), log) + ).rejects.toThrowErrorMatchingSnapshot(); expect(importModule.enrichIndex).not.toHaveBeenCalled(); expect(log.getData('CREATE')).toEqual([]); }); @@ -155,15 +214,321 @@ describe('search-index import command', (): void => { 'The update-index action is not available, ensure you have permission to perform this action.' ); const index = { name: 'index-name', label: 'test-label' }; + const webhooks = new Map(); - await expect(doCreate(mockHub, index as EnrichedSearchIndex, log)).rejects.toThrowErrorMatchingSnapshot(); + await expect( + doCreate(mockHub, index as EnrichedSearchIndex, webhooks, log) + ).rejects.toThrowErrorMatchingSnapshot(); expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining(index)); - expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index); + expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index, webhooks); expect(log.getData('CREATE')).toEqual([]); }); }); + describe('doUpdate', () => { + beforeEach(() => {}); + + it('should update an index and return report', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + } as EnrichedSearchIndex; + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + const expectedIndex = new SearchIndex({ + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + }); + + const updatedIndex = new SearchIndex(mutatedIndex); + const mockUpdate = jest.fn().mockResolvedValue(updatedIndex); + storedIndex.related.update = mockUpdate; + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + const enrichedStoredIndex = new EnrichedSearchIndex(storedIndex); + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce(enrichedStoredIndex); + jest.spyOn(importModule, 'getIndexProperties').mockReturnValueOnce(mutatedIndex); + jest.spyOn(importModule, 'enrichIndex').mockResolvedValueOnce(); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + const result = await doUpdate(hub, replicas, mutatedIndex, webhooks, log); + + expect(log.getData('UPDATE')).toMatchInlineSnapshot(` + Array [ + "stored-id", + ] + `); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(importModule.enrichIndex).toHaveBeenCalledWith(updatedIndex, mutatedIndex, webhooks); + expect(result).toEqual({ index: updatedIndex, updateStatus: UpdateStatus.UPDATED }); + expect(mockUpdate.mock.calls[0][0].toJSON()).toEqual(expectedIndex.toJSON()); + }); + + it('should skip update when no change to index and return report', async () => { + const mutatedIndex = new EnrichedSearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label', + settings: {}, + assignedContentTypes: [], + replicas: [] + }); + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce( + new EnrichedSearchIndex({ + ...storedIndex, + settings: {}, + assignedContentTypes: [], + replicas: [] + }) + ); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + const result = await doUpdate(hub, replicas, mutatedIndex, webhooks, log); + + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(result).toEqual({ index: storedIndex, updateStatus: UpdateStatus.SKIPPED }); + expect(log.getData('UPDATE')).toEqual([]); + }); + + it('should throw an error when unable to get index during update', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'matched-name', + label: 'label' + } as EnrichedSearchIndex; + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockImplementation(() => { + throw new Error('Error retrieving index'); + }); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + + await expect(doUpdate(hub, replicas, mutatedIndex, webhooks, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(log.getData('UPDATE')).toEqual([]); + }); + + it('should throw an error when unable to update index during update if a string error is returned by sdk', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + } as EnrichedSearchIndex; + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + const expectedIndex = new SearchIndex({ + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + }); + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + const mockUpdate = jest + .fn() + .mockRejectedValue('The update action is not available, ensure you have permission to perform this action.'); + storedIndex.related.update = mockUpdate; + + const enrichedStoredIndex = new EnrichedSearchIndex(storedIndex); + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce(enrichedStoredIndex); + jest.spyOn(importModule, 'getIndexProperties').mockReturnValueOnce(mutatedIndex); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + await expect(doUpdate(hub, replicas, mutatedIndex, webhooks, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(log.getData('UPDATE')).toEqual([]); + expect(mockUpdate.mock.calls[0][0].toJSON()).toEqual(expectedIndex.toJSON()); + }); + + it('should throw an error when unable to update index during update', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + } as EnrichedSearchIndex; + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + const expectedIndex = new SearchIndex({ + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + }); + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + const mockUpdate = jest.fn().mockRejectedValue(new Error('Error saving index')); + storedIndex.related.update = mockUpdate; + + const enrichedStoredIndex = new EnrichedSearchIndex(storedIndex); + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce(enrichedStoredIndex); + jest.spyOn(importModule, 'getIndexProperties').mockReturnValueOnce(mutatedIndex); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + await expect(doUpdate(hub, replicas, mutatedIndex, webhooks, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(log.getData('UPDATE')).toEqual([]); + expect(mockUpdate.mock.calls[0][0].toJSON()).toEqual(expectedIndex.toJSON()); + }); + }); + + describe('loadAndRewriteWebhooks', () => { + it('should create an id to webhook map from the webhooks loaded from the given directory', async () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1', + customPayload: { value: 'a' } + }), + 'directory/webhooks/webhook2.json': new Webhook({ + id: 'id2', + label: 'webhook2', + customPayload: { value: 'b' } + }) + }; + + const hub = new Hub({ + name: 'accountName', + settings: { + virtualStagingEnvironment: { + hostname: 'http://amplience.com' + } + } + }); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + jest.spyOn(webhookRewriter, 'rewriteDeliveryContentItem').mockImplementation(body => { + return body + '-rewrite'; + }); + + const result = await loadAndRewriteWebhooks(hub, 'directory/webhooks'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(2); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(result.get('id2')).toEqual(webhooks['directory/webhooks/webhook2.json']); + expect(webhooks['directory/webhooks/webhook1.json'].customPayload).toEqual({ value: 'a-rewrite' }); + expect(webhooks['directory/webhooks/webhook2.json'].customPayload).toEqual({ value: 'b-rewrite' }); + + expect(webhookRewriter.rewriteDeliveryContentItem).toHaveBeenNthCalledWith( + 1, + 'a', + 'accountName', + 'http://amplience.com' + ); + expect(webhookRewriter.rewriteDeliveryContentItem).toHaveBeenNthCalledWith( + 2, + 'b', + 'accountName', + 'http://amplience.com' + ); + }); + + it('should not rewrite webhook body if none is present', async () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1' + }), + 'directory/webhooks/webhook2.json': new Webhook({ + id: 'id2', + label: 'webhook2' + }) + }; + + const hub = new Hub({ + name: 'accountName', + settings: { + virtualStagingEnvironment: { + hostname: 'http://amplience.com' + } + } + }); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + jest.spyOn(webhookRewriter, 'rewriteDeliveryContentItem'); + + const result = await loadAndRewriteWebhooks(hub, 'directory/webhooks'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(2); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(result.get('id2')).toEqual(webhooks['directory/webhooks/webhook2.json']); + expect(webhooks['directory/webhooks/webhook1.json'].customPayload).toBeUndefined(); + expect(webhooks['directory/webhooks/webhook2.json'].customPayload).toBeUndefined(); + + expect(webhookRewriter.rewriteDeliveryContentItem).not.toHaveBeenCalled(); + }); + + it('should pass an undefined vse to the rewriter if the object is missing from settings', async () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1', + customPayload: { value: 'a' } + }) + }; + + const hub = new Hub({ + name: 'accountName', + settings: {} + }); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + jest.spyOn(webhookRewriter, 'rewriteDeliveryContentItem').mockImplementation(body => { + return body + '-rewrite'; + }); + + const result = await loadAndRewriteWebhooks(hub, 'directory/webhooks'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(1); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(webhooks['directory/webhooks/webhook1.json'].customPayload).toEqual({ value: 'a-rewrite' }); + + expect(webhookRewriter.rewriteDeliveryContentItem).toHaveBeenCalledWith('a', 'accountName', undefined); + }); + }); + describe('processIndices', () => { let mockTable: jest.Mock; @@ -172,8 +537,7 @@ describe('search-index import command', (): void => { mockTable.mockImplementation(jest.requireActual('table').table); }); - it('should create and update a index', async () => { - const client = (dynamicContentClientFactory as jest.Mock)(); + it('should create and update an index', async () => { const hub = new Hub(); const indicesToProcess = [ new EnrichedSearchIndex({ @@ -208,14 +572,17 @@ describe('search-index import command', (): void => { updateStatus: UpdateStatus.SKIPPED }; jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult2); + const webhooks = new Map(); + const replicas = new Map(); - await processIndices(indicesToProcess, new Map(), client, hub, new FileLog()); + await processIndices(indicesToProcess, replicas, webhooks, hub, new FileLog()); - expect(importModule.doCreate).toHaveBeenCalledWith(hub, indicesToProcess[0], expect.any(FileLog)); + expect(importModule.doCreate).toHaveBeenCalledWith(hub, indicesToProcess[0], webhooks, expect.any(FileLog)); expect(importModule.doUpdate).toHaveBeenCalledWith( hub, - expect.any(Map), + replicas, indicesToProcess[1], + webhooks, expect.any(FileLog) ); @@ -354,7 +721,7 @@ describe('search-index import command', (): void => { expect(getDefaultLogPath).toHaveBeenCalledWith('search-index', 'import', process.platform); }); - it('should create a index and update', async (): Promise => { + it('should create an index and update', async (): Promise => { const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; const fileNamesAndIndicesToImport = { 'file-1': new EnrichedSearchIndex({ @@ -383,7 +750,7 @@ describe('search-index import command', (): void => { expect(processIndices).toHaveBeenCalledWith( Object.values(fileNamesAndIndicesToImport), expect.any(Map), - expect.any(Object), + expect.any(Map), expect.any(Object), expect.any(FileLog) ); @@ -415,7 +782,7 @@ describe('search-index import command', (): void => { expect(processIndices).toHaveBeenCalledWith( [fileNamesAndIndicesToImport['file-2']], expect.any(Map), - expect.any(Object), + expect.any(Map), expect.any(Object), expect.any(FileLog) ); diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index fe618d53..e071ce20 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -1,16 +1,18 @@ import chalk from 'chalk'; -import { DynamicContent, Hub, Page, Pageable, SearchIndex, SearchIndexSettings, Sortable } from 'dc-management-sdk-js'; +import { Hub, Page, Pageable, SearchIndex, Settings, Sortable, Webhook } from 'dc-management-sdk-js'; +import { join } from 'path'; import { table } from 'table'; import { Arguments, Argv } from 'yargs'; import paginator from '../../common/dc-management-sdk-js/paginator'; import { FileLog } from '../../common/file-log'; import { createLog, getDefaultLogPath } from '../../common/log-helpers'; import { streamTableOptions } from '../../common/table/table.consts'; -import { ImportBuilderOptions } from '../../interfaces/import-builder-options.interface'; +import { ImportIndexBuilderOptions } from '../../interfaces/import-index-builder-options.interface'; import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; import { ImportResult, loadJsonFromDirectory, UpdateStatus } from '../../services/import.service'; import { ConfigurationParameters } from '../configure'; import { EnrichedSearchIndex, equals, enrichIndex as enrichServerIndex, separateReplicas } from './export'; +import { rewriteDeliveryContentItem } from './webhook-rewriter'; export const command = 'import '; @@ -38,7 +40,7 @@ const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { hub.related.searchIndexes.list(parentId, projection, options); }; -const replicaList = (index: SearchIndex, projection?: string) => { +export const replicaList = (index: SearchIndex, projection?: string) => { return (options?: Pageable & Sortable): Promise> => index.related.replicas.list(projection, options); }; @@ -94,7 +96,7 @@ export const storedIndexMapper = (index: EnrichedSearchIndex, storedIndices: Sea return new EnrichedSearchIndex(mutatedIndex); }; -const getIndexProperties = (index: SearchIndex): object => { +export const getIndexProperties = (index: SearchIndex): object => { return { label: index.label, name: index.name, @@ -103,10 +105,34 @@ const getIndexProperties = (index: SearchIndex): object => { }; }; -export const enrichIndex = async (index: SearchIndex, enrichedIndex: EnrichedSearchIndex): Promise => { +export const updateWebhookIfDifferent = async (webhook: Webhook, newWebhook: Webhook | undefined): Promise => { + if (newWebhook === undefined) { + return; + } + + await webhook.related.update(newWebhook); +}; + +export const enrichIndex = async ( + index: SearchIndex, + enrichedIndex: EnrichedSearchIndex, + webhooks: Map +): Promise => { + // Union the replicas on the server and the replicas being imported. + // This avoids replicas being detached from their parents, and thus becoming unusable. + const settings = await index.related.settings.get(); + const replicas = new Set(settings.replicas || []); + if (enrichedIndex.settings.replicas) { + enrichedIndex.settings.replicas.forEach((replica: string) => { + replicas.add(replica); + }); + } + enrichedIndex.settings.replicas = Array.from(replicas); + + // Update the search index settings. await index.related.settings.update(enrichedIndex.settings, false); - if (enrichedIndex.settings.replicas && enrichedIndex.settings.replicas.length) { + if (replicas.size) { // Replica settings must also be updated. The replicas may have changed, so fetch them again. const replicas = await paginator(replicaList(index)); @@ -141,7 +167,15 @@ export const enrichIndex = async (index: SearchIndex, enrichedIndex: EnrichedSea // Update any webhooks if they differ from the ones being imported, if the flag is provided. // Does the webhook being referenced in the saved index exist in the import? - //const webhooks = new Map(); + await updateWebhookIfDifferent(await existing.related.webhook(), webhooks.get(assignment.webhook)); + await updateWebhookIfDifferent( + await existing.related.activeContentWebhook(), + webhooks.get(assignment.activeContentWebhook) + ); + await updateWebhookIfDifferent( + await existing.related.archivedContentWebhook(), + webhooks.get(assignment.archivedContentWebhook) + ); } // Finally, remove any content type assignments that are not present in the imported index. @@ -150,11 +184,16 @@ export const enrichIndex = async (index: SearchIndex, enrichedIndex: EnrichedSea } }; -export const doCreate = async (hub: Hub, index: EnrichedSearchIndex, log: FileLog): Promise => { +export const doCreate = async ( + hub: Hub, + index: EnrichedSearchIndex, + webhooks: Map, + log: FileLog +): Promise => { try { const createdIndex = await hub.related.searchIndexes.create(new SearchIndex(getIndexProperties(index))); - await enrichIndex(createdIndex, index); + await enrichIndex(createdIndex, index, webhooks); log.addAction('CREATE', `${createdIndex.id}`); @@ -168,13 +207,17 @@ export const doUpdate = async ( hub: Hub, allReplicas: Map, index: EnrichedSearchIndex, + webhooks: Map, log: FileLog ): Promise<{ index: SearchIndex; updateStatus: UpdateStatus }> => { try { const retrievedIndex: SearchIndex = await hub.related.searchIndexes.get(index.id || ''); - const webhooks = new Map(); - if (equals(await enrichServerIndex(webhooks, allReplicas, retrievedIndex), index, false)) { + const dstWebhooks = new Map(); + + const enrichedWebhook = await enrichServerIndex(dstWebhooks, allReplicas, retrievedIndex); + + if (equals(enrichedWebhook, index, false, dstWebhooks, webhooks)) { return { index: retrievedIndex, updateStatus: UpdateStatus.SKIPPED }; } @@ -182,7 +225,7 @@ export const doUpdate = async ( const updatedIndex = await retrievedIndex.related.update(retrievedIndex); - await enrichIndex(updatedIndex, index); + await enrichIndex(updatedIndex, index, webhooks); log.addAction('UPDATE', `${retrievedIndex.id}`); @@ -192,25 +235,48 @@ export const doUpdate = async ( } }; +export const loadAndRewriteWebhooks = async (hub: Hub, dir: string): Promise> => { + const webhookList = loadJsonFromDirectory(dir, Webhook); + const webhooks = new Map(); + + for (const webhook of Object.values(webhookList)) { + webhooks.set(webhook.id as string, webhook); + } + + // Rewrite webhooks. Load VSE and account name from settings. + const account = hub.name as string; + const settings = hub.settings as Settings; + const vseObj = settings.virtualStagingEnvironment; + const vse = vseObj ? vseObj.hostname : undefined; + + webhooks.forEach(webhook => { + if (webhook.customPayload) { + webhook.customPayload.value = rewriteDeliveryContentItem(webhook.customPayload.value, account, vse); + } + }); + + return webhooks; +}; + export const processIndices = async ( indicesToProcess: EnrichedSearchIndex[], allReplicas: Map, - client: DynamicContent, + webhooks: Map, hub: Hub, log: FileLog ): Promise => { const data: string[][] = []; data.push([chalk.bold('ID'), chalk.bold('Name'), chalk.bold('Result')]); - for (const schema of indicesToProcess) { + for (const entry of indicesToProcess) { let status: ImportResult; let index: SearchIndex; - if (schema.id) { - const result = await doUpdate(hub, allReplicas, schema, log); + if (entry.id) { + const result = await doUpdate(hub, allReplicas, entry, webhooks, log); index = result.index; status = result.updateStatus === UpdateStatus.SKIPPED ? 'UP-TO-DATE' : 'UPDATED'; } else { - index = await doCreate(hub, schema, log); + index = await doCreate(hub, entry, webhooks, log); status = 'CREATED'; } data.push([index.id || '', index.name as string, status]); @@ -220,7 +286,7 @@ export const processIndices = async ( }; export const handler = async ( - argv: Arguments, + argv: Arguments, idFilter?: string[] ): Promise => { const { dir, logFile } = argv; @@ -242,8 +308,9 @@ export const handler = async ( const { storedIndices, allReplicas } = separateReplicas(allStoredIndices); const indicesToProcess = Object.values(indices).map(index => storedIndexMapper(index, storedIndices)); + const webhooks = argv.webhooks ? await loadAndRewriteWebhooks(hub, join(dir, 'webhooks')) : new Map(); - await processIndices(indicesToProcess, allReplicas, client, hub, log); + await processIndices(indicesToProcess, allReplicas, webhooks, hub, log); await log.close(); }; diff --git a/src/commands/search-index/webhook-rewriter.ts b/src/commands/search-index/webhook-rewriter.ts index ed567c33..2cc5871a 100644 --- a/src/commands/search-index/webhook-rewriter.ts +++ b/src/commands/search-index/webhook-rewriter.ts @@ -9,7 +9,11 @@ function matchAll(regex: RegExp, string: string): RegExpExecArray[] { return result; } -export function rewriteDeliveryContentItem(webhookBody: string, account: string, stagingEnvironment: string): string { +export function rewriteDeliveryContentItem( + webhookBody: string, + account: string, + stagingEnvironment: string | undefined +): string { // Current limitations - cannot handle key/value pairs where the value contains space, or a }} within quotes. // These should not affect the two variables being replaced anyways. // First, locate the withDeliveryContentItem opening tags. @@ -46,7 +50,9 @@ export function rewriteDeliveryContentItem(webhookBody: string, account: string, replaceValue = account; break; case 'stagingEnvironment': - replaceValue = stagingEnvironment; + if (stagingEnvironment !== undefined) { + replaceValue = stagingEnvironment; + } break; } diff --git a/src/interfaces/import-index-builder-options.interface.ts b/src/interfaces/import-index-builder-options.interface.ts new file mode 100644 index 00000000..a1ead6a7 --- /dev/null +++ b/src/interfaces/import-index-builder-options.interface.ts @@ -0,0 +1,5 @@ +import { ImportBuilderOptions } from './import-builder-options.interface'; + +export interface ImportIndexBuilderOptions extends ImportBuilderOptions { + webhooks?: boolean; +} From b3bb3d79033a7d124c4a116fe8aeff22638117dd Mon Sep 17 00:00:00 2001 From: Rhys Date: Tue, 10 Aug 2021 16:33:53 +0100 Subject: [PATCH 05/12] fix(search-index): missing webhooks option, fix equals check for no-webhook mode and for replicas --- src/commands/search-index/export.ts | 22 +++++++------- src/commands/search-index/import.spec.ts | 11 +++++-- src/commands/search-index/import.ts | 37 +++++++++++++++--------- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index f1040803..d8872597 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -11,7 +11,7 @@ import { } from 'dc-management-sdk-js'; import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; -import { isEqual } from 'lodash'; +import { isEqual, result } from 'lodash'; import { table } from 'table'; import { Arguments, Argv } from 'yargs'; import paginator from '../../common/dc-management-sdk-js/paginator'; @@ -100,13 +100,16 @@ export const assignedContentTypeEquals = ( b: EnrichedAssignedContentType, aWebhooks?: Map, bWebhooks?: Map -): boolean => { - return ( - !(aWebhooks && bWebhooks) || - (webhookEquals(aWebhooks.get(a.webhook), bWebhooks.get(b.webhook)) && - webhookEquals(aWebhooks.get(a.activeContentWebhook), bWebhooks.get(b.activeContentWebhook)) && - webhookEquals(aWebhooks.get(a.archivedContentWebhook), bWebhooks.get(b.archivedContentWebhook))) - ); +): boolean => + !(aWebhooks && bWebhooks) || + (webhookEquals(aWebhooks.get(a.webhook), bWebhooks.get(b.webhook)) && + webhookEquals(aWebhooks.get(a.activeContentWebhook), bWebhooks.get(b.activeContentWebhook)) && + webhookEquals(aWebhooks.get(a.archivedContentWebhook), bWebhooks.get(b.archivedContentWebhook))); + +export const ensureSettings = (settings: SearchIndexSettings): object => { + const result = ensureJSON(settings) as SearchIndexSettings; + result.replicas = result.replicas || []; + return result; }; export const equals = ( @@ -117,14 +120,13 @@ export const equals = ( bWebhooks?: Map ): boolean => a.label === b.label && - //isEqual(a.assignedContentTypes.map(x => ensureJSON(x)), b.assignedContentTypes.map(x => ensureJSON(x))) && a.assignedContentTypes .map((x, i) => assignedContentTypeEquals(x, b.assignedContentTypes[i], aWebhooks, bWebhooks)) .reduce((a, b) => a && b, true) && a.replicas.length == b.replicas.length && a.replicas.map((x, i) => replicaEquals(x, b.replicas[i], keys)).reduce((a, b) => a && b, true) && (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && - isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); + isEqual(ensureSettings(a.settings), ensureSettings(b.settings)); const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { return (options?: Pageable & Sortable): Promise> => diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts index 4e745f6d..2b85a98c 100644 --- a/src/commands/search-index/import.spec.ts +++ b/src/commands/search-index/import.spec.ts @@ -60,6 +60,13 @@ describe('search-index import command', (): void => { describe: 'Path to a log file to write to.', coerce: createLog }); + + expect(spyOption).toHaveBeenCalledWith('webhooks', { + type: 'boolean', + describe: + 'Import webhooks as well. The command will attempt to rewrite account names and staging environments in the webhook body to match the destination.', + boolean: true + }); }); }); @@ -750,7 +757,7 @@ describe('search-index import command', (): void => { expect(processIndices).toHaveBeenCalledWith( Object.values(fileNamesAndIndicesToImport), expect.any(Map), - expect.any(Map), + undefined, expect.any(Object), expect.any(FileLog) ); @@ -782,7 +789,7 @@ describe('search-index import command', (): void => { expect(processIndices).toHaveBeenCalledWith( [fileNamesAndIndicesToImport['file-2']], expect.any(Map), - expect.any(Map), + undefined, expect.any(Object), expect.any(FileLog) ); diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index e071ce20..df2f2764 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -33,6 +33,13 @@ export const builder = (yargs: Argv): void => { describe: 'Path to a log file to write to.', coerce: createLog }); + + yargs.option('webhooks', { + type: 'boolean', + describe: + 'Import webhooks as well. The command will attempt to rewrite account names and staging environments in the webhook body to match the destination.', + boolean: true + }); }; const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { @@ -116,7 +123,7 @@ export const updateWebhookIfDifferent = async (webhook: Webhook, newWebhook: Web export const enrichIndex = async ( index: SearchIndex, enrichedIndex: EnrichedSearchIndex, - webhooks: Map + webhooks: Map | undefined ): Promise => { // Union the replicas on the server and the replicas being imported. // This avoids replicas being detached from their parents, and thus becoming unusable. @@ -167,15 +174,17 @@ export const enrichIndex = async ( // Update any webhooks if they differ from the ones being imported, if the flag is provided. // Does the webhook being referenced in the saved index exist in the import? - await updateWebhookIfDifferent(await existing.related.webhook(), webhooks.get(assignment.webhook)); - await updateWebhookIfDifferent( - await existing.related.activeContentWebhook(), - webhooks.get(assignment.activeContentWebhook) - ); - await updateWebhookIfDifferent( - await existing.related.archivedContentWebhook(), - webhooks.get(assignment.archivedContentWebhook) - ); + if (webhooks) { + await updateWebhookIfDifferent(await existing.related.webhook(), webhooks.get(assignment.webhook)); + await updateWebhookIfDifferent( + await existing.related.activeContentWebhook(), + webhooks.get(assignment.activeContentWebhook) + ); + await updateWebhookIfDifferent( + await existing.related.archivedContentWebhook(), + webhooks.get(assignment.archivedContentWebhook) + ); + } } // Finally, remove any content type assignments that are not present in the imported index. @@ -187,7 +196,7 @@ export const enrichIndex = async ( export const doCreate = async ( hub: Hub, index: EnrichedSearchIndex, - webhooks: Map, + webhooks: Map | undefined, log: FileLog ): Promise => { try { @@ -207,7 +216,7 @@ export const doUpdate = async ( hub: Hub, allReplicas: Map, index: EnrichedSearchIndex, - webhooks: Map, + webhooks: Map | undefined, log: FileLog ): Promise<{ index: SearchIndex; updateStatus: UpdateStatus }> => { try { @@ -261,7 +270,7 @@ export const loadAndRewriteWebhooks = async (hub: Hub, dir: string): Promise, - webhooks: Map, + webhooks: Map | undefined, hub: Hub, log: FileLog ): Promise => { @@ -308,7 +317,7 @@ export const handler = async ( const { storedIndices, allReplicas } = separateReplicas(allStoredIndices); const indicesToProcess = Object.values(indices).map(index => storedIndexMapper(index, storedIndices)); - const webhooks = argv.webhooks ? await loadAndRewriteWebhooks(hub, join(dir, 'webhooks')) : new Map(); + const webhooks = argv.webhooks ? await loadAndRewriteWebhooks(hub, join(dir, 'webhooks')) : undefined; await processIndices(indicesToProcess, allReplicas, webhooks, hub, log); From de7df09f82d9ce11d3b813d31db57438c5d96b5a Mon Sep 17 00:00:00 2001 From: Rhys Date: Wed, 11 Aug 2021 17:31:54 +0100 Subject: [PATCH 06/12] feat(search-index): add clone step for search index --- src/commands/hub/clone.spec.ts | 32 ++- src/commands/hub/clone.ts | 3 + src/commands/hub/model/clone-hub-step.ts | 1 + .../hub/steps/index-clone-step.spec.ts | 231 ++++++++++++++++++ src/commands/hub/steps/index-clone-step.ts | 84 +++++++ src/commands/search-index/export.spec.ts | 2 +- src/commands/search-index/export.ts | 7 +- src/commands/search-index/import.spec.ts | 4 +- 8 files changed, 345 insertions(+), 19 deletions(-) create mode 100644 src/commands/hub/steps/index-clone-step.spec.ts create mode 100644 src/commands/hub/steps/index-clone-step.ts diff --git a/src/commands/hub/clone.spec.ts b/src/commands/hub/clone.spec.ts index eb3061bf..69d2b71e 100644 --- a/src/commands/hub/clone.spec.ts +++ b/src/commands/hub/clone.spec.ts @@ -8,6 +8,7 @@ import * as settings from './steps/settings-clone-step'; import * as schema from './steps/schema-clone-step'; import * as type from './steps/type-clone-step'; import * as extension from './steps/extension-clone-step'; +import * as index from './steps/index-clone-step'; import rmdir from 'rimraf'; import { CloneHubBuilderOptions } from '../../interfaces/clone-hub-builder-options'; @@ -21,7 +22,7 @@ jest.mock('readline'); jest.mock('../../services/dynamic-content-client-factory'); -let success = [true, true, true, true, true]; +let success = [true, true, true, true, true, true]; // eslint-disable-next-line @typescript-eslint/no-explicit-any function succeedOrFail(mock: any, succeed: () => boolean): jest.Mock { @@ -55,8 +56,12 @@ jest.mock('./steps/type-clone-step', () => ({ TypeCloneStep: mockStep('Clone Content Types', 'types', () => success[3]) })); +jest.mock('./steps/index-clone-step', () => ({ + IndexCloneStep: mockStep('Clone Indices', 'index', () => success[4]) +})); + jest.mock('./steps/content-clone-step', () => ({ - ContentCloneStep: mockStep('Clone Content', 'content', () => success[4]) + ContentCloneStep: mockStep('Clone Content', 'content', () => success[5]) })); jest.mock('../../common/log-helpers', () => ({ @@ -76,6 +81,7 @@ function getMocks(): jest.Mock[] { extension.ExtensionCloneStep as jest.Mock, schema.SchemaCloneStep as jest.Mock, type.TypeCloneStep as jest.Mock, + index.IndexCloneStep as jest.Mock, content.ContentCloneStep as jest.Mock ]; } @@ -269,7 +275,7 @@ describe('hub clone command', () => { it('should call all steps in order with given parameters', async () => { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; const argv: Arguments = { ...yargArgs, @@ -307,9 +313,9 @@ describe('hub clone command', () => { }); it('should handle false returns from each of the steps by stopping the process', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [i != 0, i != 1, i != 2, i != 3, i != 4]; + success = [i != 0, i != 1, i != 2, i != 3, i != 4, i != 5]; const argv: Arguments = { ...yargArgs, @@ -351,9 +357,9 @@ describe('hub clone command', () => { }); it('should start from the step given as a parameter', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; const argv: Arguments = { ...yargArgs, @@ -463,7 +469,7 @@ describe('hub clone command', () => { it('should revert all steps in order with given parameters', async () => { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); await prepareFakeLog(`temp_${process.env.JEST_WORKER_ID}/clone-revert/steps.log`); @@ -503,9 +509,9 @@ describe('hub clone command', () => { }); it('should handle exceptions from each of the revert steps by stopping the process', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [i != 0, i != 1, i != 2, i != 3, i != 4]; + success = [i != 0, i != 1, i != 2, i != 3, i != 4, i != 5]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); await prepareFakeLog(`temp_${process.env.JEST_WORKER_ID}/clone-revert/fail.log`); @@ -552,7 +558,7 @@ describe('hub clone command', () => { it('should exit early if revert log cannot be read', async () => { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); const argv: Arguments = { @@ -589,9 +595,9 @@ describe('hub clone command', () => { }); it('should start reverting from the step given as a parameter (steps in decreasing order)', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); await prepareFakeLog(`temp_${process.env.JEST_WORKER_ID}/clone-revert/step.log`); diff --git a/src/commands/hub/clone.ts b/src/commands/hub/clone.ts index fec6f576..1b0ca15b 100644 --- a/src/commands/hub/clone.ts +++ b/src/commands/hub/clone.ts @@ -11,6 +11,7 @@ import { ContentCloneStep } from './steps/content-clone-step'; import { SchemaCloneStep } from './steps/schema-clone-step'; import { SettingsCloneStep } from './steps/settings-clone-step'; import { TypeCloneStep } from './steps/type-clone-step'; +import { IndexCloneStep } from './steps/index-clone-step'; import { CloneHubState } from './model/clone-hub-state'; import { LogErrorLevel } from '../../common/archive/archive-log'; import { ExtensionCloneStep } from './steps/extension-clone-step'; @@ -29,6 +30,7 @@ export function getDefaultMappingPath(name: string, platform: string = process.p // hub-*/extensions/ // hub-*/schemas/ // hub-*/types/ +// hub-*/indices/ // hub-*/content/ // hub-*/events/ @@ -45,6 +47,7 @@ export const steps = [ new ExtensionCloneStep(), new SchemaCloneStep(), new TypeCloneStep(), + new IndexCloneStep(), new ContentCloneStep() ]; diff --git a/src/commands/hub/model/clone-hub-step.ts b/src/commands/hub/model/clone-hub-step.ts index 475369f2..b81cc9b0 100644 --- a/src/commands/hub/model/clone-hub-step.ts +++ b/src/commands/hub/model/clone-hub-step.ts @@ -5,6 +5,7 @@ export enum CloneHubStepId { Extension = 'extension', Schema = 'schema', Type = 'type', + Index = 'index', Content = 'content' } diff --git a/src/commands/hub/steps/index-clone-step.spec.ts b/src/commands/hub/steps/index-clone-step.spec.ts new file mode 100644 index 00000000..7ea8e5a7 --- /dev/null +++ b/src/commands/hub/steps/index-clone-step.spec.ts @@ -0,0 +1,231 @@ +import { Arguments } from 'yargs'; +import { FileLog } from '../../../common/file-log'; +import { ensureDirectoryExists } from '../../../common/import/directory-utils'; +import { CloneHubBuilderOptions } from '../../../interfaces/clone-hub-builder-options'; +import { ConfigurationParameters } from '../../configure'; +import { CloneHubState } from '../model/clone-hub-state'; +import { join } from 'path'; +import rmdir from 'rimraf'; + +import * as indexImport from '../../search-index/import'; +import * as indexExport from '../../search-index/export'; + +import { IndexCloneStep } from './index-clone-step'; +import { CloneHubStepId } from '../model/clone-hub-step'; + +jest.mock('../../../services/dynamic-content-client-factory'); +jest.mock('../../search-index/import'); +jest.mock('../../search-index/export'); + +function rimraf(dir: string): Promise { + return new Promise((resolve): void => { + rmdir(dir, resolve); + }); +} + +describe('index clone step', () => { + const yargArgs = { + $0: 'test', + _: ['test'] + }; + + const config = { + clientId: 'client-id', + clientSecret: 'client-id', + hubId: 'hub-id' + }; + + function reset(): void { + jest.resetAllMocks(); + } + + beforeEach(async () => { + reset(); + }); + + beforeAll(async () => { + await rimraf('temp/clone-ext/'); + }); + + afterAll(async () => { + await rimraf('temp/clone-ext/'); + }); + + function generateState(directory: string, logName: string): CloneHubState { + const argv: Arguments = { + ...yargArgs, + ...config, + logFile: new FileLog(), + + dir: directory, + + dstHubId: 'hub2-id', + dstClientId: 'acc2-id', + dstSecret: 'acc2-secret', + revertLog: Promise.resolve(new FileLog()) + }; + + return { + argv: argv, + from: { + clientId: argv.clientId as string, + clientSecret: argv.clientSecret as string, + hubId: argv.hubId as string, + ...yargArgs + }, + to: { + clientId: argv.dstClientId as string, + clientSecret: argv.dstSecret as string, + hubId: argv.dstHubId as string, + ...yargArgs + }, + path: directory, + logFile: new FileLog(join(directory, logName + '.log')) + }; + } + + it('should have the id "index"', () => { + const step = new IndexCloneStep(); + expect(step.getId()).toEqual(CloneHubStepId.Index); + }); + + it('should have the name "Clone Indices"', () => { + const step = new IndexCloneStep(); + expect(step.getName()).toEqual('Clone Indices'); + }); + + it('should call export on the source, backup and import to the destination', async () => { + const state = generateState('temp/clone-ext/run/', 'run'); + + (indexImport.handler as jest.Mock).mockResolvedValue(true); + (indexExport.handler as jest.Mock).mockResolvedValue(true); + + const step = new IndexCloneStep(); + const result = await step.run(state); + // Backup + expect(indexExport.handler).toHaveBeenNthCalledWith(1, { + dir: join(state.path, 'oldIndex'), + force: true, + logFile: state.logFile, + ...state.to + }); + + // Export + expect(indexExport.handler).toHaveBeenNthCalledWith(2, { + dir: join(state.path, 'index'), + force: true, + logFile: state.logFile, + ...state.from + }); + + expect(indexImport.handler).toBeCalledWith({ + dir: join(state.path, 'index'), + logFile: state.logFile, + webhooks: true, + ...state.to + }); + + expect(result).toBeTruthy(); + }); + + it('should fail the step when the export, backup or import fails', async () => { + const state = generateState('temp/clone-ext/run/', 'run'); + + (indexExport.handler as jest.Mock).mockRejectedValue(false); + + const step = new IndexCloneStep(); + const backupFail = await step.run(state); + + expect(backupFail).toBeFalsy(); + expect(indexExport.handler).toBeCalledTimes(1); + expect(indexImport.handler).not.toBeCalled(); + + reset(); + + (indexExport.handler as jest.Mock).mockResolvedValueOnce(true); + (indexExport.handler as jest.Mock).mockRejectedValueOnce(false); + + const exportFail = await step.run(state); + + expect(exportFail).toBeFalsy(); + expect(indexExport.handler).toBeCalledTimes(2); + expect(indexImport.handler).not.toBeCalled(); + + reset(); + + (indexExport.handler as jest.Mock).mockResolvedValue(true); + (indexImport.handler as jest.Mock).mockRejectedValue(false); + + const importFail = await step.run(state); + + expect(importFail).toBeFalsy(); + expect(indexExport.handler).toBeCalledTimes(2); + expect(indexImport.handler).toBeCalled(); + }); + + it('should pass indices with the UPDATE action to the index import command on revert, in the oldIndex folder', async () => { + const state = generateState('temp/clone-ext/revert-update/', 'revert-update'); + + const fakeLog = new FileLog(); + fakeLog.switchGroup('Clone Indices'); + fakeLog.addAction('CREATE', 'index'); + fakeLog.addAction('UPDATE', 'index2'); + + await ensureDirectoryExists('temp/clone-ext/revert-update/oldIndex'); + + state.revertLog = fakeLog; + + const step = new IndexCloneStep(); + const result = await step.revert(state); + + expect(indexImport.handler).toBeCalledWith( + { + dir: join(state.path, 'oldIndex'), + logFile: state.logFile, + ...state.to + }, + ['index2'] + ); + + expect(result).toBeTruthy(); + }); + + it('should not call import indices when no update actions can be reverted', async () => { + const state = generateState('temp/clone-ext/revert-none/', 'revert-none'); + + const fakeLog = new FileLog(); + fakeLog.switchGroup('Clone Indices'); + fakeLog.addAction('CREATE', 'index'); + + await ensureDirectoryExists('temp/clone-ext/revert-none/oldIndex'); + + state.revertLog = fakeLog; + (indexImport.handler as jest.Mock).mockRejectedValue(false); + + const step = new IndexCloneStep(); + const result = await step.revert(state); + + expect(indexImport.handler).not.toHaveBeenCalled(); + + expect(result).toBeTruthy(); + }); + + it('should return false when importing indices for revert fails', async () => { + const state = generateState('temp/clone-ext/revert-update/', 'revert-update'); + + const fakeLog = new FileLog(); + fakeLog.switchGroup('Clone Indices'); + fakeLog.addAction('CREATE', 'index'); + fakeLog.addAction('UPDATE', 'index2'); + + await ensureDirectoryExists('temp/clone-ext/revert-update/oldIndex'); + + state.revertLog = fakeLog; + (indexImport.handler as jest.Mock).mockRejectedValue(false); + + const step = new IndexCloneStep(); + const result = await step.revert(state); + + expect(result).toBeFalsy(); + }); +}); diff --git a/src/commands/hub/steps/index-clone-step.ts b/src/commands/hub/steps/index-clone-step.ts new file mode 100644 index 00000000..8e1b509a --- /dev/null +++ b/src/commands/hub/steps/index-clone-step.ts @@ -0,0 +1,84 @@ +import { CloneHubStep, CloneHubStepId } from '../model/clone-hub-step'; +import { CloneHubState } from '../model/clone-hub-state'; +import { join } from 'path'; + +import { handler as exportIndex } from '../../search-index/export'; +import { handler as importIndex } from '../../search-index/import'; +import { FileLog } from '../../../common/file-log'; +import { existsSync } from 'fs'; + +export class IndexCloneStep implements CloneHubStep { + getId(): CloneHubStepId { + return CloneHubStepId.Index; + } + + getName(): string { + return 'Clone Indices'; + } + + async run(state: CloneHubState): Promise { + try { + state.logFile.appendLine(`Exporting existing indices from destination.`); + await exportIndex({ + dir: join(state.path, 'oldIndex'), + force: true, + logFile: state.logFile, + ...state.to + }); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not export existing indices. \n${e}`); + return false; + } + + try { + state.logFile.appendLine(`Exporting indices from source.`); + await exportIndex({ + dir: join(state.path, 'index'), + force: true, + logFile: state.logFile, + ...state.from + }); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not export indices. \n${e}`); + return false; + } + + try { + await importIndex({ + dir: join(state.path, 'index'), + logFile: state.logFile, + webhooks: true, + ...state.to + }); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not import indices. \n${e}`); + return false; + } + + return true; + } + + async revert(state: CloneHubState): Promise { + // Deleting indices leaves names reserved and unusable, so CREATE actions are ignored. + const toUpdate = (state.revertLog as FileLog).getData('UPDATE', this.getName()); + + // Update using the oldIndex folder. + if (toUpdate.length > 0 && existsSync(join(state.path, 'oldIndex'))) { + try { + await importIndex( + { + dir: join(state.path, 'oldIndex'), + logFile: state.logFile, + ...state.to + }, + toUpdate.map(item => item.split(' ')[0]) + ); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not import old indices. \n${e}`); + return false; + } + } + + return true; + } +} diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts index 2bb014cd..895be203 100644 --- a/src/commands/search-index/export.spec.ts +++ b/src/commands/search-index/export.spec.ts @@ -1037,7 +1037,7 @@ describe('search-index export command', (): void => { }) ]; - await processWebhooks('export-dir', webhooks, new FileLog(), false); + await processWebhooks('export-dir', webhooks, new FileLog()); expect(mockEnsureDirectory).toHaveBeenCalledWith('export-dir/webhooks'); diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index d8872597..93efa238 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -11,7 +11,7 @@ import { } from 'dc-management-sdk-js'; import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; -import { isEqual, result } from 'lodash'; +import { isEqual } from 'lodash'; import { table } from 'table'; import { Arguments, Argv } from 'yargs'; import paginator from '../../common/dc-management-sdk-js/paginator'; @@ -384,8 +384,7 @@ export const processWebhooks = async ( outputDir: string, //previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, webhooksBeingExported: Webhook[], - log: FileLog, - force: boolean + log: FileLog ): Promise => { if (webhooksBeingExported.length === 0) { return; @@ -454,7 +453,7 @@ export const handler = async (argv: Arguments { }); describe('doUpdate', () => { - beforeEach(() => {}); + beforeEach(() => { + /* ... */ + }); it('should update an index and return report', async () => { const mutatedIndex = { From d27f78cc5cf4adb29b37d84d8a2453f135dac4a3 Mon Sep 17 00:00:00 2001 From: Rhys Date: Fri, 13 Aug 2021 14:13:34 +0100 Subject: [PATCH 07/12] fix(search-index): assign content types on search index creation, rather than after --- src/commands/search-index/import.spec.ts | 18 ++++++++++++------ src/commands/search-index/import.ts | 6 +++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts index 889385ef..53d2f8df 100644 --- a/src/commands/search-index/import.spec.ts +++ b/src/commands/search-index/import.spec.ts @@ -24,6 +24,7 @@ import { FileLog } from '../../common/file-log'; import { createLog, getDefaultLogPath } from '../../common/log-helpers'; import { EnrichedSearchIndex } from './export'; import MockPage from '../../common/dc-management-sdk-js/mock-page'; +import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; jest.mock('../../services/dynamic-content-client-factory'); jest.mock('../../view/data-presenter'); @@ -152,6 +153,9 @@ describe('search-index import command', (): void => { }); describe('doCreate', () => { + const assignedContentType = new AssignedContentType({ contentTypeUri: 'http://uri.com' }); + const assignedContentTypes = [{ contentTypeUri: 'http://uri.com' }]; + it('should create an index and return report', async () => { const mockHub = new Hub(); const log = new FileLog(); @@ -159,7 +163,8 @@ describe('search-index import command', (): void => { const mockCreate = jest.fn().mockResolvedValue(newIndex); mockHub.related.searchIndexes.create = mockCreate; jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); - const index = { name: 'index-name', label: 'test-label' }; + const indexBase = { name: 'index-name', label: 'test-label' }; + const index = { ...indexBase, assignedContentTypes: [assignedContentType] }; const webhooks = new Map(); const result = await doCreate(mockHub, index as EnrichedSearchIndex, webhooks, log); @@ -168,7 +173,7 @@ describe('search-index import command', (): void => { "created-id", ] `); - expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining(index)); + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ ...indexBase, assignedContentTypes })); expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index, webhooks); expect(result).toEqual(newIndex); }); @@ -181,7 +186,7 @@ describe('search-index import command', (): void => { throw new Error('Error creating index'); }); mockHub.related.searchIndexes.create = mockCreate; - const index = { name: 'index-name', label: 'test-label' }; + const index = { name: 'index-name', label: 'test-label', assignedContentTypes: [assignedContentType] }; await expect( doCreate(mockHub, index as EnrichedSearchIndex, new Map(), log) @@ -200,7 +205,7 @@ describe('search-index import command', (): void => { 'The create-index action is not available, ensure you have permission to perform this action.' ); mockHub.related.searchIndexes.create = mockCreate; - const index = { name: 'index-name', label: 'test-label' }; + const index = { name: 'index-name', label: 'test-label', assignedContentTypes: [assignedContentType] }; await expect( doCreate(mockHub, index as EnrichedSearchIndex, new Map(), log) @@ -220,14 +225,15 @@ describe('search-index import command', (): void => { .mockRejectedValue( 'The update-index action is not available, ensure you have permission to perform this action.' ); - const index = { name: 'index-name', label: 'test-label' }; + const indexBase = { name: 'index-name', label: 'test-label' }; + const index = { ...indexBase, assignedContentTypes: [assignedContentType] }; const webhooks = new Map(); await expect( doCreate(mockHub, index as EnrichedSearchIndex, webhooks, log) ).rejects.toThrowErrorMatchingSnapshot(); - expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining(index)); + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ ...indexBase, assignedContentTypes })); expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index, webhooks); expect(log.getData('CREATE')).toEqual([]); }); diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index df2f2764..f27e2df4 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -200,7 +200,11 @@ export const doCreate = async ( log: FileLog ): Promise => { try { - const createdIndex = await hub.related.searchIndexes.create(new SearchIndex(getIndexProperties(index))); + const assignedContentTypes = index.assignedContentTypes.map(type => ({ contentTypeUri: type.contentTypeUri })); + + const toCreate = new SearchIndex({ ...getIndexProperties(index), assignedContentTypes }); + + const createdIndex = await hub.related.searchIndexes.create(toCreate); await enrichIndex(createdIndex, index, webhooks); From bf896b85efc2187a20321c58478c6d000d4f117b Mon Sep 17 00:00:00 2001 From: Rhys Date: Mon, 16 Aug 2021 14:07:50 +0100 Subject: [PATCH 08/12] test(search-index): improve test coverage --- src/commands/search-index/export.spec.ts | 55 ++++- src/commands/search-index/export.ts | 1 - src/commands/search-index/import.spec.ts | 244 ++++++++++++++++++++++- src/commands/search-index/import.ts | 4 +- 4 files changed, 293 insertions(+), 11 deletions(-) diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts index 895be203..5c84e8ff 100644 --- a/src/commands/search-index/export.spec.ts +++ b/src/commands/search-index/export.spec.ts @@ -244,29 +244,36 @@ describe('search-index export command', (): void => { }); describe('enrichIndex tests', () => { - it('should request settings, types, keys and webhooks to enrich the given indices', async (): Promise => { - const index = new SearchIndex({ + let index: SearchIndex; + let settings: SearchIndexSettings; + let key: SearchIndexKey; + let assignedContentTypes: AssignedContentType[]; + let enrichedContentTypes: EnrichedAssignedContentType[]; + + beforeEach(() => { + index = new SearchIndex({ + id: 'id-1', name: 'account.suffix-1', suffix: 'suffix-1', label: 'Index 1', type: 'STAGING' }); - const settings = new SearchIndexSettings({ + settings = new SearchIndexSettings({ example: 'setting' }); - const key = new SearchIndexKey({ + key = new SearchIndexKey({ id: 'key-id', key: 'example-key' }); - const assignedContentTypes = [ + assignedContentTypes = [ new AssignedContentType({ contentTypeUri: 'http://1' }), new AssignedContentType({ contentTypeUri: 'http://2' }) ]; - const enrichedContentTypes: EnrichedAssignedContentType[] = []; + enrichedContentTypes = []; assignedContentTypes.forEach((type, index) => { type.related.webhook = jest.fn().mockResolvedValue(new Webhook({ id: 'webhook-' + index })); @@ -295,7 +302,9 @@ describe('search-index export command', (): void => { get: jest.fn().mockResolvedValue(key) } }; + }); + it('should request settings, types, keys and webhooks to enrich the given indices', async (): Promise => { const expectedEnriched = new EnrichedSearchIndex({ ...index.toJSON(), settings: settings, @@ -318,6 +327,40 @@ describe('search-index export command', (): void => { expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); }); + + it('should enrich replicas when present', async (): Promise => { + const allReplicas = new Map(); + + const replica = new SearchIndex({ id: 'replica-1' }); + const enrichedReplica = new EnrichedReplica(replica); + allReplicas.set('id-1', [replica]); + + jest.spyOn(exportModule, 'enrichReplica').mockResolvedValue(enrichedReplica); + + const expectedEnriched = new EnrichedSearchIndex({ + ...index.toJSON(), + settings: settings, + keys: key, + assignedContentTypes: enrichedContentTypes, + replicas: [enrichedReplica] + }); + + const enriched = await exportModule.enrichIndex(new Map(), allReplicas, index); + + expect(exportModule.enrichReplica).toHaveBeenCalledWith(replica, 0, [replica]); + + expect(index.related.settings.get).toHaveBeenCalledTimes(1); + expect(index.related.assignedContentTypes.list).toHaveBeenCalledTimes(1); + expect(index.related.keys.get).toHaveBeenCalledTimes(1); + + for (const type of assignedContentTypes) { + expect(type.related.webhook).toHaveBeenCalledTimes(1); + expect(type.related.activeContentWebhook).toHaveBeenCalledTimes(1); + expect(type.related.archivedContentWebhook).toHaveBeenCalledTimes(1); + } + + expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); + }); }); describe('getExportedWebhooks', () => { diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index 93efa238..c0fdebbf 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -382,7 +382,6 @@ export const filterWebhooks = ( export const processWebhooks = async ( outputDir: string, - //previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, webhooksBeingExported: Webhook[], log: FileLog ): Promise => { diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts index 53d2f8df..34de9579 100644 --- a/src/commands/search-index/import.spec.ts +++ b/src/commands/search-index/import.spec.ts @@ -1,5 +1,5 @@ import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; -import { SearchIndex, Hub, Webhook } from 'dc-management-sdk-js'; +import { SearchIndex, Hub, Webhook, SearchIndexSettings } from 'dc-management-sdk-js'; import * as exportModule from './export'; import * as importModule from './import'; import * as webhookRewriter from './webhook-rewriter'; @@ -8,6 +8,7 @@ import { command, doCreate, doUpdate, + enrichIndex, handler, loadAndRewriteWebhooks, LOG_FILENAME, @@ -22,9 +23,10 @@ import { loadJsonFromDirectory, UpdateStatus } from '../../services/import.servi import chalk from 'chalk'; import { FileLog } from '../../common/file-log'; import { createLog, getDefaultLogPath } from '../../common/log-helpers'; -import { EnrichedSearchIndex } from './export'; +import { EnrichedReplica, EnrichedSearchIndex } from './export'; import MockPage from '../../common/dc-management-sdk-js/mock-page'; import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; +import { join } from 'path'; jest.mock('../../services/dynamic-content-client-factory'); jest.mock('../../view/data-presenter'); @@ -239,6 +241,205 @@ describe('search-index import command', (): void => { }); }); + describe('enrichIndex', () => { + function addWebhookFns(type: AssignedContentType, webhooks: Webhook[] = []): void { + type.related.webhook = jest.fn().mockResolvedValue(webhooks[0]); + type.related.activeContentWebhook = jest.fn().mockResolvedValue(webhooks[1]); + type.related.archivedContentWebhook = jest.fn().mockResolvedValue(webhooks[2]); + type.related.unassign = jest.fn().mockReturnValue(Promise.resolve()); + } + + it('should fetch settings, content types for comparison with the index to be enriched', async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: {}, + assignedContentTypes: [] + }); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.assignedContentTypes.list = jest.fn().mockResolvedValue(new MockPage(AssignedContentType, [])); + index.related.assignedContentTypes.create = jest.fn(); + + await enrichIndex(index, enrichedIndex, undefined); + + expect(enrichedIndex.settings.replicas).toEqual([]); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).not.toHaveBeenCalled(); + }); + + it("should update settings with a union of both source and destination replicas, then update each replica's settings", async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: { + replicas: ['replica-1', 'replica-2'] + }, + assignedContentTypes: [], + replicas: [ + new EnrichedReplica({ + name: 'replica-1', + settings: new SearchIndexSettings({ setting: '1' }) + }), + new EnrichedReplica({ + name: 'replica-2', + settings: new SearchIndexSettings({ setting: '2' }) + }) + ] + }); + + const indexNames = ['replica-1', 'replica-2', 'replica-3']; + const replicas = indexNames.map(name => { + const index = new SearchIndex({ name }); + index.related.update = jest.fn().mockResolvedValue(index); + index.related.settings.update = jest + .fn() + .mockImplementation(settings => Promise.resolve(new SearchIndexSettings(settings))); + return index; + }); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings({ replicas: ['replica-3'] })); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.replicas.list = jest.fn().mockResolvedValue(new MockPage(SearchIndex, replicas)); + index.related.assignedContentTypes.list = jest.fn().mockResolvedValue(new MockPage(AssignedContentType, [])); + index.related.assignedContentTypes.create = jest.fn(); + + await enrichIndex(index, enrichedIndex, undefined); + + expect(enrichedIndex.settings.replicas).toEqual(expect.arrayContaining(['replica-1', 'replica-2', 'replica-3'])); + expect(enrichedIndex.settings.replicas.length).toEqual(3); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.replicas.list).toHaveBeenCalled(); + + for (let i = 0; i < 2; i++) { + expect((replicas[i].related.update as jest.Mock).mock.calls[0][0].name).toEqual(replicas[i].name); + expect(replicas[i].related.settings.update).toHaveBeenCalledWith(enrichedIndex.replicas[i].settings, false); + } + + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).not.toHaveBeenCalled(); + }); + + it('should assign any content types that are not yet assigned on the destination index, removing any that are no longer present', async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: {}, + assignedContentTypes: [ + new AssignedContentType({ contentTypeUri: 'http://toCreate.com' }), + new AssignedContentType({ contentTypeUri: 'http://toUpdate.com' }) + ] + }); + + const existingURIs = ['http://toUpdate.com', 'http://toDelete.com']; + const existingTypes = existingURIs.map(uri => { + const type = new AssignedContentType({ contentTypeUri: uri }); + addWebhookFns(type); + return type; + }); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.assignedContentTypes.list = jest + .fn() + .mockResolvedValue(new MockPage(AssignedContentType, existingTypes)); + + const created: AssignedContentType[] = []; + index.related.assignedContentTypes.create = jest.fn().mockImplementation(type => { + created.push(type); + addWebhookFns(type); + return type; + }); + + await enrichIndex(index, enrichedIndex, undefined); + + expect(created).toEqual([enrichedIndex.assignedContentTypes[0]]); + + expect(existingTypes[0].related.unassign).not.toHaveBeenCalled(); + expect(existingTypes[1].related.unassign).toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).toHaveBeenCalledWith(enrichedIndex.assignedContentTypes[0]); + + expect(enrichedIndex.settings.replicas).toEqual([]); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + }); + + it('should update webhooks for the destination index when they are available', async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: {}, + assignedContentTypes: [ + new AssignedContentType({ + contentTypeUri: 'http://toUpdate.com', + webhook: 'id-1', + activeContentWebhook: 'id-2' + }) + ] + }); + + const type = new AssignedContentType({ contentTypeUri: 'http://toUpdate.com' }); + const webhooks = [new Webhook({ id: 'id-1' }), new Webhook({ id: 'id-2' }), new Webhook({ id: 'id-3' })]; + addWebhookFns(type, webhooks); + + const enrichedWebhooks = new Map(); + enrichedWebhooks.set('id-1', new Webhook({ id: 'id-1', label: 'updated-1' })); + enrichedWebhooks.set('id-2', new Webhook({ id: 'id-2', label: 'updated-2' })); + + jest.spyOn(importModule, 'updateWebhookIfDifferent').mockReturnValue(Promise.resolve()); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.assignedContentTypes.list = jest.fn().mockResolvedValue(new MockPage(AssignedContentType, [type])); + index.related.assignedContentTypes.create = jest.fn(); + + await enrichIndex(index, enrichedIndex, enrichedWebhooks); + + expect(type.related.unassign).not.toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).not.toHaveBeenCalled(); + + expect(type.related.webhook).toHaveBeenCalled(); + expect(type.related.activeContentWebhook).toHaveBeenCalled(); + expect(type.related.archivedContentWebhook).toHaveBeenCalled(); + + expect(importModule.updateWebhookIfDifferent).toHaveBeenNthCalledWith( + 1, + webhooks[0], + enrichedWebhooks.get('id-1') + ); + + expect(importModule.updateWebhookIfDifferent).toHaveBeenNthCalledWith( + 2, + webhooks[1], + enrichedWebhooks.get('id-2') + ); + + expect(importModule.updateWebhookIfDifferent).toHaveBeenNthCalledWith(3, webhooks[2], undefined); + + expect(enrichedIndex.settings.replicas).toEqual([]); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + }); + }); + describe('doUpdate', () => { beforeEach(() => { /* ... */ @@ -803,6 +1004,45 @@ describe('search-index import command', (): void => { ); }); + it('should load webhooks when the webhooks argument is provided', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog(), webhooks: true }; + const fileNamesAndIndicesToImport = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'created' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'content-index-id', + name: 'index-name-2', + label: 'updated' + }) + }; + + const webhookMap = new Map(); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndicesToImport); + jest + .spyOn(importModule, 'storedIndexMapper') + .mockReturnValueOnce(fileNamesAndIndicesToImport['file-1']) + .mockReturnValueOnce(fileNamesAndIndicesToImport['file-2']); + jest.spyOn(importModule, 'processIndices').mockResolvedValueOnce(); + jest.spyOn(importModule, 'loadAndRewriteWebhooks').mockResolvedValueOnce(webhookMap); + + await handler(argv); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(loadAndRewriteWebhooks).toHaveBeenCalledWith(expect.any(Object), join('my-dir', 'webhooks')); + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalled(); + expect(processIndices).toHaveBeenCalledWith( + Object.values(fileNamesAndIndicesToImport), + expect.any(Map), + webhookMap, + expect.any(Object), + expect.any(FileLog) + ); + }); + it('should throw an error when no content found in import directory', async (): Promise => { const argv = { ...yargArgs, ...config, dir: 'my-empty-dir', logFile: new FileLog() }; diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index f27e2df4..436085c4 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -224,7 +224,7 @@ export const doUpdate = async ( log: FileLog ): Promise<{ index: SearchIndex; updateStatus: UpdateStatus }> => { try { - const retrievedIndex: SearchIndex = await hub.related.searchIndexes.get(index.id || ''); + const retrievedIndex: SearchIndex = await hub.related.searchIndexes.get(index.id as string); const dstWebhooks = new Map(); @@ -292,7 +292,7 @@ export const processIndices = async ( index = await doCreate(hub, entry, webhooks, log); status = 'CREATED'; } - data.push([index.id || '', index.name as string, status]); + data.push([index.id as string, index.name as string, status]); } log.appendLine(table(data, streamTableOptions)); From 0cbf80b184359fb077fdc57d2d00337fec116def Mon Sep 17 00:00:00 2001 From: Rhys Date: Mon, 16 Aug 2021 15:47:57 +0100 Subject: [PATCH 09/12] refactor(search-index): indices -> indexes --- src/commands/hub/clone.spec.ts | 2 +- src/commands/hub/clone.ts | 2 +- .../hub/steps/index-clone-step.spec.ts | 16 +- src/commands/hub/steps/index-clone-step.ts | 16 +- .../__snapshots__/export.spec.ts.snap | 16 +- .../__snapshots__/import.spec.ts.snap | 4 +- src/commands/search-index/export.spec.ts | 246 +++++++++--------- src/commands/search-index/export.ts | 76 +++--- src/commands/search-index/import.spec.ts | 90 +++---- src/commands/search-index/import.ts | 44 ++-- 10 files changed, 256 insertions(+), 256 deletions(-) diff --git a/src/commands/hub/clone.spec.ts b/src/commands/hub/clone.spec.ts index 69d2b71e..58bff44d 100644 --- a/src/commands/hub/clone.spec.ts +++ b/src/commands/hub/clone.spec.ts @@ -57,7 +57,7 @@ jest.mock('./steps/type-clone-step', () => ({ })); jest.mock('./steps/index-clone-step', () => ({ - IndexCloneStep: mockStep('Clone Indices', 'index', () => success[4]) + IndexCloneStep: mockStep('Clone Indexes', 'index', () => success[4]) })); jest.mock('./steps/content-clone-step', () => ({ diff --git a/src/commands/hub/clone.ts b/src/commands/hub/clone.ts index 1b0ca15b..45686328 100644 --- a/src/commands/hub/clone.ts +++ b/src/commands/hub/clone.ts @@ -30,7 +30,7 @@ export function getDefaultMappingPath(name: string, platform: string = process.p // hub-*/extensions/ // hub-*/schemas/ // hub-*/types/ -// hub-*/indices/ +// hub-*/indexes/ // hub-*/content/ // hub-*/events/ diff --git a/src/commands/hub/steps/index-clone-step.spec.ts b/src/commands/hub/steps/index-clone-step.spec.ts index 7ea8e5a7..649dbbcd 100644 --- a/src/commands/hub/steps/index-clone-step.spec.ts +++ b/src/commands/hub/steps/index-clone-step.spec.ts @@ -89,9 +89,9 @@ describe('index clone step', () => { expect(step.getId()).toEqual(CloneHubStepId.Index); }); - it('should have the name "Clone Indices"', () => { + it('should have the name "Clone Indexes"', () => { const step = new IndexCloneStep(); - expect(step.getName()).toEqual('Clone Indices'); + expect(step.getName()).toEqual('Clone Indexes'); }); it('should call export on the source, backup and import to the destination', async () => { @@ -163,11 +163,11 @@ describe('index clone step', () => { expect(indexImport.handler).toBeCalled(); }); - it('should pass indices with the UPDATE action to the index import command on revert, in the oldIndex folder', async () => { + it('should pass indexes with the UPDATE action to the index import command on revert, in the oldIndex folder', async () => { const state = generateState('temp/clone-ext/revert-update/', 'revert-update'); const fakeLog = new FileLog(); - fakeLog.switchGroup('Clone Indices'); + fakeLog.switchGroup('Clone Indexes'); fakeLog.addAction('CREATE', 'index'); fakeLog.addAction('UPDATE', 'index2'); @@ -190,11 +190,11 @@ describe('index clone step', () => { expect(result).toBeTruthy(); }); - it('should not call import indices when no update actions can be reverted', async () => { + it('should not call import indexes when no update actions can be reverted', async () => { const state = generateState('temp/clone-ext/revert-none/', 'revert-none'); const fakeLog = new FileLog(); - fakeLog.switchGroup('Clone Indices'); + fakeLog.switchGroup('Clone Indexes'); fakeLog.addAction('CREATE', 'index'); await ensureDirectoryExists('temp/clone-ext/revert-none/oldIndex'); @@ -210,11 +210,11 @@ describe('index clone step', () => { expect(result).toBeTruthy(); }); - it('should return false when importing indices for revert fails', async () => { + it('should return false when importing indexes for revert fails', async () => { const state = generateState('temp/clone-ext/revert-update/', 'revert-update'); const fakeLog = new FileLog(); - fakeLog.switchGroup('Clone Indices'); + fakeLog.switchGroup('Clone Indexes'); fakeLog.addAction('CREATE', 'index'); fakeLog.addAction('UPDATE', 'index2'); diff --git a/src/commands/hub/steps/index-clone-step.ts b/src/commands/hub/steps/index-clone-step.ts index 8e1b509a..a4aa03bb 100644 --- a/src/commands/hub/steps/index-clone-step.ts +++ b/src/commands/hub/steps/index-clone-step.ts @@ -13,12 +13,12 @@ export class IndexCloneStep implements CloneHubStep { } getName(): string { - return 'Clone Indices'; + return 'Clone Indexes'; } async run(state: CloneHubState): Promise { try { - state.logFile.appendLine(`Exporting existing indices from destination.`); + state.logFile.appendLine(`Exporting existing indexes from destination.`); await exportIndex({ dir: join(state.path, 'oldIndex'), force: true, @@ -26,12 +26,12 @@ export class IndexCloneStep implements CloneHubStep { ...state.to }); } catch (e) { - state.logFile.appendLine(`ERROR: Could not export existing indices. \n${e}`); + state.logFile.appendLine(`ERROR: Could not export existing indexes. \n${e}`); return false; } try { - state.logFile.appendLine(`Exporting indices from source.`); + state.logFile.appendLine(`Exporting indexes from source.`); await exportIndex({ dir: join(state.path, 'index'), force: true, @@ -39,7 +39,7 @@ export class IndexCloneStep implements CloneHubStep { ...state.from }); } catch (e) { - state.logFile.appendLine(`ERROR: Could not export indices. \n${e}`); + state.logFile.appendLine(`ERROR: Could not export indexes. \n${e}`); return false; } @@ -51,7 +51,7 @@ export class IndexCloneStep implements CloneHubStep { ...state.to }); } catch (e) { - state.logFile.appendLine(`ERROR: Could not import indices. \n${e}`); + state.logFile.appendLine(`ERROR: Could not import indexes. \n${e}`); return false; } @@ -59,7 +59,7 @@ export class IndexCloneStep implements CloneHubStep { } async revert(state: CloneHubState): Promise { - // Deleting indices leaves names reserved and unusable, so CREATE actions are ignored. + // Deleting indexes leaves names reserved and unusable, so CREATE actions are ignored. const toUpdate = (state.revertLog as FileLog).getData('UPDATE', this.getName()); // Update using the oldIndex folder. @@ -74,7 +74,7 @@ export class IndexCloneStep implements CloneHubStep { toUpdate.map(item => item.split(' ')[0]) ); } catch (e) { - state.logFile.appendLine(`ERROR: Could not import old indices. \n${e}`); + state.logFile.appendLine(`ERROR: Could not import old indexes. \n${e}`); return false; } } diff --git a/src/commands/search-index/__snapshots__/export.spec.ts.snap b/src/commands/search-index/__snapshots__/export.spec.ts.snap index 36e3e4e4..19aefba1 100644 --- a/src/commands/search-index/__snapshots__/export.spec.ts.snap +++ b/src/commands/search-index/__snapshots__/export.spec.ts.snap @@ -1,11 +1,11 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`search-index export command filterIndicesById should throw an error for ids which do not exist in the list of indices 1`] = ` +exports[`search-index export command filterIndexesById should throw an error for ids which do not exist in the list of indexes 1`] = ` "The following ID(s) could not be found: ['index-id-4']. Nothing was exported, exiting." `; -exports[`search-index export command getIndexExports should return a list of indices to export and a list of filenames that will be updated 1`] = ` +exports[`search-index export command getIndexExports should return a list of indexes to export and a list of filenames that will be updated 1`] = ` Array [ Array [ Object { @@ -78,7 +78,7 @@ Array [ ] `; -exports[`search-index export command getIndexExports should return a list of indices to export and a list of filenames that will be updated 2`] = ` +exports[`search-index export command getIndexExports should return a list of indexes to export and a list of filenames that will be updated 2`] = ` Array [ Object { "filename": "export-dir/export-filename-1.json", @@ -109,7 +109,7 @@ Array [ ] `; -exports[`search-index export command getIndexExports should return a list of indices to export and a list of filenames that will be updated 3`] = ` +exports[`search-index export command getIndexExports should return a list of indexes to export and a list of filenames that will be updated 3`] = ` Array [ Object { "filename": "export-dir/export-filename-2.json", @@ -118,7 +118,7 @@ Array [ ] `; -exports[`search-index export command getIndexExports should return a list of indices to export and no filenames that will be updated (first export) 1`] = ` +exports[`search-index export command getIndexExports should return a list of indexes to export and no filenames that will be updated (first export) 1`] = ` Array [ Array [ Object { @@ -153,7 +153,7 @@ Array [ ] `; -exports[`search-index export command getIndexExports should return a list of indices to export and no filenames that will be updated (first export) 2`] = ` +exports[`search-index export command getIndexExports should return a list of indexes to export and no filenames that will be updated (first export) 2`] = ` Array [ Object { "filename": "export-dir/export-filename-1.json", @@ -184,10 +184,10 @@ Array [ ] `; -exports[`search-index export command processIndices should output a message if no indices to export from hub 1`] = ` +exports[`search-index export command processIndexes should output a message if no indexes to export from hub 1`] = ` Array [ Array [ - "No search indices to export from this hub, exiting. + "No search indexes to export from this hub, exiting. ", ], ] diff --git a/src/commands/search-index/__snapshots__/import.spec.ts.snap b/src/commands/search-index/__snapshots__/import.spec.ts.snap index 8a7b9d4e..4b226cf4 100644 --- a/src/commands/search-index/__snapshots__/import.spec.ts.snap +++ b/src/commands/search-index/__snapshots__/import.spec.ts.snap @@ -24,10 +24,10 @@ exports[`search-index import command doUpdate should throw an error when unable exports[`search-index import command doUpdate should throw an error when unable to update index during update if a string error is returned by sdk 1`] = `"Error updating index not-matched-name: undefined"`; -exports[`search-index import command handler tests should throw an error when no content found in import directory 1`] = `"No indices found in my-empty-dir"`; +exports[`search-index import command handler tests should throw an error when no content found in import directory 1`] = `"No indexes found in my-empty-dir"`; exports[`search-index import command validateNoDuplicateIndexNames should throw and error when there are duplicate uris 1`] = ` -"Indices must have unique name values. Duplicate values found:- +"Indexes must have unique name values. Duplicate values found:- name: 'index-name-1' in files: ['file-1', 'file-4'] name: 'index-name-2' in files: ['file-2', 'file-3']" `; diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts index 5c84e8ff..2130597e 100644 --- a/src/commands/search-index/export.spec.ts +++ b/src/commands/search-index/export.spec.ts @@ -3,12 +3,12 @@ import * as directoryUtils from '../../common/import/directory-utils'; import { builder, command, - filterIndicesById, + filterIndexesById, getIndexExports, getExportRecordForIndex, handler, LOG_FILENAME, - processIndices, + processIndexes, EnrichedSearchIndex, EnrichedAssignedContentType, webhookEquals, @@ -68,13 +68,13 @@ describe('search-index export command', (): void => { expect(spyOption).toHaveBeenCalledWith('id', { type: 'string', describe: - 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indices for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indices at the same time.', + 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indexes for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indexes at the same time.', requiresArg: true }); expect(spyOption).toHaveBeenCalledWith('f', { type: 'boolean', boolean: true, - describe: 'Overwrite search indices without asking.' + describe: 'Overwrite search indexes without asking.' }); expect(spyOption).toHaveBeenCalledWith('logFile', { type: 'string', @@ -201,7 +201,7 @@ describe('search-index export command', (): void => { }); describe('enrichReplica tests', () => { - it('should request settings and keys to enrich the given indices', async (): Promise => { + it('should request settings and keys to enrich the given indexes', async (): Promise => { const index = new SearchIndex({ name: 'account.suffix-1', suffix: 'suffix-1', @@ -304,7 +304,7 @@ describe('search-index export command', (): void => { }; }); - it('should request settings, types, keys and webhooks to enrich the given indices', async (): Promise => { + it('should request settings, types, keys and webhooks to enrich the given indexes', async (): Promise => { const expectedEnriched = new EnrichedSearchIndex({ ...index.toJSON(), settings: settings, @@ -410,7 +410,7 @@ describe('search-index export command', (): void => { describe('getIndexExports', () => { let getExportRecordForIndexSpy: jest.SpyInstance; - const indicesToExport = [ + const indexesToExport = [ new EnrichedSearchIndex({ name: 'index-name-1', label: 'index 1', @@ -431,16 +431,16 @@ describe('search-index export command', (): void => { }) ]; - const exportedIndices = { - 'export-dir/export-filename-1.json': indicesToExport[0], - 'export-dir/export-filename-2.json': indicesToExport[1] + const exportedIndexes = { + 'export-dir/export-filename-1.json': indexesToExport[0], + 'export-dir/export-filename-2.json': indexesToExport[1] }; beforeEach(() => { getExportRecordForIndexSpy = jest.spyOn(exportModule, 'getExportRecordForIndex'); }); - it('should return a list of indices to export and no filenames that will be updated (first export)', () => { + it('should return a list of indexes to export and no filenames that will be updated (first export)', () => { const exportedWebhooks = new Map(); jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); @@ -448,15 +448,15 @@ describe('search-index export command', (): void => { .mockReturnValueOnce({ filename: 'export-dir/export-filename-1.json', status: 'CREATED', - index: indicesToExport[0] + index: indexesToExport[0] }) .mockReturnValueOnce({ filename: 'export-dir/export-filename-2.json', status: 'CREATED', - index: indicesToExport[1] + index: indexesToExport[1] }); - const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, indicesToExport, new Map()); + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, indexesToExport, new Map()); expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); @@ -464,7 +464,7 @@ describe('search-index export command', (): void => { expect(updatedExportsMap).toEqual([]); }); - it('should return a list of indices to export and a list of filenames that will be updated', () => { + it('should return a list of indexes to export and a list of filenames that will be updated', () => { const exportedWebhooks = new Map(); jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); @@ -472,18 +472,18 @@ describe('search-index export command', (): void => { .mockReturnValueOnce({ filename: 'export-dir/export-filename-1.json', status: 'CREATED', - index: indicesToExport[0] + index: indexesToExport[0] }) .mockReturnValueOnce({ filename: 'export-dir/export-filename-2.json', status: 'UPDATED', - index: indicesToExport[1] + index: indexesToExport[1] }); const [allExports, updatedExportsMap] = getIndexExports( 'export-dir', - exportedIndices, - indicesToExport, + exportedIndexes, + indexesToExport, new Map() ); @@ -493,7 +493,7 @@ describe('search-index export command', (): void => { expect(updatedExportsMap).toMatchSnapshot(); }); - it('should not return a list of indices to export or a list of filenames that will be updated', () => { + it('should not return a list of indexes to export or a list of filenames that will be updated', () => { const exportedWebhooks = new Map(); jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, [], new Map()); @@ -541,7 +541,7 @@ describe('search-index export command', (): void => { }; it('should create export for any newly exported index', async () => { - const exportedIndices = { + const exportedIndexes = { 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ name: 'index-name-1', label: 'index 1', @@ -561,15 +561,15 @@ describe('search-index export command', (): void => { jest.spyOn(exportServiceModule, 'uniqueFilenamePath').mockReturnValueOnce('export-dir/export-filename-3.json'); - const existingIndices = Object.keys(exportedIndices); + const existingIndexes = Object.keys(exportedIndexes); - const result = getExportRecordForIndex(newIndexToExport, 'export-dir', exportedIndices, new Map(), new Map()); + const result = getExportRecordForIndex(newIndexToExport, 'export-dir', exportedIndexes, new Map(), new Map()); expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledWith( 'export-dir', newIndexToExport.name, 'json', - existingIndices + existingIndexes ); expect(result).toEqual({ filename: 'export-dir/export-filename-3.json', @@ -579,7 +579,7 @@ describe('search-index export command', (): void => { }); it('should update export for any index with different content', async () => { - const exportedIndices = { + const exportedIndexes = { 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ name: 'index-name-1', label: 'index 1', @@ -600,7 +600,7 @@ describe('search-index export command', (): void => { jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); - const result = getExportRecordForIndex(updatedIndexToExport, 'export-dir', exportedIndices, new Map(), new Map()); + const result = getExportRecordForIndex(updatedIndexToExport, 'export-dir', exportedIndexes, new Map(), new Map()); expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); expect(result).toEqual({ @@ -611,7 +611,7 @@ describe('search-index export command', (): void => { }); it('should not update export for any index with same content', async () => { - const exportedIndices = { + const exportedIndexes = { 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ name: 'index-name-1', label: 'index 1', @@ -635,7 +635,7 @@ describe('search-index export command', (): void => { const result = getExportRecordForIndex( unchangedIndexToExport, 'export-dir', - exportedIndices, + exportedIndexes, new Map(), new Map() ); @@ -649,7 +649,7 @@ describe('search-index export command', (): void => { }); }); - describe('filterIndicesById', () => { + describe('filterIndexesById', () => { const listToFilter = [ new EnrichedSearchIndex({ id: 'index-id-1', @@ -665,29 +665,29 @@ describe('search-index export command', (): void => { }) ]; - it('should return the indices matching the given uris', async () => { - const result = filterIndicesById(listToFilter, ['index-id-1', 'index-id-3']); + it('should return the indexes matching the given uris', async () => { + const result = filterIndexesById(listToFilter, ['index-id-1', 'index-id-3']); expect(result).toEqual(expect.arrayContaining([listToFilter[0], listToFilter[2]])); }); - it('should return all the indices because there are no URIs to filter', async () => { - const result = filterIndicesById(listToFilter, []); + it('should return all the indexes because there are no URIs to filter', async () => { + const result = filterIndexesById(listToFilter, []); expect(result).toEqual(listToFilter); }); - it('should throw an error for ids which do not exist in the list of indices', async () => { + it('should throw an error for ids which do not exist in the list of indexes', async () => { expect(() => - filterIndicesById(listToFilter, ['index-id-1', 'index-id-4', 'index-id-3']) + filterIndexesById(listToFilter, ['index-id-1', 'index-id-4', 'index-id-3']) ).toThrowErrorMatchingSnapshot(); }); }); - describe('processIndices', () => { + describe('processIndexes', () => { let mockEnsureDirectory: jest.Mock; let mockTable: jest.Mock; let stdoutSpy: jest.SpyInstance; - const indicesToProcess = [ + const indexesToProcess = [ new EnrichedSearchIndex({ id: 'index-id-1', name: 'index-name-1', @@ -708,7 +708,7 @@ describe('search-index export command', (): void => { }) ]; - const exportedIndices = [ + const exportedIndexes = [ { name: 'index-name-1', label: 'index 1', @@ -739,37 +739,37 @@ describe('search-index export command', (): void => { jest.resetAllMocks(); }); - it('should output export files for the given indices if nothing previously exported', async () => { + it('should output export files for the given indexes if nothing previously exported', async () => { jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ [ { filename: 'export-dir/export-filename-1.json', status: 'CREATED', - index: indicesToProcess[0] + index: indexesToProcess[0] }, { filename: 'export-dir/export-filename-2.json', status: 'CREATED', - index: indicesToProcess[1] + index: indexesToProcess[1] }, { filename: 'export-dir/export-filename-3.json', status: 'CREATED', - index: indicesToProcess[2] + index: indexesToProcess[2] } ], [] ]); - const previouslyExportedIndices = {}; + const previouslyExportedIndexes = {}; const webhooks = new Map(); - await processIndices('export-dir', previouslyExportedIndices, indicesToProcess, webhooks, new FileLog(), false); + await processIndexes('export-dir', previouslyExportedIndexes, indexesToProcess, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', - previouslyExportedIndices, - indicesToProcess, + previouslyExportedIndexes, + indexesToProcess, webhooks ); @@ -779,17 +779,17 @@ describe('search-index export command', (): void => { expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( 1, 'export-dir/export-filename-1.json', - expect.objectContaining(exportedIndices[0]) + expect.objectContaining(exportedIndexes[0]) ); expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( 2, 'export-dir/export-filename-2.json', - expect.objectContaining(exportedIndices[1]) + expect.objectContaining(exportedIndexes[1]) ); expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( 3, 'export-dir/export-filename-3.json', - expect.objectContaining(exportedIndices[2]) + expect.objectContaining(exportedIndexes[2]) ); expect(mockTable).toHaveBeenCalledTimes(1); @@ -797,20 +797,20 @@ describe('search-index export command', (): void => { 1, [ [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], - ['export-dir/export-filename-1.json', indicesToProcess[0].name, 'CREATED'], - ['export-dir/export-filename-2.json', indicesToProcess[1].name, 'CREATED'], - ['export-dir/export-filename-3.json', indicesToProcess[2].name, 'CREATED'] + ['export-dir/export-filename-1.json', indexesToProcess[0].name, 'CREATED'], + ['export-dir/export-filename-2.json', indexesToProcess[1].name, 'CREATED'], + ['export-dir/export-filename-3.json', indexesToProcess[2].name, 'CREATED'] ], streamTableOptions ); }); - it('should output a message if no indices to export from hub', async () => { + it('should output a message if no indexes to export from hub', async () => { jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([[], []]); - const previouslyExportedIndices = {}; + const previouslyExportedIndexes = {}; - await processIndices('export-dir', previouslyExportedIndices, [], new Map(), new FileLog(), false); + await processIndexes('export-dir', previouslyExportedIndexes, [], new Map(), new FileLog(), false); expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(0); @@ -825,34 +825,34 @@ describe('search-index export command', (): void => { { filename: 'export-dir/export-filename-1.json', status: 'UP-TO-DATE', - index: indicesToProcess[0] + index: indexesToProcess[0] }, { filename: 'export-dir/export-filename-2.json', status: 'UP-TO-DATE', - index: indicesToProcess[1] + index: indexesToProcess[1] }, { filename: 'export-dir/export-filename-3.json', status: 'UP-TO-DATE', - index: indicesToProcess[2] + index: indexesToProcess[2] } ], [] ]); - const previouslyExportedIndices = { - 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) + const previouslyExportedIndexes = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndexes[1]) }; const webhooks = new Map(); - const indices = [...indicesToProcess]; - await processIndices('export-dir', previouslyExportedIndices, indices, webhooks, new FileLog(), false); + const indexes = [...indexesToProcess]; + await processIndexes('export-dir', previouslyExportedIndexes, indexes, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', - previouslyExportedIndices, - indices, + previouslyExportedIndexes, + indexes, webhooks ); @@ -864,17 +864,17 @@ describe('search-index export command', (): void => { 1, [ [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], - ['export-dir/export-filename-1.json', indicesToProcess[0].name, 'UP-TO-DATE'], - ['export-dir/export-filename-2.json', indicesToProcess[1].name, 'UP-TO-DATE'], - ['export-dir/export-filename-3.json', indicesToProcess[2].name, 'UP-TO-DATE'] + ['export-dir/export-filename-1.json', indexesToProcess[0].name, 'UP-TO-DATE'], + ['export-dir/export-filename-2.json', indexesToProcess[1].name, 'UP-TO-DATE'], + ['export-dir/export-filename-3.json', indexesToProcess[2].name, 'UP-TO-DATE'] ], streamTableOptions ); }); it('should update the existing export file for a changed index', async () => { - const mutatedIndices = [...indicesToProcess]; - mutatedIndices[1] = new EnrichedSearchIndex({ + const mutatedIndexes = [...indexesToProcess]; + mutatedIndexes[1] = new EnrichedSearchIndex({ id: 'index-id-2', name: 'index-name-2', label: 'index 2 - mutated label', @@ -888,39 +888,39 @@ describe('search-index export command', (): void => { { filename: 'export-dir/export-filename-1.json', status: 'UP-TO-DATE', - index: mutatedIndices[0] + index: mutatedIndexes[0] }, { filename: 'export-dir/export-filename-2.json', status: 'UPDATED', - index: mutatedIndices[1] + index: mutatedIndexes[1] }, { filename: 'export-dir/export-filename-3.json', status: 'UP-TO-DATE', - index: mutatedIndices[2] + index: mutatedIndexes[2] } ], [ { filename: 'export-dir/export-filename-2.json', - uri: mutatedIndices[1].id as string + uri: mutatedIndexes[1].id as string } ] ]); - const previouslyExportedIndices = { - 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) + const previouslyExportedIndexes = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndexes[1]) }; const webhooks = new Map(); - await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, webhooks, new FileLog(), false); + await processIndexes('export-dir', previouslyExportedIndexes, mutatedIndexes, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', - previouslyExportedIndices, - mutatedIndices, + previouslyExportedIndexes, + mutatedIndexes, webhooks ); @@ -932,17 +932,17 @@ describe('search-index export command', (): void => { 1, [ [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], - ['export-dir/export-filename-1.json', indicesToProcess[0].name, 'UP-TO-DATE'], - ['export-dir/export-filename-2.json', indicesToProcess[1].name, 'UPDATED'], - ['export-dir/export-filename-3.json', indicesToProcess[2].name, 'UP-TO-DATE'] + ['export-dir/export-filename-1.json', indexesToProcess[0].name, 'UP-TO-DATE'], + ['export-dir/export-filename-2.json', indexesToProcess[1].name, 'UPDATED'], + ['export-dir/export-filename-3.json', indexesToProcess[2].name, 'UP-TO-DATE'] ], streamTableOptions ); }); it('should not update anything if the user says "n" to the overwrite prompt', async () => { - const mutatedIndices = [...indicesToProcess]; - mutatedIndices[1] = new EnrichedSearchIndex({ + const mutatedIndexes = [...indexesToProcess]; + mutatedIndexes[1] = new EnrichedSearchIndex({ id: 'index-id-2', name: 'index-name-2', label: 'index 2 - mutated label', @@ -955,39 +955,39 @@ describe('search-index export command', (): void => { { filename: 'export-dir/export-filename-1.json', status: 'UP-TO-DATE', - index: mutatedIndices[0] + index: mutatedIndexes[0] }, { filename: 'export-dir/export-filename-2.json', status: 'UPDATED', - index: mutatedIndices[1] + index: mutatedIndexes[1] }, { filename: 'export-dir/export-filename-3.json', status: 'UP-TO-DATE', - index: mutatedIndices[2] + index: mutatedIndexes[2] } ], [ { filename: 'export-dir/export-filename-2.json', - uri: mutatedIndices[1].id as string + uri: mutatedIndexes[1].id as string } ] ]); - const previouslyExportedIndices = { - 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndices[1]) + const previouslyExportedIndexes = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndexes[1]) }; const webhooks = new Map(); - await processIndices('export-dir', previouslyExportedIndices, mutatedIndices, webhooks, new FileLog(), false); + await processIndexes('export-dir', previouslyExportedIndexes, mutatedIndexes, webhooks, new FileLog(), false); expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); expect(exportModule.getIndexExports).toHaveBeenCalledWith( 'export-dir', - previouslyExportedIndices, - mutatedIndices, + previouslyExportedIndexes, + mutatedIndexes, webhooks ); @@ -998,7 +998,7 @@ describe('search-index export command', (): void => { }); describe('filterWebhooks', () => { - it('should return a webhook mapping consisting of only webhooks in the given search indices', () => { + it('should return a webhook mapping consisting of only webhooks in the given search indexes', () => { const webhooks = new Map([ ['id1', new Webhook({ id: 'id1', label: 'webhook1' })], ['id2', new Webhook({ id: 'id2', label: 'webhook2' })], @@ -1006,7 +1006,7 @@ describe('search-index export command', (): void => { ['id4', new Webhook({ id: 'id4', label: 'webhook4' })] ]); - const indices = [ + const indexes = [ new EnrichedSearchIndex({ assignedContentTypes: [ new EnrichedAssignedContentType({ @@ -1027,7 +1027,7 @@ describe('search-index export command', (): void => { }) ]; - const result = filterWebhooks(webhooks, indices); + const result = filterWebhooks(webhooks, indexes); expect(result.size).toEqual(3); expect(result.get('id1')).toEqual(webhooks.get('id1')); @@ -1035,7 +1035,7 @@ describe('search-index export command', (): void => { expect(result.get('id4')).toEqual(webhooks.get('id4')); }); - it('should filter all webhooks if no indices are provided', () => { + it('should filter all webhooks if no indexes are provided', () => { const webhooks = new Map([ ['id1', new Webhook({ id: 'id1', label: 'webhook1' })], ['id2', new Webhook({ id: 'id2', label: 'webhook2' })], @@ -1043,8 +1043,8 @@ describe('search-index export command', (): void => { ['id4', new Webhook({ id: 'id4', label: 'webhook4' })] ]); - const indices: EnrichedSearchIndex[] = []; - const result = filterWebhooks(webhooks, indices); + const indexes: EnrichedSearchIndex[] = []; + const result = filterWebhooks(webhooks, indexes); expect(result.size).toEqual(0); }); @@ -1117,7 +1117,7 @@ describe('search-index export command', (): void => { describe('separateReplicas', () => { it('it should separate replicas into a mapping', () => { - const indices = [ + const indexes = [ new SearchIndex({ id: 'parent', label: 'not-replica', @@ -1140,15 +1140,15 @@ describe('search-index export command', (): void => { }) ]; - const { storedIndices, allReplicas } = exportModule.separateReplicas(indices); + const { storedIndexes, allReplicas } = exportModule.separateReplicas(indexes); - expect(storedIndices).toEqual([indices[0], indices[3]]); + expect(storedIndexes).toEqual([indexes[0], indexes[3]]); expect(allReplicas.size).toEqual(1); - expect(allReplicas.get('parent')).toEqual([indices[1], indices[2]]); + expect(allReplicas.get('parent')).toEqual([indexes[1], indexes[2]]); }); it('it should return no replicas when none are present', () => { - const indices = [ + const indexes = [ new SearchIndex({ id: 'parent', label: 'not-replica', @@ -1161,16 +1161,16 @@ describe('search-index export command', (): void => { }) ]; - const { storedIndices, allReplicas } = exportModule.separateReplicas(indices); + const { storedIndexes, allReplicas } = exportModule.separateReplicas(indexes); - expect(storedIndices).toEqual(indices); + expect(storedIndexes).toEqual(indexes); expect(allReplicas.size).toEqual(0); }); - it('it should return nothing when no indices are provided', () => { - const { storedIndices, allReplicas } = exportModule.separateReplicas([]); + it('it should return nothing when no indexes are provided', () => { + const { storedIndexes, allReplicas } = exportModule.separateReplicas([]); - expect(storedIndices).toEqual([]); + expect(storedIndexes).toEqual([]); expect(allReplicas.size).toEqual(0); }); }); @@ -1186,7 +1186,7 @@ describe('search-index export command', (): void => { hubId: 'hub-id' }; - const indicesToExport: EnrichedSearchIndex[] = [ + const indexesToExport: EnrichedSearchIndex[] = [ new EnrichedSearchIndex({ name: 'account.suffix-1', suffix: 'suffix-1', @@ -1215,7 +1215,7 @@ describe('search-index export command', (): void => { beforeEach(() => { (loadJsonFromDirectory as jest.Mock).mockReturnValue([]); - const listResponse = new MockPage(SearchIndex, indicesToExport); + const listResponse = new MockPage(SearchIndex, indexesToExport); mockList = jest.fn().mockResolvedValue(listResponse); mockGetHub = jest.fn().mockResolvedValue({ @@ -1231,7 +1231,7 @@ describe('search-index export command', (): void => { get: mockGetHub } }); - jest.spyOn(exportModule, 'processIndices').mockResolvedValue(); + jest.spyOn(exportModule, 'processIndexes').mockResolvedValue(); }); it('should use getDefaultLogPath for LOG_FILENAME with process.platform as default', function() { @@ -1240,15 +1240,15 @@ describe('search-index export command', (): void => { expect(getDefaultLogPath).toHaveBeenCalledWith('search-index', 'export', process.platform); }); - it('should export all indices for the current hub if no ids specified', async (): Promise => { + it('should export all indexes for the current hub if no ids specified', async (): Promise => { const schemaIdsToExport: string[] | undefined = undefined; const argv = { ...yargArgs, ...config, dir: 'my-dir', schemaId: schemaIdsToExport, logFile: new FileLog() }; - const filteredIndicesToExport = [...indicesToExport]; + const filteredIndexesToExport = [...indexesToExport]; jest .spyOn(exportModule, 'enrichIndex') .mockImplementation((x, y, z) => Promise.resolve(z as EnrichedSearchIndex)); - jest.spyOn(exportModule, 'filterIndicesById').mockReturnValue(filteredIndicesToExport); + jest.spyOn(exportModule, 'filterIndexesById').mockReturnValue(filteredIndexesToExport); await handler(argv); @@ -1257,26 +1257,26 @@ describe('search-index export command', (): void => { expect(mockList).toHaveBeenCalledWith(undefined, undefined, { size: 100 }); expect(loadJsonFromDirectory).toHaveBeenCalledWith(argv.dir, EnrichedSearchIndex); expect(validateNoDuplicateIndexNames).toHaveBeenCalled(); - expect(exportModule.filterIndicesById).toHaveBeenCalledWith(indicesToExport, []); - expect(exportModule.processIndices).toHaveBeenCalledWith( + expect(exportModule.filterIndexesById).toHaveBeenCalledWith(indexesToExport, []); + expect(exportModule.processIndexes).toHaveBeenCalledWith( argv.dir, [], - filteredIndicesToExport, + filteredIndexesToExport, expect.any(Map), expect.any(FileLog), false ); }); - it('should export only selected indices if ids specified', async (): Promise => { + it('should export only selected indexes if ids specified', async (): Promise => { const idsToExport: string[] | undefined = ['index-id-2']; const argv = { ...yargArgs, ...config, dir: 'my-dir', id: idsToExport, logFile: new FileLog() }; - const filteredIndicesToExport = [indicesToExport[1]]; + const filteredIndexesToExport = [indexesToExport[1]]; jest .spyOn(exportModule, 'enrichIndex') .mockImplementation((x, y, z) => Promise.resolve(z as EnrichedSearchIndex)); - jest.spyOn(exportModule, 'filterIndicesById').mockReturnValue(filteredIndicesToExport); + jest.spyOn(exportModule, 'filterIndexesById').mockReturnValue(filteredIndexesToExport); await handler(argv); @@ -1284,11 +1284,11 @@ describe('search-index export command', (): void => { expect(mockList).toHaveBeenCalled(); expect(loadJsonFromDirectory).toHaveBeenCalledWith(argv.dir, EnrichedSearchIndex); expect(validateNoDuplicateIndexNames).toHaveBeenCalled(); - expect(exportModule.filterIndicesById).toHaveBeenCalledWith(indicesToExport, idsToExport); - expect(exportModule.processIndices).toHaveBeenCalledWith( + expect(exportModule.filterIndexesById).toHaveBeenCalledWith(indexesToExport, idsToExport); + expect(exportModule.processIndexes).toHaveBeenCalledWith( argv.dir, [], - filteredIndicesToExport, + filteredIndexesToExport, expect.any(Map), expect.any(FileLog), false diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts index c0fdebbf..7ad9fc2a 100644 --- a/src/commands/search-index/export.ts +++ b/src/commands/search-index/export.ts @@ -35,7 +35,7 @@ import { ExportBuilderOptions } from '../../interfaces/export-builder-options.in export const command = 'export '; -export const desc = 'Export Search Indices'; +export const desc = 'Export Search Indexes'; export const LOG_FILENAME = (platform: string = process.platform): string => getDefaultLogPath('search-index', 'export', platform); @@ -49,14 +49,14 @@ export const builder = (yargs: Argv): void => { .option('id', { type: 'string', describe: - 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indices for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indices at the same time.', + 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indexes for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indexes at the same time.', requiresArg: true }) .alias('f', 'force') .option('f', { type: 'boolean', boolean: true, - describe: 'Overwrite search indices without asking.' + describe: 'Overwrite search indexes without asking.' }) .option('logFile', { type: 'string', @@ -159,7 +159,7 @@ export class EnrichedSearchIndex extends SearchIndex { } } -export const filterIndicesById = (listToFilter: SearchIndex[], indexIdList: string[]): SearchIndex[] => { +export const filterIndexesById = (listToFilter: SearchIndex[], indexIdList: string[]): SearchIndex[] => { if (indexIdList.length === 0) { return listToFilter; } @@ -234,17 +234,17 @@ interface ExportRecord { export const getExportRecordForIndex = ( index: EnrichedSearchIndex, outputDir: string, - previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, + previouslyExportedIndexes: { [filename: string]: EnrichedSearchIndex }, previouslyExportedWebhooks: Map, webhooksBeingExported: Map ): ExportRecord => { - const indexOfExportedIndex = Object.values(previouslyExportedIndices).findIndex(c => c.name === index.name); + const indexOfExportedIndex = Object.values(previouslyExportedIndexes).findIndex(c => c.name === index.name); if (indexOfExportedIndex < 0) { - const filename = uniqueFilenamePath(outputDir, index.name, 'json', Object.keys(previouslyExportedIndices)); + const filename = uniqueFilenamePath(outputDir, index.name, 'json', Object.keys(previouslyExportedIndexes)); // This filename is now used. - previouslyExportedIndices[filename] = index; + previouslyExportedIndexes[filename] = index; return { filename: filename, @@ -252,8 +252,8 @@ export const getExportRecordForIndex = ( index }; } - const filename = Object.keys(previouslyExportedIndices)[indexOfExportedIndex]; - const previouslyExportedIndex = Object.values(previouslyExportedIndices)[indexOfExportedIndex]; + const filename = Object.keys(previouslyExportedIndexes)[indexOfExportedIndex]; + const previouslyExportedIndex = Object.values(previouslyExportedIndexes)[indexOfExportedIndex]; if (equals(previouslyExportedIndex, index, true, previouslyExportedWebhooks, webhooksBeingExported)) { return { filename, status: 'UP-TO-DATE', index }; @@ -288,15 +288,15 @@ export const getExportedWebhooks = (outputDir: string): Map => export const getIndexExports = ( outputDir: string, - previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, - indicesBeingExported: EnrichedSearchIndex[], + previouslyExportedIndexes: { [filename: string]: EnrichedSearchIndex }, + indexesBeingExported: EnrichedSearchIndex[], webhooksBeingExported: Map ): [ExportRecord[], ExportsMap[]] => { const allExports: ExportRecord[] = []; const updatedExportsMap: ExportsMap[] = []; // uri x filename const previouslyExportedWebhooks = getExportedWebhooks(outputDir); - for (const index of indicesBeingExported) { + for (const index of indexesBeingExported) { if (!index.name) { continue; } @@ -304,7 +304,7 @@ export const getIndexExports = ( const exportRecord = getExportRecordForIndex( index, outputDir, - previouslyExportedIndices, + previouslyExportedIndexes, previouslyExportedWebhooks, webhooksBeingExported ); @@ -318,23 +318,23 @@ export const getIndexExports = ( return [allExports, updatedExportsMap]; }; -export const processIndices = async ( +export const processIndexes = async ( outputDir: string, - previouslyExportedIndices: { [filename: string]: EnrichedSearchIndex }, - indicesBeingExported: EnrichedSearchIndex[], + previouslyExportedIndexes: { [filename: string]: EnrichedSearchIndex }, + indexesBeingExported: EnrichedSearchIndex[], webhooksBeingExported: Map, log: FileLog, force: boolean ): Promise => { - if (indicesBeingExported.length === 0) { - nothingExportedExit(log, 'No search indices to export from this hub, exiting.'); + if (indexesBeingExported.length === 0) { + nothingExportedExit(log, 'No search indexes to export from this hub, exiting.'); return; } const [allExports, updatedExportsMap] = getIndexExports( outputDir, - previouslyExportedIndices, - indicesBeingExported, + previouslyExportedIndexes, + indexesBeingExported, webhooksBeingExported ); if ( @@ -355,7 +355,7 @@ export const processIndices = async ( delete index.id; // do not export id writeJsonToFile(filename, index); } else { - indicesBeingExported.splice(indicesBeingExported.indexOf(index), 1); + indexesBeingExported.splice(indexesBeingExported.indexOf(index), 1); } data.push([filename, index.name as string, status]); } @@ -365,11 +365,11 @@ export const processIndices = async ( export const filterWebhooks = ( webhooks: Map, - filteredIndices: EnrichedSearchIndex[] + filteredIndexes: EnrichedSearchIndex[] ): Map => { const filtered = new Map(); - for (const index of filteredIndices) { + for (const index of filteredIndexes) { for (const type of index.assignedContentTypes) { filtered.set(type.webhook, webhooks.get(type.webhook) as Webhook); filtered.set(type.activeContentWebhook, webhooks.get(type.activeContentWebhook) as Webhook); @@ -409,13 +409,13 @@ export const processWebhooks = async ( }; export const separateReplicas = ( - allIndices: SearchIndex[] -): { storedIndices: SearchIndex[]; allReplicas: Map } => { - const storedIndices: SearchIndex[] = []; + allIndexes: SearchIndex[] +): { storedIndexes: SearchIndex[]; allReplicas: Map } => { + const storedIndexes: SearchIndex[] = []; const allReplicas = new Map(); - for (const index of allIndices) { + for (const index of allIndexes) { if (index.parentId == null) { - storedIndices.push(index); + storedIndexes.push(index); } else { let list = allReplicas.get(index.parentId); @@ -428,7 +428,7 @@ export const separateReplicas = ( } } - return { storedIndices, allReplicas }; + return { storedIndexes, allReplicas }; }; export const handler = async (argv: Arguments): Promise => { @@ -437,21 +437,21 @@ export const handler = async (argv: Arguments(dir, EnrichedSearchIndex); - validateNoDuplicateIndexNames(previouslyExportedIndices); + const previouslyExportedIndexes = loadJsonFromDirectory(dir, EnrichedSearchIndex); + validateNoDuplicateIndexNames(previouslyExportedIndexes); - const allStoredIndices = await paginator(searchIndexList(hub)); - const { storedIndices, allReplicas } = separateReplicas(allStoredIndices); + const allStoredIndexes = await paginator(searchIndexList(hub)); + const { storedIndexes, allReplicas } = separateReplicas(allStoredIndexes); const idArray: string[] = id ? (Array.isArray(id) ? id : [id]) : []; - const filteredIndices = filterIndicesById(storedIndices, idArray); + const filteredIndexes = filterIndexesById(storedIndexes, idArray); const webhooks = new Map(); - const enrichedIndices = await Promise.all(filteredIndices.map(index => enrichIndex(webhooks, allReplicas, index))); + const enrichedIndexes = await Promise.all(filteredIndexes.map(index => enrichIndex(webhooks, allReplicas, index))); - await processIndices(dir, previouslyExportedIndices, enrichedIndices, webhooks, log, force || false); + await processIndexes(dir, previouslyExportedIndexes, enrichedIndexes, webhooks, log, force || false); - const filteredWebhooks = filterWebhooks(webhooks, enrichedIndices); + const filteredWebhooks = filterWebhooks(webhooks, enrichedIndexes); await processWebhooks(dir, Array.from(filteredWebhooks.values()), log); await log.close(); diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts index 34de9579..6eae228f 100644 --- a/src/commands/search-index/import.spec.ts +++ b/src/commands/search-index/import.spec.ts @@ -12,7 +12,7 @@ import { handler, loadAndRewriteWebhooks, LOG_FILENAME, - processIndices, + processIndexes, storedIndexMapper, validateNoDuplicateIndexNames } from './import'; @@ -53,7 +53,7 @@ describe('search-index import command', (): void => { builder(argv); expect(spyPositional).toHaveBeenCalledWith('dir', { - describe: 'Directory containing Search Indices', + describe: 'Directory containing Search Indexes', type: 'string' }); @@ -745,7 +745,7 @@ describe('search-index import command', (): void => { }); }); - describe('processIndices', () => { + describe('processIndexes', () => { let mockTable: jest.Mock; beforeEach(() => { @@ -755,7 +755,7 @@ describe('search-index import command', (): void => { it('should create and update an index', async () => { const hub = new Hub(); - const indicesToProcess = [ + const indexesToProcess = [ new EnrichedSearchIndex({ name: 'index-name', label: 'created', @@ -775,29 +775,29 @@ describe('search-index import command', (): void => { const createdIndex = new EnrichedSearchIndex({ id: 'created-id', - ...indicesToProcess[0].toJSON() + ...indexesToProcess[0].toJSON() }); jest.spyOn(importModule, 'doCreate').mockResolvedValueOnce(createdIndex); const doUpdateResult1 = { - index: indicesToProcess[1], + index: indexesToProcess[1], updateStatus: UpdateStatus.UPDATED }; jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult1); const doUpdateResult2 = { - index: indicesToProcess[2], + index: indexesToProcess[2], updateStatus: UpdateStatus.SKIPPED }; jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult2); const webhooks = new Map(); const replicas = new Map(); - await processIndices(indicesToProcess, replicas, webhooks, hub, new FileLog()); + await processIndexes(indexesToProcess, replicas, webhooks, hub, new FileLog()); - expect(importModule.doCreate).toHaveBeenCalledWith(hub, indicesToProcess[0], webhooks, expect.any(FileLog)); + expect(importModule.doCreate).toHaveBeenCalledWith(hub, indexesToProcess[0], webhooks, expect.any(FileLog)); expect(importModule.doUpdate).toHaveBeenCalledWith( hub, replicas, - indicesToProcess[1], + indexesToProcess[1], webhooks, expect.any(FileLog) ); @@ -818,7 +818,7 @@ describe('search-index import command', (): void => { describe('validateNoDuplicateIndexNames', function() { it('should not throw an error when there are no duplicates', () => { - const indicesToProcess = { + const indexesToProcess = { 'file-1': new EnrichedSearchIndex({ name: 'index-name-1' }), @@ -827,11 +827,11 @@ describe('search-index import command', (): void => { }) }; - expect(() => validateNoDuplicateIndexNames(indicesToProcess)).not.toThrow(); + expect(() => validateNoDuplicateIndexNames(indexesToProcess)).not.toThrow(); }); it('should throw and error when there are duplicate uris', () => { - const indicesToProcess = { + const indexesToProcess = { 'file-1': new EnrichedSearchIndex({ name: 'index-name-1' }), @@ -846,13 +846,13 @@ describe('search-index import command', (): void => { }) }; - expect(() => validateNoDuplicateIndexNames(indicesToProcess)).toThrowErrorMatchingSnapshot(); + expect(() => validateNoDuplicateIndexNames(indexesToProcess)).toThrowErrorMatchingSnapshot(); }); }); - describe('filterIndicesById', function() { - it('should delete indices without a matching id', () => { - const indicesToProcess = { + describe('filterIndexesById', function() { + it('should delete indexes without a matching id', () => { + const indexesToProcess = { 'file-1': new EnrichedSearchIndex({ id: 'index-id-1', name: 'index-name-1' @@ -863,15 +863,15 @@ describe('search-index import command', (): void => { }) }; - const expectedResult = { 'file-2': indicesToProcess['file-2'] }; + const expectedResult = { 'file-2': indexesToProcess['file-2'] }; - importModule.filterIndicesById(['index-id-2'], indicesToProcess); + importModule.filterIndexesById(['index-id-2'], indexesToProcess); - expect(indicesToProcess).toEqual(expectedResult); + expect(indexesToProcess).toEqual(expectedResult); }); - it('should remove all indices if no ids are given', () => { - const indicesToProcess = { + it('should remove all indexes if no ids are given', () => { + const indexesToProcess = { 'file-1': new EnrichedSearchIndex({ id: 'index-id-1', name: 'index-name-1' @@ -882,9 +882,9 @@ describe('search-index import command', (): void => { }) }; - importModule.filterIndicesById([], indicesToProcess); + importModule.filterIndexesById([], indexesToProcess); - expect(indicesToProcess).toEqual({}); + expect(indexesToProcess).toEqual({}); }); }); @@ -939,7 +939,7 @@ describe('search-index import command', (): void => { it('should create an index and update', async (): Promise => { const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; - const fileNamesAndIndicesToImport = { + const fileNamesAndIndexesToImport = { 'file-1': new EnrichedSearchIndex({ name: 'index-name-1', label: 'created' @@ -951,20 +951,20 @@ describe('search-index import command', (): void => { }) }; - (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndicesToImport); + (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndexesToImport); jest .spyOn(importModule, 'storedIndexMapper') - .mockReturnValueOnce(fileNamesAndIndicesToImport['file-1']) - .mockReturnValueOnce(fileNamesAndIndicesToImport['file-2']); - jest.spyOn(importModule, 'processIndices').mockResolvedValueOnce(); + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-1']) + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); + jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); await handler(argv); expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); expect(mockGetHub).toHaveBeenCalledWith('hub-id'); expect(mockList).toHaveBeenCalled(); - expect(processIndices).toHaveBeenCalledWith( - Object.values(fileNamesAndIndicesToImport), + expect(processIndexes).toHaveBeenCalledWith( + Object.values(fileNamesAndIndexesToImport), expect.any(Map), undefined, expect.any(Object), @@ -972,9 +972,9 @@ describe('search-index import command', (): void => { ); }); - it('should call filterIndicesById when a list of ids is provided', async (): Promise => { + it('should call filterIndexesById when a list of ids is provided', async (): Promise => { const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; - const fileNamesAndIndicesToImport = { + const fileNamesAndIndexesToImport = { 'file-1': new EnrichedSearchIndex({ id: 'index-id-1', name: 'index-name-1', @@ -987,16 +987,16 @@ describe('search-index import command', (): void => { }) }; - (loadJsonFromDirectory as jest.Mock).mockReturnValue({ ...fileNamesAndIndicesToImport }); - jest.spyOn(importModule, 'storedIndexMapper').mockReturnValueOnce(fileNamesAndIndicesToImport['file-2']); - jest.spyOn(importModule, 'processIndices').mockResolvedValueOnce(); + (loadJsonFromDirectory as jest.Mock).mockReturnValue({ ...fileNamesAndIndexesToImport }); + jest.spyOn(importModule, 'storedIndexMapper').mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); + jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); await handler(argv, ['index-id-2']); expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); expect(mockGetHub).toHaveBeenCalledWith('hub-id'); - expect(processIndices).toHaveBeenCalledWith( - [fileNamesAndIndicesToImport['file-2']], + expect(processIndexes).toHaveBeenCalledWith( + [fileNamesAndIndexesToImport['file-2']], expect.any(Map), undefined, expect.any(Object), @@ -1006,7 +1006,7 @@ describe('search-index import command', (): void => { it('should load webhooks when the webhooks argument is provided', async (): Promise => { const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog(), webhooks: true }; - const fileNamesAndIndicesToImport = { + const fileNamesAndIndexesToImport = { 'file-1': new EnrichedSearchIndex({ name: 'index-name-1', label: 'created' @@ -1020,12 +1020,12 @@ describe('search-index import command', (): void => { const webhookMap = new Map(); - (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndicesToImport); + (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndexesToImport); jest .spyOn(importModule, 'storedIndexMapper') - .mockReturnValueOnce(fileNamesAndIndicesToImport['file-1']) - .mockReturnValueOnce(fileNamesAndIndicesToImport['file-2']); - jest.spyOn(importModule, 'processIndices').mockResolvedValueOnce(); + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-1']) + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); + jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); jest.spyOn(importModule, 'loadAndRewriteWebhooks').mockResolvedValueOnce(webhookMap); await handler(argv); @@ -1034,8 +1034,8 @@ describe('search-index import command', (): void => { expect(loadAndRewriteWebhooks).toHaveBeenCalledWith(expect.any(Object), join('my-dir', 'webhooks')); expect(mockGetHub).toHaveBeenCalledWith('hub-id'); expect(mockList).toHaveBeenCalled(); - expect(processIndices).toHaveBeenCalledWith( - Object.values(fileNamesAndIndicesToImport), + expect(processIndexes).toHaveBeenCalledWith( + Object.values(fileNamesAndIndexesToImport), expect.any(Map), webhookMap, expect.any(Object), diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index 436085c4..2880bde1 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -23,7 +23,7 @@ export const LOG_FILENAME = (platform: string = process.platform): string => export const builder = (yargs: Argv): void => { yargs.positional('dir', { - describe: 'Directory containing Search Indices', + describe: 'Directory containing Search Indexes', type: 'string' }); @@ -55,11 +55,11 @@ export const replicaList = (index: SearchIndex, projection?: string) => { type IndexName = string; type IndexFile = string; -export const validateNoDuplicateIndexNames = (importedIndices: { +export const validateNoDuplicateIndexNames = (importedIndexes: { [filename: string]: EnrichedSearchIndex; }): void | never => { const nameToFilenameMap = new Map(); // map: name x filenames - for (const [filename, index] of Object.entries(importedIndices)) { + for (const [filename, index] of Object.entries(importedIndexes)) { if (index.name) { const otherFilenames: string[] = nameToFilenameMap.get(index.name) || []; if (filename) { @@ -76,28 +76,28 @@ export const validateNoDuplicateIndexNames = (importedIndices: { if (uniqueDuplicateNames.length > 0) { throw new Error( - `Indices must have unique name values. Duplicate values found:-\n${uniqueDuplicateNames + `Indexes must have unique name values. Duplicate values found:-\n${uniqueDuplicateNames .map(([name, filenames]) => ` name: '${name}' in files: [${filenames.map(f => `'${f}'`).join(', ')}]`) .join('\n')}` ); } }; -export const filterIndicesById = ( +export const filterIndexesById = ( idFilter: string[], - importedIndices: { + importedIndexes: { [filename: string]: SearchIndex; } ): void | never => { - for (const [filename, index] of Object.entries(importedIndices)) { + for (const [filename, index] of Object.entries(importedIndexes)) { if (idFilter.indexOf(index.id as string) === -1) { - delete importedIndices[filename]; + delete importedIndexes[filename]; } } }; -export const storedIndexMapper = (index: EnrichedSearchIndex, storedIndices: SearchIndex[]): EnrichedSearchIndex => { - const found = storedIndices.find(stored => stored.name === index.name); +export const storedIndexMapper = (index: EnrichedSearchIndex, storedIndexes: SearchIndex[]): EnrichedSearchIndex => { + const found = storedIndexes.find(stored => stored.name === index.name); const mutatedIndex = found ? { ...index, id: found.id } : index; return new EnrichedSearchIndex(mutatedIndex); @@ -271,8 +271,8 @@ export const loadAndRewriteWebhooks = async (hub: Hub, dir: string): Promise, webhooks: Map | undefined, hub: Hub, @@ -281,7 +281,7 @@ export const processIndices = async ( const data: string[][] = []; data.push([chalk.bold('ID'), chalk.bold('Name'), chalk.bold('Result')]); - for (const entry of indicesToProcess) { + for (const entry of indexesToProcess) { let status: ImportResult; let index: SearchIndex; if (entry.id) { @@ -306,24 +306,24 @@ export const handler = async ( const client = dynamicContentClientFactory(argv); const hub = await client.hubs.get(argv.hubId); const log = logFile.open(); - const indices = loadJsonFromDirectory(dir, EnrichedSearchIndex); - if (Object.keys(indices).length === 0) { - throw new Error(`No indices found in ${dir}`); + const indexes = loadJsonFromDirectory(dir, EnrichedSearchIndex); + if (Object.keys(indexes).length === 0) { + throw new Error(`No indexes found in ${dir}`); } - validateNoDuplicateIndexNames(indices); + validateNoDuplicateIndexNames(indexes); if (idFilter) { - filterIndicesById(idFilter, indices); + filterIndexesById(idFilter, indexes); } - const allStoredIndices = await paginator(searchIndexList(hub)); - const { storedIndices, allReplicas } = separateReplicas(allStoredIndices); + const allStoredIndexes = await paginator(searchIndexList(hub)); + const { storedIndexes, allReplicas } = separateReplicas(allStoredIndexes); - const indicesToProcess = Object.values(indices).map(index => storedIndexMapper(index, storedIndices)); + const indexesToProcess = Object.values(indexes).map(index => storedIndexMapper(index, storedIndexes)); const webhooks = argv.webhooks ? await loadAndRewriteWebhooks(hub, join(dir, 'webhooks')) : undefined; - await processIndices(indicesToProcess, allReplicas, webhooks, hub, log); + await processIndexes(indexesToProcess, allReplicas, webhooks, hub, log); await log.close(); }; From c7d5e8c257d7170b68d069f182c850da5b1ba092 Mon Sep 17 00:00:00 2001 From: Rhys Date: Mon, 16 Aug 2021 17:51:51 +0100 Subject: [PATCH 10/12] fix(search-index): rewrite the index name to match the target hub --- src/commands/search-index/import.spec.ts | 31 +++++++++++++++++++++++- src/commands/search-index/import.ts | 19 +++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts index 6eae228f..ca937d69 100644 --- a/src/commands/search-index/import.spec.ts +++ b/src/commands/search-index/import.spec.ts @@ -14,7 +14,8 @@ import { LOG_FILENAME, processIndexes, storedIndexMapper, - validateNoDuplicateIndexNames + validateNoDuplicateIndexNames, + rewriteIndexNames } from './import'; import Yargs from 'yargs/yargs'; import { table } from 'table'; @@ -850,6 +851,26 @@ describe('search-index import command', (): void => { }); }); + describe('rewriteIndexNames', function() { + it("should rewrite index names to contain the given hub's name", () => { + const indexesToProcess = { + 'file-1': new EnrichedSearchIndex({ + name: 'oldHub.index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + name: 'index-name-2' + }) + }; + + const hub = new Hub({ name: 'newHub' }); + + expect(() => rewriteIndexNames(hub, indexesToProcess)).not.toThrow(); + + expect(indexesToProcess['file-1'].name).toEqual('newHub.index-name-1'); + expect(indexesToProcess['file-2'].name).toEqual('newHub.index-name-2'); + }); + }); + describe('filterIndexesById', function() { it('should delete indexes without a matching id', () => { const indexesToProcess = { @@ -957,10 +978,12 @@ describe('search-index import command', (): void => { .mockReturnValueOnce(fileNamesAndIndexesToImport['file-1']) .mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); + jest.spyOn(importModule, 'rewriteIndexNames'); await handler(argv); expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(rewriteIndexNames).toHaveBeenCalledWith(expect.any(Object), fileNamesAndIndexesToImport); expect(mockGetHub).toHaveBeenCalledWith('hub-id'); expect(mockList).toHaveBeenCalled(); expect(processIndexes).toHaveBeenCalledWith( @@ -990,10 +1013,14 @@ describe('search-index import command', (): void => { (loadJsonFromDirectory as jest.Mock).mockReturnValue({ ...fileNamesAndIndexesToImport }); jest.spyOn(importModule, 'storedIndexMapper').mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); + jest.spyOn(importModule, 'rewriteIndexNames'); await handler(argv, ['index-id-2']); expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(rewriteIndexNames).toHaveBeenCalledWith(expect.any(Object), { + 'file-2': fileNamesAndIndexesToImport['file-2'] + }); expect(mockGetHub).toHaveBeenCalledWith('hub-id'); expect(processIndexes).toHaveBeenCalledWith( [fileNamesAndIndexesToImport['file-2']], @@ -1027,10 +1054,12 @@ describe('search-index import command', (): void => { .mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); jest.spyOn(importModule, 'loadAndRewriteWebhooks').mockResolvedValueOnce(webhookMap); + jest.spyOn(importModule, 'rewriteIndexNames'); await handler(argv); expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(rewriteIndexNames).toHaveBeenCalledWith(expect.any(Object), fileNamesAndIndexesToImport); expect(loadAndRewriteWebhooks).toHaveBeenCalledWith(expect.any(Object), join('my-dir', 'webhooks')); expect(mockGetHub).toHaveBeenCalledWith('hub-id'); expect(mockList).toHaveBeenCalled(); diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts index 2880bde1..12dbca03 100644 --- a/src/commands/search-index/import.ts +++ b/src/commands/search-index/import.ts @@ -83,6 +83,24 @@ export const validateNoDuplicateIndexNames = (importedIndexes: { } }; +export const rewriteIndexNames = ( + hub: Hub, + importedIndexes: { + [filename: string]: EnrichedSearchIndex; + } +): void | never => { + for (const index of Object.values(importedIndexes)) { + const name = index.name as string; + const firstDot = name.indexOf('.'); + + if (firstDot == -1) { + index.name = `${hub.name}.${name}`; + } else { + index.name = `${hub.name}${name.substring(firstDot)}`; + } + } +}; + export const filterIndexesById = ( idFilter: string[], importedIndexes: { @@ -312,6 +330,7 @@ export const handler = async ( } validateNoDuplicateIndexNames(indexes); + rewriteIndexNames(hub, indexes); if (idFilter) { filterIndexesById(idFilter, indexes); From 5fafc0d44f61e24afa546449415ae0d32f659695 Mon Sep 17 00:00:00 2001 From: Rhys Date: Thu, 19 Aug 2021 09:54:06 +0100 Subject: [PATCH 11/12] fix(search-index): remove unrelated change --- src/commands/hub/steps/settings-clone-step.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/hub/steps/settings-clone-step.ts b/src/commands/hub/steps/settings-clone-step.ts index d63abf2f..c04af375 100644 --- a/src/commands/hub/steps/settings-clone-step.ts +++ b/src/commands/hub/steps/settings-clone-step.ts @@ -58,7 +58,7 @@ export class SettingsCloneStep implements CloneHubStep { await importSettings({ filePath: join(state.path, 'settings', matchingFile), - //mapFile: state.argv.mapFile, + mapFile: state.argv.mapFile, force: state.argv.force, logFile: state.logFile, ...state.to From 2f873a951145127d520de0db779a65ef67ad8b45 Mon Sep 17 00:00:00 2001 From: Rhys Date: Fri, 17 Sep 2021 09:39:59 +0100 Subject: [PATCH 12/12] build: update dc-management-sdk-js --- package-lock.json | 13 +++++++------ package.json | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 872865fb..ee5a60f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "ajv": "^6.12.3", "axios": "^0.21.1", "chalk": "^2.4.2", - "dc-management-sdk-js": "github:rs-amp/dc-management-sdk-js#feature/search-index-webhook", + "dc-management-sdk-js": "^1.14.0", "lodash": "^4.17.21", "node-fetch": "^2.6.1", "promise-retry": "^2.0.1", @@ -4391,9 +4391,9 @@ } }, "node_modules/dc-management-sdk-js": { - "version": "1.13.0", - "resolved": "git+ssh://git@github.com/rs-amp/dc-management-sdk-js.git#9dd4a827bc00401daae8c7cdd2a30d0a143a1654", - "license": "Apache-2.0", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.14.0.tgz", + "integrity": "sha512-M92cWMkwq8EscX7zklxEVNFNYadoyZEbz5+rta8JpPOxMGW9ysDitJETZ+4BNJxlnzi377PNQdlHyHfgnjKprg==", "dependencies": { "axios": "^0.21.1", "url-template": "^2.0.8" @@ -15713,8 +15713,9 @@ "dev": true }, "dc-management-sdk-js": { - "version": "git+ssh://git@github.com/rs-amp/dc-management-sdk-js.git#9dd4a827bc00401daae8c7cdd2a30d0a143a1654", - "from": "dc-management-sdk-js@github:rs-amp/dc-management-sdk-js#feature/search-index-webhook", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.14.0.tgz", + "integrity": "sha512-M92cWMkwq8EscX7zklxEVNFNYadoyZEbz5+rta8JpPOxMGW9ysDitJETZ+4BNJxlnzi377PNQdlHyHfgnjKprg==", "requires": { "axios": "^0.21.1", "url-template": "^2.0.8" diff --git a/package.json b/package.json index 9cafa96a..6e212541 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "ajv": "^6.12.3", "axios": "^0.21.1", "chalk": "^2.4.2", - "dc-management-sdk-js": "github:rs-amp/dc-management-sdk-js#feature/search-index-webhook", + "dc-management-sdk-js": "^1.14.0", "lodash": "^4.17.21", "node-fetch": "^2.6.1", "promise-retry": "^2.0.1",