diff --git a/package-lock.json b/package-lock.json index 2eede7cf..6301b4e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,13 +12,14 @@ "ajv": "^6.12.3", "axios": "^0.21.1", "chalk": "^2.4.2", - "dc-management-sdk-js": "^1.13.0", + "dc-management-sdk-js": "^1.14.0", "lodash": "^4.17.21", "node-fetch": "^2.6.1", "promise-retry": "^2.0.1", "rimraf": "^3.0.0", "sanitize-filename": "^1.6.3", "table": "^5.4.6", + "url-template": "^2.0.8", "yargs": "^14.0.0" }, "bin": { @@ -4390,9 +4391,9 @@ } }, "node_modules/dc-management-sdk-js": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.13.0.tgz", - "integrity": "sha512-E97UYNvDqLQ80SvxV1T73/1k6Qb43+kV043QJIiB5QgYIiyRIleBOIX5NCzZzzb65Ti0D7WOvSqYHoVM8lQ4Ag==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.14.0.tgz", + "integrity": "sha512-M92cWMkwq8EscX7zklxEVNFNYadoyZEbz5+rta8JpPOxMGW9ysDitJETZ+4BNJxlnzi377PNQdlHyHfgnjKprg==", "dependencies": { "axios": "^0.21.1", "url-template": "^2.0.8" @@ -15712,9 +15713,9 @@ "dev": true }, "dc-management-sdk-js": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.13.0.tgz", - "integrity": "sha512-E97UYNvDqLQ80SvxV1T73/1k6Qb43+kV043QJIiB5QgYIiyRIleBOIX5NCzZzzb65Ti0D7WOvSqYHoVM8lQ4Ag==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/dc-management-sdk-js/-/dc-management-sdk-js-1.14.0.tgz", + "integrity": "sha512-M92cWMkwq8EscX7zklxEVNFNYadoyZEbz5+rta8JpPOxMGW9ysDitJETZ+4BNJxlnzi377PNQdlHyHfgnjKprg==", "requires": { "axios": "^0.21.1", "url-template": "^2.0.8" diff --git a/package.json b/package.json index 0a58fbd6..6e212541 100644 --- a/package.json +++ b/package.json @@ -115,13 +115,14 @@ "ajv": "^6.12.3", "axios": "^0.21.1", "chalk": "^2.4.2", - "dc-management-sdk-js": "^1.13.0", + "dc-management-sdk-js": "^1.14.0", "lodash": "^4.17.21", "node-fetch": "^2.6.1", "promise-retry": "^2.0.1", "rimraf": "^3.0.0", "sanitize-filename": "^1.6.3", "table": "^5.4.6", + "url-template": "^2.0.8", "yargs": "^14.0.0" }, "engines": { diff --git a/src/__snapshots__/cli.spec.ts.snap b/src/__snapshots__/cli.spec.ts.snap index 02465bd7..70e5ee40 100644 --- a/src/__snapshots__/cli.spec.ts.snap +++ b/src/__snapshots__/cli.spec.ts.snap @@ -12,6 +12,7 @@ Commands: dc-cli event Event dc-cli extension Extension dc-cli hub Hub + dc-cli search-index Search Index dc-cli settings Settings Options: diff --git a/src/commands/hub/clone.spec.ts b/src/commands/hub/clone.spec.ts index e8366939..6a0e69e8 100644 --- a/src/commands/hub/clone.spec.ts +++ b/src/commands/hub/clone.spec.ts @@ -8,6 +8,7 @@ import * as settings from './steps/settings-clone-step'; import * as schema from './steps/schema-clone-step'; import * as type from './steps/type-clone-step'; import * as extension from './steps/extension-clone-step'; +import * as index from './steps/index-clone-step'; import rmdir from 'rimraf'; import { CloneHubBuilderOptions } from '../../interfaces/clone-hub-builder-options'; @@ -21,7 +22,7 @@ jest.mock('readline'); jest.mock('../../services/dynamic-content-client-factory'); -let success = [true, true, true, true, true]; +let success = [true, true, true, true, true, true]; // eslint-disable-next-line @typescript-eslint/no-explicit-any function succeedOrFail(mock: any, succeed: () => boolean): jest.Mock { @@ -55,8 +56,12 @@ jest.mock('./steps/type-clone-step', () => ({ TypeCloneStep: mockStep('Clone Content Types', 'types', () => success[3]) })); +jest.mock('./steps/index-clone-step', () => ({ + IndexCloneStep: mockStep('Clone Indexes', 'index', () => success[4]) +})); + jest.mock('./steps/content-clone-step', () => ({ - ContentCloneStep: mockStep('Clone Content', 'content', () => success[4]) + ContentCloneStep: mockStep('Clone Content', 'content', () => success[5]) })); jest.mock('../../common/log-helpers', () => ({ @@ -76,6 +81,7 @@ function getMocks(): jest.Mock[] { extension.ExtensionCloneStep as jest.Mock, schema.SchemaCloneStep as jest.Mock, type.TypeCloneStep as jest.Mock, + index.IndexCloneStep as jest.Mock, content.ContentCloneStep as jest.Mock ]; } @@ -264,7 +270,7 @@ describe('hub clone command', () => { it('should call all steps in order with given parameters', async () => { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; const argv: Arguments = { ...yargArgs, @@ -302,9 +308,9 @@ describe('hub clone command', () => { }); it('should handle false returns from each of the steps by stopping the process', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [i != 0, i != 1, i != 2, i != 3, i != 4]; + success = [i != 0, i != 1, i != 2, i != 3, i != 4, i != 5]; const argv: Arguments = { ...yargArgs, @@ -346,9 +352,9 @@ describe('hub clone command', () => { }); it('should start from the step given as a parameter', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; const argv: Arguments = { ...yargArgs, @@ -458,7 +464,7 @@ describe('hub clone command', () => { it('should revert all steps in order with given parameters', async () => { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); await prepareFakeLog(`temp_${process.env.JEST_WORKER_ID}/clone-revert/steps.log`); @@ -498,9 +504,9 @@ describe('hub clone command', () => { }); it('should handle exceptions from each of the revert steps by stopping the process', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [i != 0, i != 1, i != 2, i != 3, i != 4]; + success = [i != 0, i != 1, i != 2, i != 3, i != 4, i != 5]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); await prepareFakeLog(`temp_${process.env.JEST_WORKER_ID}/clone-revert/fail.log`); @@ -547,7 +553,7 @@ describe('hub clone command', () => { it('should exit early if revert log cannot be read', async () => { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); const argv: Arguments = { @@ -584,9 +590,9 @@ describe('hub clone command', () => { }); it('should start reverting from the step given as a parameter (steps in decreasing order)', async () => { - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { clearMocks(); - success = [true, true, true, true, true]; + success = [true, true, true, true, true, true]; await ensureDirectoryExists(`temp_${process.env.JEST_WORKER_ID}/clone-revert/`); await prepareFakeLog(`temp_${process.env.JEST_WORKER_ID}/clone-revert/step.log`); diff --git a/src/commands/hub/clone.ts b/src/commands/hub/clone.ts index 7d68a166..23b88375 100644 --- a/src/commands/hub/clone.ts +++ b/src/commands/hub/clone.ts @@ -10,6 +10,7 @@ import { ContentCloneStep } from './steps/content-clone-step'; import { SchemaCloneStep } from './steps/schema-clone-step'; import { SettingsCloneStep } from './steps/settings-clone-step'; import { TypeCloneStep } from './steps/type-clone-step'; +import { IndexCloneStep } from './steps/index-clone-step'; import { CloneHubState } from './model/clone-hub-state'; import { LogErrorLevel } from '../../common/archive/archive-log'; import { ExtensionCloneStep } from './steps/extension-clone-step'; @@ -28,6 +29,7 @@ export function getDefaultMappingPath(name: string, platform: string = process.p // hub-*/extensions/ // hub-*/schemas/ // hub-*/types/ +// hub-*/indexes/ // hub-*/content/ // hub-*/events/ @@ -44,6 +46,7 @@ export const steps = [ new ExtensionCloneStep(), new SchemaCloneStep(), new TypeCloneStep(), + new IndexCloneStep(), new ContentCloneStep() ]; diff --git a/src/commands/hub/model/clone-hub-step.ts b/src/commands/hub/model/clone-hub-step.ts index 475369f2..b81cc9b0 100644 --- a/src/commands/hub/model/clone-hub-step.ts +++ b/src/commands/hub/model/clone-hub-step.ts @@ -5,6 +5,7 @@ export enum CloneHubStepId { Extension = 'extension', Schema = 'schema', Type = 'type', + Index = 'index', Content = 'content' } diff --git a/src/commands/hub/steps/index-clone-step.spec.ts b/src/commands/hub/steps/index-clone-step.spec.ts new file mode 100644 index 00000000..649dbbcd --- /dev/null +++ b/src/commands/hub/steps/index-clone-step.spec.ts @@ -0,0 +1,231 @@ +import { Arguments } from 'yargs'; +import { FileLog } from '../../../common/file-log'; +import { ensureDirectoryExists } from '../../../common/import/directory-utils'; +import { CloneHubBuilderOptions } from '../../../interfaces/clone-hub-builder-options'; +import { ConfigurationParameters } from '../../configure'; +import { CloneHubState } from '../model/clone-hub-state'; +import { join } from 'path'; +import rmdir from 'rimraf'; + +import * as indexImport from '../../search-index/import'; +import * as indexExport from '../../search-index/export'; + +import { IndexCloneStep } from './index-clone-step'; +import { CloneHubStepId } from '../model/clone-hub-step'; + +jest.mock('../../../services/dynamic-content-client-factory'); +jest.mock('../../search-index/import'); +jest.mock('../../search-index/export'); + +function rimraf(dir: string): Promise { + return new Promise((resolve): void => { + rmdir(dir, resolve); + }); +} + +describe('index clone step', () => { + const yargArgs = { + $0: 'test', + _: ['test'] + }; + + const config = { + clientId: 'client-id', + clientSecret: 'client-id', + hubId: 'hub-id' + }; + + function reset(): void { + jest.resetAllMocks(); + } + + beforeEach(async () => { + reset(); + }); + + beforeAll(async () => { + await rimraf('temp/clone-ext/'); + }); + + afterAll(async () => { + await rimraf('temp/clone-ext/'); + }); + + function generateState(directory: string, logName: string): CloneHubState { + const argv: Arguments = { + ...yargArgs, + ...config, + logFile: new FileLog(), + + dir: directory, + + dstHubId: 'hub2-id', + dstClientId: 'acc2-id', + dstSecret: 'acc2-secret', + revertLog: Promise.resolve(new FileLog()) + }; + + return { + argv: argv, + from: { + clientId: argv.clientId as string, + clientSecret: argv.clientSecret as string, + hubId: argv.hubId as string, + ...yargArgs + }, + to: { + clientId: argv.dstClientId as string, + clientSecret: argv.dstSecret as string, + hubId: argv.dstHubId as string, + ...yargArgs + }, + path: directory, + logFile: new FileLog(join(directory, logName + '.log')) + }; + } + + it('should have the id "index"', () => { + const step = new IndexCloneStep(); + expect(step.getId()).toEqual(CloneHubStepId.Index); + }); + + it('should have the name "Clone Indexes"', () => { + const step = new IndexCloneStep(); + expect(step.getName()).toEqual('Clone Indexes'); + }); + + it('should call export on the source, backup and import to the destination', async () => { + const state = generateState('temp/clone-ext/run/', 'run'); + + (indexImport.handler as jest.Mock).mockResolvedValue(true); + (indexExport.handler as jest.Mock).mockResolvedValue(true); + + const step = new IndexCloneStep(); + const result = await step.run(state); + // Backup + expect(indexExport.handler).toHaveBeenNthCalledWith(1, { + dir: join(state.path, 'oldIndex'), + force: true, + logFile: state.logFile, + ...state.to + }); + + // Export + expect(indexExport.handler).toHaveBeenNthCalledWith(2, { + dir: join(state.path, 'index'), + force: true, + logFile: state.logFile, + ...state.from + }); + + expect(indexImport.handler).toBeCalledWith({ + dir: join(state.path, 'index'), + logFile: state.logFile, + webhooks: true, + ...state.to + }); + + expect(result).toBeTruthy(); + }); + + it('should fail the step when the export, backup or import fails', async () => { + const state = generateState('temp/clone-ext/run/', 'run'); + + (indexExport.handler as jest.Mock).mockRejectedValue(false); + + const step = new IndexCloneStep(); + const backupFail = await step.run(state); + + expect(backupFail).toBeFalsy(); + expect(indexExport.handler).toBeCalledTimes(1); + expect(indexImport.handler).not.toBeCalled(); + + reset(); + + (indexExport.handler as jest.Mock).mockResolvedValueOnce(true); + (indexExport.handler as jest.Mock).mockRejectedValueOnce(false); + + const exportFail = await step.run(state); + + expect(exportFail).toBeFalsy(); + expect(indexExport.handler).toBeCalledTimes(2); + expect(indexImport.handler).not.toBeCalled(); + + reset(); + + (indexExport.handler as jest.Mock).mockResolvedValue(true); + (indexImport.handler as jest.Mock).mockRejectedValue(false); + + const importFail = await step.run(state); + + expect(importFail).toBeFalsy(); + expect(indexExport.handler).toBeCalledTimes(2); + expect(indexImport.handler).toBeCalled(); + }); + + it('should pass indexes with the UPDATE action to the index import command on revert, in the oldIndex folder', async () => { + const state = generateState('temp/clone-ext/revert-update/', 'revert-update'); + + const fakeLog = new FileLog(); + fakeLog.switchGroup('Clone Indexes'); + fakeLog.addAction('CREATE', 'index'); + fakeLog.addAction('UPDATE', 'index2'); + + await ensureDirectoryExists('temp/clone-ext/revert-update/oldIndex'); + + state.revertLog = fakeLog; + + const step = new IndexCloneStep(); + const result = await step.revert(state); + + expect(indexImport.handler).toBeCalledWith( + { + dir: join(state.path, 'oldIndex'), + logFile: state.logFile, + ...state.to + }, + ['index2'] + ); + + expect(result).toBeTruthy(); + }); + + it('should not call import indexes when no update actions can be reverted', async () => { + const state = generateState('temp/clone-ext/revert-none/', 'revert-none'); + + const fakeLog = new FileLog(); + fakeLog.switchGroup('Clone Indexes'); + fakeLog.addAction('CREATE', 'index'); + + await ensureDirectoryExists('temp/clone-ext/revert-none/oldIndex'); + + state.revertLog = fakeLog; + (indexImport.handler as jest.Mock).mockRejectedValue(false); + + const step = new IndexCloneStep(); + const result = await step.revert(state); + + expect(indexImport.handler).not.toHaveBeenCalled(); + + expect(result).toBeTruthy(); + }); + + it('should return false when importing indexes for revert fails', async () => { + const state = generateState('temp/clone-ext/revert-update/', 'revert-update'); + + const fakeLog = new FileLog(); + fakeLog.switchGroup('Clone Indexes'); + fakeLog.addAction('CREATE', 'index'); + fakeLog.addAction('UPDATE', 'index2'); + + await ensureDirectoryExists('temp/clone-ext/revert-update/oldIndex'); + + state.revertLog = fakeLog; + (indexImport.handler as jest.Mock).mockRejectedValue(false); + + const step = new IndexCloneStep(); + const result = await step.revert(state); + + expect(result).toBeFalsy(); + }); +}); diff --git a/src/commands/hub/steps/index-clone-step.ts b/src/commands/hub/steps/index-clone-step.ts new file mode 100644 index 00000000..a4aa03bb --- /dev/null +++ b/src/commands/hub/steps/index-clone-step.ts @@ -0,0 +1,84 @@ +import { CloneHubStep, CloneHubStepId } from '../model/clone-hub-step'; +import { CloneHubState } from '../model/clone-hub-state'; +import { join } from 'path'; + +import { handler as exportIndex } from '../../search-index/export'; +import { handler as importIndex } from '../../search-index/import'; +import { FileLog } from '../../../common/file-log'; +import { existsSync } from 'fs'; + +export class IndexCloneStep implements CloneHubStep { + getId(): CloneHubStepId { + return CloneHubStepId.Index; + } + + getName(): string { + return 'Clone Indexes'; + } + + async run(state: CloneHubState): Promise { + try { + state.logFile.appendLine(`Exporting existing indexes from destination.`); + await exportIndex({ + dir: join(state.path, 'oldIndex'), + force: true, + logFile: state.logFile, + ...state.to + }); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not export existing indexes. \n${e}`); + return false; + } + + try { + state.logFile.appendLine(`Exporting indexes from source.`); + await exportIndex({ + dir: join(state.path, 'index'), + force: true, + logFile: state.logFile, + ...state.from + }); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not export indexes. \n${e}`); + return false; + } + + try { + await importIndex({ + dir: join(state.path, 'index'), + logFile: state.logFile, + webhooks: true, + ...state.to + }); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not import indexes. \n${e}`); + return false; + } + + return true; + } + + async revert(state: CloneHubState): Promise { + // Deleting indexes leaves names reserved and unusable, so CREATE actions are ignored. + const toUpdate = (state.revertLog as FileLog).getData('UPDATE', this.getName()); + + // Update using the oldIndex folder. + if (toUpdate.length > 0 && existsSync(join(state.path, 'oldIndex'))) { + try { + await importIndex( + { + dir: join(state.path, 'oldIndex'), + logFile: state.logFile, + ...state.to + }, + toUpdate.map(item => item.split(' ')[0]) + ); + } catch (e) { + state.logFile.appendLine(`ERROR: Could not import old indexes. \n${e}`); + return false; + } + } + + return true; + } +} diff --git a/src/commands/search-index.ts b/src/commands/search-index.ts new file mode 100644 index 00000000..ff53211d --- /dev/null +++ b/src/commands/search-index.ts @@ -0,0 +1,16 @@ +import { Argv } from 'yargs'; +import YargsCommandBuilderOptions from '../common/yargs/yargs-command-builder-options'; + +export const command = 'search-index'; + +export const desc = 'Search Index'; + +export const builder = (yargs: Argv): Argv => + yargs + .commandDir('search-index', YargsCommandBuilderOptions) + .demandCommand() + .help(); + +export const handler = (): void => { + /* do nothing */ +}; diff --git a/src/commands/search-index/__snapshots__/export.spec.ts.snap b/src/commands/search-index/__snapshots__/export.spec.ts.snap new file mode 100644 index 00000000..19aefba1 --- /dev/null +++ b/src/commands/search-index/__snapshots__/export.spec.ts.snap @@ -0,0 +1,194 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`search-index export command filterIndexesById should throw an error for ids which do not exist in the list of indexes 1`] = ` +"The following ID(s) could not be found: ['index-id-4']. +Nothing was exported, exiting." +`; + +exports[`search-index export command getIndexExports should return a list of indexes to export and a list of filenames that will be updated 1`] = ` +Array [ + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir", + Object { + "export-dir/export-filename-1.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir/export-filename-2.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + }, + Map {}, + Map {}, + ], + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "export-dir", + Object { + "export-dir/export-filename-1.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir/export-filename-2.json": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + }, + Map {}, + Map {}, + ], +] +`; + +exports[`search-index export command getIndexExports should return a list of indexes to export and a list of filenames that will be updated 2`] = ` +Array [ + Object { + "filename": "export-dir/export-filename-1.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "status": "CREATED", + }, + Object { + "filename": "export-dir/export-filename-2.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "status": "UPDATED", + }, +] +`; + +exports[`search-index export command getIndexExports should return a list of indexes to export and a list of filenames that will be updated 3`] = ` +Array [ + Object { + "filename": "export-dir/export-filename-2.json", + "uri": "index-name-2", + }, +] +`; + +exports[`search-index export command getIndexExports should return a list of indexes to export and no filenames that will be updated (first export) 1`] = ` +Array [ + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "export-dir", + Object {}, + Map {}, + Map {}, + ], + Array [ + Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "export-dir", + Object {}, + Map {}, + Map {}, + ], +] +`; + +exports[`search-index export command getIndexExports should return a list of indexes to export and no filenames that will be updated (first export) 2`] = ` +Array [ + Object { + "filename": "export-dir/export-filename-1.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-1", + }, + ], + "label": "index 1", + "name": "index-name-1", + }, + "status": "CREATED", + }, + Object { + "filename": "export-dir/export-filename-2.json", + "index": Object { + "assignedContentTypes": Array [ + Object { + "id": "assigned-type-2", + }, + ], + "label": "index 2", + "name": "index-name-2", + }, + "status": "CREATED", + }, +] +`; + +exports[`search-index export command processIndexes should output a message if no indexes to export from hub 1`] = ` +Array [ + Array [ + "No search indexes to export from this hub, exiting. +", + ], +] +`; diff --git a/src/commands/search-index/__snapshots__/import.spec.ts.snap b/src/commands/search-index/__snapshots__/import.spec.ts.snap new file mode 100644 index 00000000..4b226cf4 --- /dev/null +++ b/src/commands/search-index/__snapshots__/import.spec.ts.snap @@ -0,0 +1,33 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`search-index import command doCreate should throw an error when enrichIndex fails 1`] = ` +"Error creating index index-name: + +The update-index action is not available, ensure you have permission to perform this action." +`; + +exports[`search-index import command doCreate should throw an error when index create fails 1`] = ` +"Error creating index index-name: + +Error: Error creating index" +`; + +exports[`search-index import command doCreate should throw an error when index create fails if a string error is returned by the sdk 1`] = ` +"Error creating index index-name: + +The create-index action is not available, ensure you have permission to perform this action." +`; + +exports[`search-index import command doUpdate should throw an error when unable to get index during update 1`] = `"Error updating index matched-name: Error retrieving index"`; + +exports[`search-index import command doUpdate should throw an error when unable to update index during update 1`] = `"Error updating index not-matched-name: Error saving index"`; + +exports[`search-index import command doUpdate should throw an error when unable to update index during update if a string error is returned by sdk 1`] = `"Error updating index not-matched-name: undefined"`; + +exports[`search-index import command handler tests should throw an error when no content found in import directory 1`] = `"No indexes found in my-empty-dir"`; + +exports[`search-index import command validateNoDuplicateIndexNames should throw and error when there are duplicate uris 1`] = ` +"Indexes must have unique name values. Duplicate values found:- + name: 'index-name-1' in files: ['file-1', 'file-4'] + name: 'index-name-2' in files: ['file-2', 'file-3']" +`; diff --git a/src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap b/src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap new file mode 100644 index 00000000..9d17879a --- /dev/null +++ b/src/commands/search-index/__snapshots__/webhook-rewriter.spec.ts.snap @@ -0,0 +1,48 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`webhook-rewriter tests should escape quotes and backslashes 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"\\"quotedString\\\\\\\\with\\\\\\\\backslash\\"\\" stagingEnvironment=\\"back\\\\\\\\slash\\"}} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should rewrite a simple webhook 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\" stagingEnvironment=\\"staging\\"}} + \\"content\\" : \\"{{~#each (first (pluck contentBlocks \\"content\\") 4)~}}{{~#each this.values~}}{{#if (eq locale \\"en-GB\\")}}{{{truncate value 1000}}}{{/if}}{{~/each~}}{{#unless @last}};{{/unless~}}{{~/each~}}\\" +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should rewrite multiple tags within the same webhook 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\" stagingEnvironment=\\"staging\\"}} + {{example1}} +{{/withDeliveryContentItem}} + \\"key\\": \\"value\\", +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\" stagingEnvironment=\\"staging\\"}} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should rewrite only one argument if the other is not present 1`] = ` +"{ +{{#withDeliveryContentItem contentItemId=payload.id account=\\"account\\"}} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; + +exports[`webhook-rewriter tests should still replace values with unusual whitespace 1`] = ` +"{ +{{#withDeliveryContentItem + contentItemId=payload.id + account=\\"replaced\\" + stagingEnvironment=\\"whitespace\\" }} + {{example1}} +{{/withDeliveryContentItem}} +}" +`; diff --git a/src/commands/search-index/export.spec.ts b/src/commands/search-index/export.spec.ts new file mode 100644 index 00000000..2130597e --- /dev/null +++ b/src/commands/search-index/export.spec.ts @@ -0,0 +1,1298 @@ +import * as exportModule from './export'; +import * as directoryUtils from '../../common/import/directory-utils'; +import { + builder, + command, + filterIndexesById, + getIndexExports, + getExportRecordForIndex, + handler, + LOG_FILENAME, + processIndexes, + EnrichedSearchIndex, + EnrichedAssignedContentType, + webhookEquals, + replicaEquals, + EnrichedReplica, + getExportedWebhooks, + processWebhooks, + filterWebhooks +} from './export'; +import Yargs from 'yargs/yargs'; +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { SearchIndex, SearchIndexSettings, Webhook } from 'dc-management-sdk-js'; +import MockPage from '../../common/dc-management-sdk-js/mock-page'; +import * as exportServiceModule from '../../services/export.service'; +import { table } from 'table'; +import chalk from 'chalk'; +import { loadJsonFromDirectory } from '../../services/import.service'; +import { FileLog } from '../../common/file-log'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { validateNoDuplicateIndexNames } from './import'; +import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; +import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; + +jest.mock('../../services/dynamic-content-client-factory'); +jest.mock('./import'); +jest.mock('../../services/import.service'); +jest.mock('../../common/import/directory-utils'); +jest.mock('table'); +jest.mock('../../common/log-helpers'); + +describe('search-index export command', (): void => { + afterEach((): void => { + jest.restoreAllMocks(); + }); + + afterAll(() => { + jest.resetModules(); + }); + + it('should implement an export command', () => { + expect(command).toEqual('export '); + }); + + describe('builder tests', () => { + it('should configure yargs', () => { + const argv = Yargs(process.argv.slice(2)); + const spyPositional = jest.spyOn(argv, 'positional').mockReturnThis(); + const spyOption = jest.spyOn(argv, 'option').mockReturnThis(); + + builder(argv); + + expect(spyPositional).toHaveBeenCalledWith('dir', { + describe: 'Output directory for the exported Search Index definitions', + type: 'string' + }); + expect(spyOption).toHaveBeenCalledWith('id', { + type: 'string', + describe: + 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indexes for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indexes at the same time.', + requiresArg: true + }); + expect(spyOption).toHaveBeenCalledWith('f', { + type: 'boolean', + boolean: true, + describe: 'Overwrite search indexes without asking.' + }); + expect(spyOption).toHaveBeenCalledWith('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); + }); + }); + + describe('webhookEquals', () => { + it('should match undefined webhooks, and return false if only one argument is undefined', async () => { + expect(webhookEquals(undefined, undefined)).toBeTruthy(); + expect(webhookEquals(new Webhook(), undefined)).toBeFalsy(); + expect(webhookEquals(undefined, new Webhook())).toBeFalsy(); + }); + + it('should compare webhooks on all parameters', async () => { + const exampleWebhook = { + method: 'GET', + secret: 'bananabread', + label: 'webhook', + active: true, + customPayload: { type: 'example', value: 'text' }, + events: ['event1'], + filters: [{ type: 'equal', arguments: [{ jsonPath: 'path' }] }], + handlers: ['handler1'], + headers: [{ key: 'key', value: 'value' }] + }; + + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, unrelatedProperty: true })) + ).toBeTruthy(); + + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, method: 'POST' })) + ).toBeFalsy(); + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, secret: 'applepie' })) + ).toBeFalsy(); + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, label: 'webhook2' })) + ).toBeFalsy(); + expect(webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, active: false }))).toBeFalsy(); + expect( + webhookEquals( + new Webhook(exampleWebhook), + new Webhook({ ...exampleWebhook, customPayload: { type: 'example', value: 'text2' } }) + ) + ).toBeFalsy(); + expect( + webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, events: ['event1', 'event2'] })) + ).toBeFalsy(); + expect( + webhookEquals( + new Webhook(exampleWebhook), + new Webhook({ ...exampleWebhook, filters: [{ type: 'in', arguments: [{ jsonPath: 'path' }] }] }) + ) + ).toBeFalsy(); + expect(webhookEquals(new Webhook(exampleWebhook), new Webhook({ ...exampleWebhook, handlers: [] }))).toBeFalsy(); + expect( + webhookEquals( + new Webhook(exampleWebhook), + new Webhook({ ...exampleWebhook, headers: [{ key: 'key', value: 'value' }, { key: 'key', value: 'value2' }] }) + ) + ).toBeFalsy(); + }); + }); + + describe('replicaEquals', () => { + it('should compare replicas on all settings and label', async () => { + const exampleReplica = { + label: 'replicaIndex', + settings: { example: 'object', example2: 'a' } + }; + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, unrelatedProperty: true }), + false + ) + ).toBeTruthy(); + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, label: 'different' }), + false + ) + ).toBeFalsy(); + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, settings: { example: 'object', example2: 'b' } }), + false + ) + ).toBeFalsy(); + }); + + it('should only compare keys when keys argument is true', async () => { + const exampleReplica = { + label: 'replicaIndex', + keys: { key: 'expected' }, + settings: { example: 'object', example2: 'a' } + }; + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, keys: { key: 'unexpected' } }), + false + ) + ).toBeTruthy(); + + expect( + replicaEquals( + new EnrichedReplica(exampleReplica), + new EnrichedReplica({ ...exampleReplica, keys: { key: 'unexpected' } }), + true + ) + ).toBeFalsy(); + }); + }); + + describe('enrichReplica tests', () => { + it('should request settings and keys to enrich the given indexes', async (): Promise => { + const index = new SearchIndex({ + name: 'account.suffix-1', + suffix: 'suffix-1', + label: 'Index 1', + type: 'STAGING' + }); + + const settings = new SearchIndexSettings({ + example: 'setting' + }); + + const key = new SearchIndexKey({ + id: 'key-id', + key: 'example-key' + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (index as any).related = { + settings: { + get: jest.fn().mockResolvedValue(settings) + }, + keys: { + get: jest.fn().mockResolvedValue(key) + } + }; + + const expectedEnriched = new EnrichedReplica({ + ...index.toJSON(), + settings: settings, + keys: key + }); + + const enriched = await exportModule.enrichReplica(index); + + expect(index.related.settings.get).toHaveBeenCalledTimes(1); + expect(index.related.keys.get).toHaveBeenCalledTimes(1); + + expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); + }); + }); + + describe('enrichIndex tests', () => { + let index: SearchIndex; + let settings: SearchIndexSettings; + let key: SearchIndexKey; + let assignedContentTypes: AssignedContentType[]; + let enrichedContentTypes: EnrichedAssignedContentType[]; + + beforeEach(() => { + index = new SearchIndex({ + id: 'id-1', + name: 'account.suffix-1', + suffix: 'suffix-1', + label: 'Index 1', + type: 'STAGING' + }); + + settings = new SearchIndexSettings({ + example: 'setting' + }); + + key = new SearchIndexKey({ + id: 'key-id', + key: 'example-key' + }); + + assignedContentTypes = [ + new AssignedContentType({ contentTypeUri: 'http://1' }), + new AssignedContentType({ contentTypeUri: 'http://2' }) + ]; + + enrichedContentTypes = []; + + assignedContentTypes.forEach((type, index) => { + type.related.webhook = jest.fn().mockResolvedValue(new Webhook({ id: 'webhook-' + index })); + type.related.activeContentWebhook = jest.fn().mockResolvedValue(new Webhook({ id: 'webhook-active-' + index })); + type.related.archivedContentWebhook = jest + .fn() + .mockResolvedValue(new Webhook({ id: 'webhook-archive-' + index })); + + enrichedContentTypes[index] = new EnrichedAssignedContentType({ + contentTypeUri: type.contentTypeUri, + webhook: 'webhook-' + index, + activeContentWebhook: 'webhook-active-' + index, + archivedContentWebhook: 'webhook-archive-' + index + }); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (index as any).related = { + settings: { + get: jest.fn().mockResolvedValue(settings) + }, + assignedContentTypes: { + list: jest.fn().mockResolvedValue(new MockPage(AssignedContentType, assignedContentTypes)) + }, + keys: { + get: jest.fn().mockResolvedValue(key) + } + }; + }); + + it('should request settings, types, keys and webhooks to enrich the given indexes', async (): Promise => { + const expectedEnriched = new EnrichedSearchIndex({ + ...index.toJSON(), + settings: settings, + keys: key, + assignedContentTypes: enrichedContentTypes, + replicas: [] + }); + + const enriched = await exportModule.enrichIndex(new Map(), new Map(), index); + + expect(index.related.settings.get).toHaveBeenCalledTimes(1); + expect(index.related.assignedContentTypes.list).toHaveBeenCalledTimes(1); + expect(index.related.keys.get).toHaveBeenCalledTimes(1); + + for (const type of assignedContentTypes) { + expect(type.related.webhook).toHaveBeenCalledTimes(1); + expect(type.related.activeContentWebhook).toHaveBeenCalledTimes(1); + expect(type.related.archivedContentWebhook).toHaveBeenCalledTimes(1); + } + + expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); + }); + + it('should enrich replicas when present', async (): Promise => { + const allReplicas = new Map(); + + const replica = new SearchIndex({ id: 'replica-1' }); + const enrichedReplica = new EnrichedReplica(replica); + allReplicas.set('id-1', [replica]); + + jest.spyOn(exportModule, 'enrichReplica').mockResolvedValue(enrichedReplica); + + const expectedEnriched = new EnrichedSearchIndex({ + ...index.toJSON(), + settings: settings, + keys: key, + assignedContentTypes: enrichedContentTypes, + replicas: [enrichedReplica] + }); + + const enriched = await exportModule.enrichIndex(new Map(), allReplicas, index); + + expect(exportModule.enrichReplica).toHaveBeenCalledWith(replica, 0, [replica]); + + expect(index.related.settings.get).toHaveBeenCalledTimes(1); + expect(index.related.assignedContentTypes.list).toHaveBeenCalledTimes(1); + expect(index.related.keys.get).toHaveBeenCalledTimes(1); + + for (const type of assignedContentTypes) { + expect(type.related.webhook).toHaveBeenCalledTimes(1); + expect(type.related.activeContentWebhook).toHaveBeenCalledTimes(1); + expect(type.related.archivedContentWebhook).toHaveBeenCalledTimes(1); + } + + expect(enriched.toJSON()).toEqual(expectedEnriched.toJSON()); + }); + }); + + describe('getExportedWebhooks', () => { + it('should create an id to webhook map from the webhooks loaded from the given directory', () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1' + }), + 'directory/webhooks/webhook2.json': new Webhook({ + id: 'id2', + label: 'webhook2' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + + const result = getExportedWebhooks('directory'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(2); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(result.get('id2')).toEqual(webhooks['directory/webhooks/webhook2.json']); + }); + + it('should ignore webhooks without an id', () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1' + }), + 'directory/webhooks/webhook2.json': new Webhook({ + label: 'webhook2' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + + const result = getExportedWebhooks('directory'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(1); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + }); + }); + + describe('getIndexExports', () => { + let getExportRecordForIndexSpy: jest.SpyInstance; + + const indexesToExport = [ + new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + assignedContentTypes: [ + new AssignedContentType({ + id: 'assigned-type-1' + }) + ] + }), + new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + assignedContentTypes: [ + new AssignedContentType({ + id: 'assigned-type-2' + }) + ] + }) + ]; + + const exportedIndexes = { + 'export-dir/export-filename-1.json': indexesToExport[0], + 'export-dir/export-filename-2.json': indexesToExport[1] + }; + + beforeEach(() => { + getExportRecordForIndexSpy = jest.spyOn(exportModule, 'getExportRecordForIndex'); + }); + + it('should return a list of indexes to export and no filenames that will be updated (first export)', () => { + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + + getExportRecordForIndexSpy + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-1.json', + status: 'CREATED', + index: indexesToExport[0] + }) + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-2.json', + status: 'CREATED', + index: indexesToExport[1] + }); + + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, indexesToExport, new Map()); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); + expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); + expect(allExports).toMatchSnapshot(); + expect(updatedExportsMap).toEqual([]); + }); + + it('should return a list of indexes to export and a list of filenames that will be updated', () => { + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + + getExportRecordForIndexSpy + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-1.json', + status: 'CREATED', + index: indexesToExport[0] + }) + .mockReturnValueOnce({ + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: indexesToExport[1] + }); + + const [allExports, updatedExportsMap] = getIndexExports( + 'export-dir', + exportedIndexes, + indexesToExport, + new Map() + ); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(2); + expect(getExportRecordForIndexSpy.mock.calls).toMatchSnapshot(); + expect(allExports).toMatchSnapshot(); + expect(updatedExportsMap).toMatchSnapshot(); + }); + + it('should not return a list of indexes to export or a list of filenames that will be updated', () => { + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + const [allExports, updatedExportsMap] = getIndexExports('export-dir', {}, [], new Map()); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(0); + expect(allExports).toEqual([]); + expect(updatedExportsMap).toEqual([]); + }); + + it('should skip any that are missing a name', () => { + const exportedWebhooks = new Map(); + jest.spyOn(exportModule, 'getExportedWebhooks').mockReturnValueOnce(exportedWebhooks); + + const [allExports, updatedExportsMap] = getIndexExports( + 'export-dir', + {}, + [ + new EnrichedSearchIndex({ + label: 'index 1' + }) + ], + new Map() + ); + + expect(getExportRecordForIndexSpy).toHaveBeenCalledTimes(0); + expect(allExports).toEqual([]); + expect(updatedExportsMap).toEqual([]); + }); + }); + + describe('getExportRecordForIndex', () => { + const extraResources = { + settings: new SearchIndexSettings({ + setting: 'test' + }), + keys: new SearchIndexKey({ + key: 'test' + }), + assignedContentTypes: [ + new AssignedContentType({ + id: 'assigned-type-1' + }) + ], + replicas: [] + }; + + it('should create export for any newly exported index', async () => { + const exportedIndexes = { + 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + ...extraResources + }), + 'export-dir/export-filename-2.json': new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + ...extraResources + }) + }; + const newIndexToExport = new EnrichedSearchIndex({ + name: 'index-name-3', + label: 'index 3', + ...extraResources + }); + + jest.spyOn(exportServiceModule, 'uniqueFilenamePath').mockReturnValueOnce('export-dir/export-filename-3.json'); + + const existingIndexes = Object.keys(exportedIndexes); + + const result = getExportRecordForIndex(newIndexToExport, 'export-dir', exportedIndexes, new Map(), new Map()); + + expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledWith( + 'export-dir', + newIndexToExport.name, + 'json', + existingIndexes + ); + expect(result).toEqual({ + filename: 'export-dir/export-filename-3.json', + status: 'CREATED', + index: newIndexToExport + }); + }); + + it('should update export for any index with different content', async () => { + const exportedIndexes = { + 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + ...extraResources + }), + 'export-dir/export-filename-2.json': new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + ...extraResources + }) + }; + const updatedIndexToExport = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2 - mutated label', + ...extraResources + }); + + jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); + + const result = getExportRecordForIndex(updatedIndexToExport, 'export-dir', exportedIndexes, new Map(), new Map()); + + expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); + expect(result).toEqual({ + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: updatedIndexToExport + }); + }); + + it('should not update export for any index with same content', async () => { + const exportedIndexes = { + 'export-dir/export-filename-1.json': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'index 1', + ...extraResources + }), + 'export-dir/export-filename-2.json': new EnrichedSearchIndex({ + name: 'index-name-2', + label: 'index 2', + ...extraResources + }) + }; + const unchangedIndexToExport = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2', + ...extraResources + }); + + jest.spyOn(exportServiceModule, 'uniqueFilenamePath'); + + const result = getExportRecordForIndex( + unchangedIndexToExport, + 'export-dir', + exportedIndexes, + new Map(), + new Map() + ); + + expect(exportServiceModule.uniqueFilenamePath).toHaveBeenCalledTimes(0); + expect(result).toEqual({ + filename: 'export-dir/export-filename-2.json', + status: 'UP-TO-DATE', + index: unchangedIndexToExport + }); + }); + }); + + describe('filterIndexesById', () => { + const listToFilter = [ + new EnrichedSearchIndex({ + id: 'index-id-1', + label: 'index 1' + }), + new EnrichedSearchIndex({ + id: 'index-id-2', + label: 'index 2' + }), + new EnrichedSearchIndex({ + id: 'index-id-3', + label: 'index 3' + }) + ]; + + it('should return the indexes matching the given uris', async () => { + const result = filterIndexesById(listToFilter, ['index-id-1', 'index-id-3']); + expect(result).toEqual(expect.arrayContaining([listToFilter[0], listToFilter[2]])); + }); + + it('should return all the indexes because there are no URIs to filter', async () => { + const result = filterIndexesById(listToFilter, []); + expect(result).toEqual(listToFilter); + }); + + it('should throw an error for ids which do not exist in the list of indexes', async () => { + expect(() => + filterIndexesById(listToFilter, ['index-id-1', 'index-id-4', 'index-id-3']) + ).toThrowErrorMatchingSnapshot(); + }); + }); + + describe('processIndexes', () => { + let mockEnsureDirectory: jest.Mock; + let mockTable: jest.Mock; + let stdoutSpy: jest.SpyInstance; + + const indexesToProcess = [ + new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1', + label: 'index 1', + status: 'ACTIVE' + }), + new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2', + status: 'ACTIVE' + }), + new EnrichedSearchIndex({ + id: 'index-id-3', + name: 'index-name-3', + label: 'index 3', + status: 'ACTIVE' + }) + ]; + + const exportedIndexes = [ + { + name: 'index-name-1', + label: 'index 1', + status: 'ACTIVE' + }, + { + name: 'index-name-2', + label: 'index 2', + status: 'ACTIVE' + }, + { + name: 'index-name-3', + label: 'index 3', + status: 'ACTIVE' + } + ]; + + beforeEach(() => { + mockEnsureDirectory = directoryUtils.ensureDirectoryExists as jest.Mock; + mockTable = table as jest.Mock; + mockTable.mockImplementation(jest.requireActual('table').table); + jest.spyOn(exportServiceModule, 'writeJsonToFile').mockImplementation(); + stdoutSpy = jest.spyOn(process.stdout, 'write'); + stdoutSpy.mockImplementation(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should output export files for the given indexes if nothing previously exported', async () => { + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'CREATED', + index: indexesToProcess[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'CREATED', + index: indexesToProcess[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'CREATED', + index: indexesToProcess[2] + } + ], + [] + ]); + + const previouslyExportedIndexes = {}; + const webhooks = new Map(); + await processIndexes('export-dir', previouslyExportedIndexes, indexesToProcess, webhooks, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndexes, + indexesToProcess, + webhooks + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); + + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(3); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 1, + 'export-dir/export-filename-1.json', + expect.objectContaining(exportedIndexes[0]) + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 2, + 'export-dir/export-filename-2.json', + expect.objectContaining(exportedIndexes[1]) + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 3, + 'export-dir/export-filename-3.json', + expect.objectContaining(exportedIndexes[2]) + ); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], + ['export-dir/export-filename-1.json', indexesToProcess[0].name, 'CREATED'], + ['export-dir/export-filename-2.json', indexesToProcess[1].name, 'CREATED'], + ['export-dir/export-filename-3.json', indexesToProcess[2].name, 'CREATED'] + ], + streamTableOptions + ); + }); + + it('should output a message if no indexes to export from hub', async () => { + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([[], []]); + + const previouslyExportedIndexes = {}; + + await processIndexes('export-dir', previouslyExportedIndexes, [], new Map(), new FileLog(), false); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(0); + expect(stdoutSpy.mock.calls).toMatchSnapshot(); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(0); + expect(mockTable).toHaveBeenCalledTimes(0); + }); + + it('should not output any export files if a previous export exists and the index is unchanged', async () => { + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'UP-TO-DATE', + index: indexesToProcess[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'UP-TO-DATE', + index: indexesToProcess[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'UP-TO-DATE', + index: indexesToProcess[2] + } + ], + [] + ]); + + const previouslyExportedIndexes = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndexes[1]) + }; + const webhooks = new Map(); + const indexes = [...indexesToProcess]; + await processIndexes('export-dir', previouslyExportedIndexes, indexes, webhooks, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndexes, + indexes, + webhooks + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(0); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], + ['export-dir/export-filename-1.json', indexesToProcess[0].name, 'UP-TO-DATE'], + ['export-dir/export-filename-2.json', indexesToProcess[1].name, 'UP-TO-DATE'], + ['export-dir/export-filename-3.json', indexesToProcess[2].name, 'UP-TO-DATE'] + ], + streamTableOptions + ); + }); + + it('should update the existing export file for a changed index', async () => { + const mutatedIndexes = [...indexesToProcess]; + mutatedIndexes[1] = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2 - mutated label', + status: 'ACTIVE' + }); + + jest.spyOn(exportServiceModule, 'promptToOverwriteExports').mockResolvedValueOnce(true); + + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'UP-TO-DATE', + index: mutatedIndexes[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: mutatedIndexes[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'UP-TO-DATE', + index: mutatedIndexes[2] + } + ], + [ + { + filename: 'export-dir/export-filename-2.json', + uri: mutatedIndexes[1].id as string + } + ] + ]); + + const previouslyExportedIndexes = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndexes[1]) + }; + const webhooks = new Map(); + + await processIndexes('export-dir', previouslyExportedIndexes, mutatedIndexes, webhooks, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndexes, + mutatedIndexes, + webhooks + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(1); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(1); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')], + ['export-dir/export-filename-1.json', indexesToProcess[0].name, 'UP-TO-DATE'], + ['export-dir/export-filename-2.json', indexesToProcess[1].name, 'UPDATED'], + ['export-dir/export-filename-3.json', indexesToProcess[2].name, 'UP-TO-DATE'] + ], + streamTableOptions + ); + }); + + it('should not update anything if the user says "n" to the overwrite prompt', async () => { + const mutatedIndexes = [...indexesToProcess]; + mutatedIndexes[1] = new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'index 2 - mutated label', + status: 'ACTIVE' + }); + + jest.spyOn(exportServiceModule, 'promptToOverwriteExports').mockResolvedValueOnce(false); + jest.spyOn(exportModule, 'getIndexExports').mockReturnValueOnce([ + [ + { + filename: 'export-dir/export-filename-1.json', + status: 'UP-TO-DATE', + index: mutatedIndexes[0] + }, + { + filename: 'export-dir/export-filename-2.json', + status: 'UPDATED', + index: mutatedIndexes[1] + }, + { + filename: 'export-dir/export-filename-3.json', + status: 'UP-TO-DATE', + index: mutatedIndexes[2] + } + ], + [ + { + filename: 'export-dir/export-filename-2.json', + uri: mutatedIndexes[1].id as string + } + ] + ]); + + const previouslyExportedIndexes = { + 'export-dir/export-filename-2.json': new EnrichedSearchIndex(exportedIndexes[1]) + }; + const webhooks = new Map(); + + await processIndexes('export-dir', previouslyExportedIndexes, mutatedIndexes, webhooks, new FileLog(), false); + + expect(exportModule.getIndexExports).toHaveBeenCalledTimes(1); + expect(exportModule.getIndexExports).toHaveBeenCalledWith( + 'export-dir', + previouslyExportedIndexes, + mutatedIndexes, + webhooks + ); + + expect(mockEnsureDirectory).toHaveBeenCalledTimes(0); + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(0); + expect(mockTable).toHaveBeenCalledTimes(0); + }); + }); + + describe('filterWebhooks', () => { + it('should return a webhook mapping consisting of only webhooks in the given search indexes', () => { + const webhooks = new Map([ + ['id1', new Webhook({ id: 'id1', label: 'webhook1' })], + ['id2', new Webhook({ id: 'id2', label: 'webhook2' })], + ['id3', new Webhook({ id: 'id3', label: 'webhook3' })], + ['id4', new Webhook({ id: 'id4', label: 'webhook4' })] + ]); + + const indexes = [ + new EnrichedSearchIndex({ + assignedContentTypes: [ + new EnrichedAssignedContentType({ + webhook: 'id1', + activeContentWebhook: 'id2', + archivedContentWebhook: 'id4' + }) + ] + }), + new EnrichedSearchIndex({ + assignedContentTypes: [ + new EnrichedAssignedContentType({ + webhook: 'id4', + activeContentWebhook: 'id1', + archivedContentWebhook: 'id2' + }) + ] + }) + ]; + + const result = filterWebhooks(webhooks, indexes); + + expect(result.size).toEqual(3); + expect(result.get('id1')).toEqual(webhooks.get('id1')); + expect(result.get('id2')).toEqual(webhooks.get('id2')); + expect(result.get('id4')).toEqual(webhooks.get('id4')); + }); + + it('should filter all webhooks if no indexes are provided', () => { + const webhooks = new Map([ + ['id1', new Webhook({ id: 'id1', label: 'webhook1' })], + ['id2', new Webhook({ id: 'id2', label: 'webhook2' })], + ['id3', new Webhook({ id: 'id3', label: 'webhook3' })], + ['id4', new Webhook({ id: 'id4', label: 'webhook4' })] + ]); + + const indexes: EnrichedSearchIndex[] = []; + const result = filterWebhooks(webhooks, indexes); + + expect(result.size).toEqual(0); + }); + }); + + describe('processWebhooks', () => { + let mockEnsureDirectory: jest.Mock; + let mockTable: jest.Mock; + let stdoutSpy: jest.SpyInstance; + + beforeEach(() => { + mockEnsureDirectory = directoryUtils.ensureDirectoryExists as jest.Mock; + mockTable = table as jest.Mock; + mockTable.mockImplementation(jest.requireActual('table').table); + jest.spyOn(exportServiceModule, 'writeJsonToFile').mockImplementation(); + stdoutSpy = jest.spyOn(process.stdout, 'write'); + stdoutSpy.mockImplementation(); + }); + + it('should export webhooks, printing a table of all exported files', async () => { + const webhooks = [ + new Webhook({ + id: 'id1', + label: 'webhook1' + }), + new Webhook({ + id: 'id2', + label: 'webhook2' + }), + new Webhook({ + id: 'id3', + label: 'webhook2' + }) + ]; + + await processWebhooks('export-dir', webhooks, new FileLog()); + + expect(mockEnsureDirectory).toHaveBeenCalledWith('export-dir/webhooks'); + + expect(exportServiceModule.writeJsonToFile).toHaveBeenCalledTimes(3); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 1, + 'export-dir/webhooks/webhook1.json', + webhooks[0] + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 2, + 'export-dir/webhooks/webhook2.json', + webhooks[1] + ); + expect(exportServiceModule.writeJsonToFile).toHaveBeenNthCalledWith( + 3, + 'export-dir/webhooks/webhook2-1.json', + webhooks[2] + ); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('File'), chalk.bold('Label'), chalk.bold('Result')], + ['export-dir/webhooks/webhook1.json', webhooks[0].label, 'UPDATED'], + ['export-dir/webhooks/webhook2.json', webhooks[1].label, 'UPDATED'], + ['export-dir/webhooks/webhook2-1.json', webhooks[2].label, 'UPDATED'] + ], + streamTableOptions + ); + }); + }); + + describe('separateReplicas', () => { + it('it should separate replicas into a mapping', () => { + const indexes = [ + new SearchIndex({ + id: 'parent', + label: 'not-replica', + parentId: null + }), + new SearchIndex({ + id: 'child', + label: 'replica', + parentId: 'parent' + }), + new SearchIndex({ + id: 'child2', + label: 'replica2', + parentId: 'parent' + }), + new SearchIndex({ + id: 'not-parent', + label: 'not-replica2', + parentId: null + }) + ]; + + const { storedIndexes, allReplicas } = exportModule.separateReplicas(indexes); + + expect(storedIndexes).toEqual([indexes[0], indexes[3]]); + expect(allReplicas.size).toEqual(1); + expect(allReplicas.get('parent')).toEqual([indexes[1], indexes[2]]); + }); + + it('it should return no replicas when none are present', () => { + const indexes = [ + new SearchIndex({ + id: 'parent', + label: 'not-replica', + parentId: null + }), + new SearchIndex({ + id: 'not-parent', + label: 'not-replica2', + parentId: null + }) + ]; + + const { storedIndexes, allReplicas } = exportModule.separateReplicas(indexes); + + expect(storedIndexes).toEqual(indexes); + expect(allReplicas.size).toEqual(0); + }); + + it('it should return nothing when no indexes are provided', () => { + const { storedIndexes, allReplicas } = exportModule.separateReplicas([]); + + expect(storedIndexes).toEqual([]); + expect(allReplicas.size).toEqual(0); + }); + }); + + describe('handler tests', () => { + const yargArgs = { + $0: 'test', + _: ['test'] + }; + const config = { + clientId: 'client-id', + clientSecret: 'client-id', + hubId: 'hub-id' + }; + + const indexesToExport: EnrichedSearchIndex[] = [ + new EnrichedSearchIndex({ + name: 'account.suffix-1', + suffix: 'suffix-1', + label: 'Index 1', + type: 'STAGING', + settings: {}, + keys: {}, + assignedContentTypes: [], + replicas: [] + }), + new EnrichedSearchIndex({ + name: 'account.suffix-2', + suffix: 'suffix-2', + label: 'Index 2', + type: 'STAGING', + settings: {}, + keys: {}, + assignedContentTypes: [], + replicas: [] + }) + ]; + + let mockGetHub: jest.Mock; + let mockList: jest.Mock; + + beforeEach(() => { + (loadJsonFromDirectory as jest.Mock).mockReturnValue([]); + + const listResponse = new MockPage(SearchIndex, indexesToExport); + mockList = jest.fn().mockResolvedValue(listResponse); + + mockGetHub = jest.fn().mockResolvedValue({ + related: { + searchIndexes: { + list: mockList + } + } + }); + + (dynamicContentClientFactory as jest.Mock).mockReturnValue({ + hubs: { + get: mockGetHub + } + }); + jest.spyOn(exportModule, 'processIndexes').mockResolvedValue(); + }); + + it('should use getDefaultLogPath for LOG_FILENAME with process.platform as default', function() { + LOG_FILENAME(); + + expect(getDefaultLogPath).toHaveBeenCalledWith('search-index', 'export', process.platform); + }); + + it('should export all indexes for the current hub if no ids specified', async (): Promise => { + const schemaIdsToExport: string[] | undefined = undefined; + const argv = { ...yargArgs, ...config, dir: 'my-dir', schemaId: schemaIdsToExport, logFile: new FileLog() }; + + const filteredIndexesToExport = [...indexesToExport]; + jest + .spyOn(exportModule, 'enrichIndex') + .mockImplementation((x, y, z) => Promise.resolve(z as EnrichedSearchIndex)); + jest.spyOn(exportModule, 'filterIndexesById').mockReturnValue(filteredIndexesToExport); + + await handler(argv); + + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalledTimes(1); + expect(mockList).toHaveBeenCalledWith(undefined, undefined, { size: 100 }); + expect(loadJsonFromDirectory).toHaveBeenCalledWith(argv.dir, EnrichedSearchIndex); + expect(validateNoDuplicateIndexNames).toHaveBeenCalled(); + expect(exportModule.filterIndexesById).toHaveBeenCalledWith(indexesToExport, []); + expect(exportModule.processIndexes).toHaveBeenCalledWith( + argv.dir, + [], + filteredIndexesToExport, + expect.any(Map), + expect.any(FileLog), + false + ); + }); + + it('should export only selected indexes if ids specified', async (): Promise => { + const idsToExport: string[] | undefined = ['index-id-2']; + const argv = { ...yargArgs, ...config, dir: 'my-dir', id: idsToExport, logFile: new FileLog() }; + + const filteredIndexesToExport = [indexesToExport[1]]; + jest + .spyOn(exportModule, 'enrichIndex') + .mockImplementation((x, y, z) => Promise.resolve(z as EnrichedSearchIndex)); + jest.spyOn(exportModule, 'filterIndexesById').mockReturnValue(filteredIndexesToExport); + + await handler(argv); + + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalled(); + expect(loadJsonFromDirectory).toHaveBeenCalledWith(argv.dir, EnrichedSearchIndex); + expect(validateNoDuplicateIndexNames).toHaveBeenCalled(); + expect(exportModule.filterIndexesById).toHaveBeenCalledWith(indexesToExport, idsToExport); + expect(exportModule.processIndexes).toHaveBeenCalledWith( + argv.dir, + [], + filteredIndexesToExport, + expect.any(Map), + expect.any(FileLog), + false + ); + }); + }); +}); diff --git a/src/commands/search-index/export.ts b/src/commands/search-index/export.ts new file mode 100644 index 00000000..7ad9fc2a --- /dev/null +++ b/src/commands/search-index/export.ts @@ -0,0 +1,458 @@ +import chalk from 'chalk'; +import { + HalResource, + Hub, + Page, + Pageable, + SearchIndex, + SearchIndexSettings, + Sortable, + Webhook +} from 'dc-management-sdk-js'; +import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; +import { SearchIndexKey } from 'dc-management-sdk-js/build/main/lib/model/SearchIndexKey'; +import { isEqual } from 'lodash'; +import { table } from 'table'; +import { Arguments, Argv } from 'yargs'; +import paginator from '../../common/dc-management-sdk-js/paginator'; +import { FileLog } from '../../common/file-log'; +import { ensureDirectoryExists } from '../../common/import/directory-utils'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { streamTableOptions } from '../../common/table/table.consts'; +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { + ExportResult, + nothingExportedExit, + promptToOverwriteExports, + uniqueFilenamePath, + writeJsonToFile +} from '../../services/export.service'; +import { loadJsonFromDirectory } from '../../services/import.service'; +import { ConfigurationParameters } from '../configure'; +import { validateNoDuplicateIndexNames } from './import'; +import { join } from 'path'; +import { ExportBuilderOptions } from '../../interfaces/export-builder-options.interface'; + +export const command = 'export '; + +export const desc = 'Export Search Indexes'; + +export const LOG_FILENAME = (platform: string = process.platform): string => + getDefaultLogPath('search-index', 'export', platform); + +export const builder = (yargs: Argv): void => { + yargs + .positional('dir', { + describe: 'Output directory for the exported Search Index definitions', + type: 'string' + }) + .option('id', { + type: 'string', + describe: + 'The ID of a Search Index to be exported.\nIf no --id option is given, all search indexes for the hub are exported.\nA single --id option may be given to export a single Search Index.\nMultiple --id options may be given to export multiple search indexes at the same time.', + requiresArg: true + }) + .alias('f', 'force') + .option('f', { + type: 'boolean', + boolean: true, + describe: 'Overwrite search indexes without asking.' + }) + .option('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); +}; + +const ensureJSON = (resource: HalResource): object => { + return resource.toJSON != null ? resource.toJSON() : resource; +}; + +export const webhookEquals = (a?: Webhook, b?: Webhook): boolean => { + if (a === undefined) { + return b === undefined; + } else if (b === undefined) { + return false; + } + + return ( + a.method === b.method && + a.secret === b.secret && + a.label === b.label && + a.active === b.active && + isEqual(a.customPayload, b.customPayload) && + isEqual(a.events, b.events) && + isEqual(a.filters, b.filters) && + isEqual(a.handlers, b.handlers) && + isEqual(a.headers, b.headers) + ); +}; + +export const replicaEquals = (a: EnrichedReplica, b: EnrichedReplica, keys: boolean): boolean => + a.label === b.label && + (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && + isEqual(ensureJSON(a.settings), ensureJSON(b.settings)); + +export const assignedContentTypeEquals = ( + a: EnrichedAssignedContentType, + b: EnrichedAssignedContentType, + aWebhooks?: Map, + bWebhooks?: Map +): boolean => + !(aWebhooks && bWebhooks) || + (webhookEquals(aWebhooks.get(a.webhook), bWebhooks.get(b.webhook)) && + webhookEquals(aWebhooks.get(a.activeContentWebhook), bWebhooks.get(b.activeContentWebhook)) && + webhookEquals(aWebhooks.get(a.archivedContentWebhook), bWebhooks.get(b.archivedContentWebhook))); + +export const ensureSettings = (settings: SearchIndexSettings): object => { + const result = ensureJSON(settings) as SearchIndexSettings; + result.replicas = result.replicas || []; + return result; +}; + +export const equals = ( + a: EnrichedSearchIndex, + b: EnrichedSearchIndex, + keys = true, + aWebhooks?: Map, + bWebhooks?: Map +): boolean => + a.label === b.label && + a.assignedContentTypes + .map((x, i) => assignedContentTypeEquals(x, b.assignedContentTypes[i], aWebhooks, bWebhooks)) + .reduce((a, b) => a && b, true) && + a.replicas.length == b.replicas.length && + a.replicas.map((x, i) => replicaEquals(x, b.replicas[i], keys)).reduce((a, b) => a && b, true) && + (!keys || isEqual(ensureJSON(a.keys), ensureJSON(b.keys))) && + isEqual(ensureSettings(a.settings), ensureSettings(b.settings)); + +const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { + return (options?: Pageable & Sortable): Promise> => + hub.related.searchIndexes.list(parentId, projection, options); +}; + +export class EnrichedAssignedContentType extends AssignedContentType { + webhook: string; + activeContentWebhook: string; + archivedContentWebhook: string; +} + +export class EnrichedReplica extends SearchIndex { + settings: SearchIndexSettings; + keys: SearchIndexKey; +} + +export class EnrichedSearchIndex extends SearchIndex { + settings: SearchIndexSettings; + keys: SearchIndexKey; + assignedContentTypes: EnrichedAssignedContentType[]; + replicas: EnrichedReplica[]; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public toJSON(): any { + const result = super.toJSON(); + + result.assignedContentTypes = result.assignedContentTypes.map((type: AssignedContentType) => type.toJSON()); + return result; + } +} + +export const filterIndexesById = (listToFilter: SearchIndex[], indexIdList: string[]): SearchIndex[] => { + if (indexIdList.length === 0) { + return listToFilter; + } + + const unmatchedIndexUriList: string[] = indexIdList.filter(id => !listToFilter.some(index => index.id === id)); + if (unmatchedIndexUriList.length > 0) { + throw new Error( + `The following ID(s) could not be found: [${unmatchedIndexUriList + .map(u => `'${u}'`) + .join(', ')}].\nNothing was exported, exiting.` + ); + } + + return listToFilter.filter(index => indexIdList.some(id => index.id === id)); +}; + +export const registerWebhook = (mapping: Map, webhook: Webhook): string => { + mapping.set(webhook.id as string, webhook); + + return webhook.id as string; +}; + +export const enrichReplica = async (replica: SearchIndex): Promise => { + const enrichedReplica = new EnrichedReplica(replica); + + enrichedReplica.settings = await replica.related.settings.get(); + enrichedReplica.keys = await replica.related.keys.get(); + + return enrichedReplica; +}; + +export const enrichIndex = async ( + webhooks: Map, + allReplicas: Map, + index: SearchIndex +): Promise => { + const enrichedIndex = new EnrichedSearchIndex(index); + + enrichedIndex.settings = await index.related.settings.get(); + const types = await paginator(index.related.assignedContentTypes.list); + enrichedIndex.keys = await index.related.keys.get(); + + const replicas = allReplicas.get(index.id as string); + if (replicas) { + enrichedIndex.replicas = await Promise.all(replicas.map(enrichReplica)); + } else { + enrichedIndex.replicas = []; + } + + const enrichedTypes: EnrichedAssignedContentType[] = []; + + for (const type of types) { + const enriched = new EnrichedAssignedContentType(type); + enriched.webhook = registerWebhook(webhooks, await type.related.webhook()); + enriched.activeContentWebhook = registerWebhook(webhooks, await type.related.activeContentWebhook()); + enriched.archivedContentWebhook = registerWebhook(webhooks, await type.related.archivedContentWebhook()); + + enrichedTypes.push(enriched); + } + + enrichedIndex.assignedContentTypes = enrichedTypes; + + return enrichedIndex; +}; + +interface ExportRecord { + readonly filename: string; + readonly status: ExportResult; + readonly index: EnrichedSearchIndex; +} + +export const getExportRecordForIndex = ( + index: EnrichedSearchIndex, + outputDir: string, + previouslyExportedIndexes: { [filename: string]: EnrichedSearchIndex }, + previouslyExportedWebhooks: Map, + webhooksBeingExported: Map +): ExportRecord => { + const indexOfExportedIndex = Object.values(previouslyExportedIndexes).findIndex(c => c.name === index.name); + + if (indexOfExportedIndex < 0) { + const filename = uniqueFilenamePath(outputDir, index.name, 'json', Object.keys(previouslyExportedIndexes)); + + // This filename is now used. + previouslyExportedIndexes[filename] = index; + + return { + filename: filename, + status: 'CREATED', + index + }; + } + const filename = Object.keys(previouslyExportedIndexes)[indexOfExportedIndex]; + const previouslyExportedIndex = Object.values(previouslyExportedIndexes)[indexOfExportedIndex]; + + if (equals(previouslyExportedIndex, index, true, previouslyExportedWebhooks, webhooksBeingExported)) { + return { filename, status: 'UP-TO-DATE', index }; + } + + return { + filename, + status: 'UPDATED', + index + }; +}; + +type ExportsMap = { + uri: string; + filename: string; +}; + +export const getExportedWebhooks = (outputDir: string): Map => { + const exportedWebhooks = loadJsonFromDirectory(join(outputDir, 'webhooks'), Webhook); + const webhookList = Object.values(exportedWebhooks); + + const webhooks = new Map(); + + for (const webhook of webhookList) { + if (webhook.id) { + webhooks.set(webhook.id, webhook); + } + } + + return webhooks; +}; + +export const getIndexExports = ( + outputDir: string, + previouslyExportedIndexes: { [filename: string]: EnrichedSearchIndex }, + indexesBeingExported: EnrichedSearchIndex[], + webhooksBeingExported: Map +): [ExportRecord[], ExportsMap[]] => { + const allExports: ExportRecord[] = []; + const updatedExportsMap: ExportsMap[] = []; // uri x filename + const previouslyExportedWebhooks = getExportedWebhooks(outputDir); + + for (const index of indexesBeingExported) { + if (!index.name) { + continue; + } + + const exportRecord = getExportRecordForIndex( + index, + outputDir, + previouslyExportedIndexes, + previouslyExportedWebhooks, + webhooksBeingExported + ); + + allExports.push(exportRecord); + + if (exportRecord.status === 'UPDATED') { + updatedExportsMap.push({ uri: index.name, filename: exportRecord.filename }); + } + } + return [allExports, updatedExportsMap]; +}; + +export const processIndexes = async ( + outputDir: string, + previouslyExportedIndexes: { [filename: string]: EnrichedSearchIndex }, + indexesBeingExported: EnrichedSearchIndex[], + webhooksBeingExported: Map, + log: FileLog, + force: boolean +): Promise => { + if (indexesBeingExported.length === 0) { + nothingExportedExit(log, 'No search indexes to export from this hub, exiting.'); + return; + } + + const [allExports, updatedExportsMap] = getIndexExports( + outputDir, + previouslyExportedIndexes, + indexesBeingExported, + webhooksBeingExported + ); + if ( + allExports.length === 0 || + (Object.keys(updatedExportsMap).length > 0 && !(force || (await promptToOverwriteExports(updatedExportsMap, log)))) + ) { + nothingExportedExit(log); + return; + } + + await ensureDirectoryExists(outputDir); + + const data: string[][] = []; + + data.push([chalk.bold('File'), chalk.bold('Name'), chalk.bold('Result')]); + for (const { filename, status, index } of allExports) { + if (status !== 'UP-TO-DATE') { + delete index.id; // do not export id + writeJsonToFile(filename, index); + } else { + indexesBeingExported.splice(indexesBeingExported.indexOf(index), 1); + } + data.push([filename, index.name as string, status]); + } + + log.appendLine(table(data, streamTableOptions)); +}; + +export const filterWebhooks = ( + webhooks: Map, + filteredIndexes: EnrichedSearchIndex[] +): Map => { + const filtered = new Map(); + + for (const index of filteredIndexes) { + for (const type of index.assignedContentTypes) { + filtered.set(type.webhook, webhooks.get(type.webhook) as Webhook); + filtered.set(type.activeContentWebhook, webhooks.get(type.activeContentWebhook) as Webhook); + filtered.set(type.archivedContentWebhook, webhooks.get(type.archivedContentWebhook) as Webhook); + } + } + + return filtered; +}; + +export const processWebhooks = async ( + outputDir: string, + webhooksBeingExported: Webhook[], + log: FileLog +): Promise => { + if (webhooksBeingExported.length === 0) { + return; + } + + log.appendLine('Exporting Webhooks...'); + + const previouslyExportedWebhooks: { [filename: string]: Webhook } = {}; + const base = join(outputDir, 'webhooks'); + await ensureDirectoryExists(base); + + const data: string[][] = []; + + data.push([chalk.bold('File'), chalk.bold('Label'), chalk.bold('Result')]); + for (const webhook of webhooksBeingExported) { + const filename = uniqueFilenamePath(base, webhook.label, 'json', Object.keys(previouslyExportedWebhooks)); + previouslyExportedWebhooks[filename] = webhook; + writeJsonToFile(filename, webhook); + data.push([filename, webhook.label as string, 'UPDATED']); + } + + log.appendLine(table(data, streamTableOptions)); +}; + +export const separateReplicas = ( + allIndexes: SearchIndex[] +): { storedIndexes: SearchIndex[]; allReplicas: Map } => { + const storedIndexes: SearchIndex[] = []; + const allReplicas = new Map(); + for (const index of allIndexes) { + if (index.parentId == null) { + storedIndexes.push(index); + } else { + let list = allReplicas.get(index.parentId); + + if (list == null) { + list = []; + allReplicas.set(index.parentId, list); + } + + list.push(index); + } + } + + return { storedIndexes, allReplicas }; +}; + +export const handler = async (argv: Arguments): Promise => { + const { dir, id, logFile, force } = argv; + const client = dynamicContentClientFactory(argv); + const hub = await client.hubs.get(argv.hubId); + const log = logFile.open(); + + const previouslyExportedIndexes = loadJsonFromDirectory(dir, EnrichedSearchIndex); + validateNoDuplicateIndexNames(previouslyExportedIndexes); + + const allStoredIndexes = await paginator(searchIndexList(hub)); + const { storedIndexes, allReplicas } = separateReplicas(allStoredIndexes); + + const idArray: string[] = id ? (Array.isArray(id) ? id : [id]) : []; + const filteredIndexes = filterIndexesById(storedIndexes, idArray); + + const webhooks = new Map(); + const enrichedIndexes = await Promise.all(filteredIndexes.map(index => enrichIndex(webhooks, allReplicas, index))); + + await processIndexes(dir, previouslyExportedIndexes, enrichedIndexes, webhooks, log, force || false); + + const filteredWebhooks = filterWebhooks(webhooks, enrichedIndexes); + await processWebhooks(dir, Array.from(filteredWebhooks.values()), log); + + await log.close(); +}; diff --git a/src/commands/search-index/import.spec.ts b/src/commands/search-index/import.spec.ts new file mode 100644 index 00000000..ca937d69 --- /dev/null +++ b/src/commands/search-index/import.spec.ts @@ -0,0 +1,1083 @@ +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { SearchIndex, Hub, Webhook, SearchIndexSettings } from 'dc-management-sdk-js'; +import * as exportModule from './export'; +import * as importModule from './import'; +import * as webhookRewriter from './webhook-rewriter'; +import { + builder, + command, + doCreate, + doUpdate, + enrichIndex, + handler, + loadAndRewriteWebhooks, + LOG_FILENAME, + processIndexes, + storedIndexMapper, + validateNoDuplicateIndexNames, + rewriteIndexNames +} from './import'; +import Yargs from 'yargs/yargs'; +import { table } from 'table'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { loadJsonFromDirectory, UpdateStatus } from '../../services/import.service'; +import chalk from 'chalk'; +import { FileLog } from '../../common/file-log'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { EnrichedReplica, EnrichedSearchIndex } from './export'; +import MockPage from '../../common/dc-management-sdk-js/mock-page'; +import { AssignedContentType } from 'dc-management-sdk-js/build/main/lib/model/AssignedContentType'; +import { join } from 'path'; + +jest.mock('../../services/dynamic-content-client-factory'); +jest.mock('../../view/data-presenter'); +jest.mock('../../services/import.service'); +jest.mock('fs'); +jest.mock('table'); +jest.mock('../../common/log-helpers'); + +describe('search-index import command', (): void => { + afterEach((): void => { + jest.restoreAllMocks(); + }); + + it('should implement an import command', () => { + expect(command).toEqual('import '); + }); + + describe('builder tests', () => { + it('should configure yargs', () => { + const argv = Yargs(process.argv.slice(2)); + const spyPositional = jest.spyOn(argv, 'positional').mockReturnThis(); + const spyOption = jest.spyOn(argv, 'option').mockReturnThis(); + + builder(argv); + + expect(spyPositional).toHaveBeenCalledWith('dir', { + describe: 'Directory containing Search Indexes', + type: 'string' + }); + + expect(spyOption).toHaveBeenCalledWith('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); + + expect(spyOption).toHaveBeenCalledWith('webhooks', { + type: 'boolean', + describe: + 'Import webhooks as well. The command will attempt to rewrite account names and staging environments in the webhook body to match the destination.', + boolean: true + }); + }); + }); + + describe('storedIndexMapper', () => { + it('it should map to a stored index', () => { + const importedIndex = new EnrichedSearchIndex({ + name: 'matched-name', + label: 'mutated-label' + }); + const storedIndex = [new SearchIndex({ id: 'stored-id', name: 'matched-name', label: 'label' })]; + const result = storedIndexMapper(importedIndex, storedIndex); + + expect(result).toEqual( + expect.objectContaining({ + id: 'stored-id', + name: 'matched-name', + label: 'mutated-label' + }) + ); + }); + + it('should not map to a stored index', () => { + const importedIndex = new EnrichedSearchIndex({ + name: 'not-matched-name', + label: 'mutated-label' + }); + const storedIndex = [new SearchIndex({ id: 'stored-id', name: 'matched-name', label: 'label' })]; + const result = storedIndexMapper(importedIndex, storedIndex); + + expect(result).toEqual(expect.objectContaining({ name: 'not-matched-name', label: 'mutated-label' })); + }); + }); + + describe('replicaList', () => { + it('should return a function that lists the replicas for the given index, with the given projection', async () => { + const replicas = new MockPage(SearchIndex, [new SearchIndex({ label: 'webhook1' })]); + const index = new SearchIndex({ label: 'webhook1' }); + index.related.replicas.list = jest.fn().mockResolvedValue(replicas); + + const resultFn = await importModule.replicaList(index, 'projection'); + + const noOptionsResult = await resultFn(); + expect(noOptionsResult).toEqual(replicas); + expect(index.related.replicas.list).toHaveBeenCalledWith('projection', undefined); + + const optionsResult = await resultFn({ sort: 'sort' }); + expect(optionsResult).toEqual(replicas); + expect(index.related.replicas.list).toHaveBeenCalledWith('projection', { sort: 'sort' }); + }); + + it('should return a function that lists the replicas for the given index, with no projection', async () => { + const replicas = new MockPage(SearchIndex, [new SearchIndex({ label: 'webhook1' })]); + const index = new SearchIndex({ label: 'webhook1' }); + index.related.replicas.list = jest.fn().mockResolvedValue(replicas); + + const resultFn = await importModule.replicaList(index); + + const result = await resultFn(); + expect(result).toEqual(replicas); + expect(index.related.replicas.list).toHaveBeenCalledWith(undefined, undefined); + }); + }); + + describe('updateWebhookIfDifferent', () => { + it('should update the webhook if newWebhook is defined', async () => { + const webhook = new Webhook({ label: 'webhook1' }); + webhook.related.update = jest.fn().mockResolvedValue(webhook); + const newWebhook = new Webhook({ label: 'webhook2' }); + + await importModule.updateWebhookIfDifferent(webhook, newWebhook); + + expect(webhook.related.update).toHaveBeenCalledWith(newWebhook); + }); + + it('should not update the webhook if newWebhook is undefined', async () => { + const webhook = new Webhook({ label: 'webhook1' }); + webhook.related.update = jest.fn().mockResolvedValue(webhook); + + await importModule.updateWebhookIfDifferent(webhook, undefined); + + expect(webhook.related.update).not.toHaveBeenCalled(); + }); + }); + + describe('doCreate', () => { + const assignedContentType = new AssignedContentType({ contentTypeUri: 'http://uri.com' }); + const assignedContentTypes = [{ contentTypeUri: 'http://uri.com' }]; + + it('should create an index and return report', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + const newIndex = new SearchIndex({ id: 'created-id' }); + const mockCreate = jest.fn().mockResolvedValue(newIndex); + mockHub.related.searchIndexes.create = mockCreate; + jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); + const indexBase = { name: 'index-name', label: 'test-label' }; + const index = { ...indexBase, assignedContentTypes: [assignedContentType] }; + const webhooks = new Map(); + const result = await doCreate(mockHub, index as EnrichedSearchIndex, webhooks, log); + + expect(log.getData('CREATE')).toMatchInlineSnapshot(` + Array [ + "created-id", + ] + `); + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ ...indexBase, assignedContentTypes })); + expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index, webhooks); + expect(result).toEqual(newIndex); + }); + + it('should throw an error when index create fails', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); + const mockCreate = jest.fn().mockImplementation(() => { + throw new Error('Error creating index'); + }); + mockHub.related.searchIndexes.create = mockCreate; + const index = { name: 'index-name', label: 'test-label', assignedContentTypes: [assignedContentType] }; + + await expect( + doCreate(mockHub, index as EnrichedSearchIndex, new Map(), log) + ).rejects.toThrowErrorMatchingSnapshot(); + expect(importModule.enrichIndex).not.toHaveBeenCalled(); + expect(log.getData('CREATE')).toEqual([]); + }); + + it('should throw an error when index create fails if a string error is returned by the sdk', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + jest.spyOn(importModule, 'enrichIndex').mockResolvedValue(); + const mockCreate = jest + .fn() + .mockRejectedValue( + 'The create-index action is not available, ensure you have permission to perform this action.' + ); + mockHub.related.searchIndexes.create = mockCreate; + const index = { name: 'index-name', label: 'test-label', assignedContentTypes: [assignedContentType] }; + + await expect( + doCreate(mockHub, index as EnrichedSearchIndex, new Map(), log) + ).rejects.toThrowErrorMatchingSnapshot(); + expect(importModule.enrichIndex).not.toHaveBeenCalled(); + expect(log.getData('CREATE')).toEqual([]); + }); + + it('should throw an error when enrichIndex fails', async () => { + const mockHub = new Hub(); + const log = new FileLog(); + const newIndex = new SearchIndex({ id: 'created-id' }); + const mockCreate = jest.fn().mockResolvedValue(newIndex); + mockHub.related.searchIndexes.create = mockCreate; + jest + .spyOn(importModule, 'enrichIndex') + .mockRejectedValue( + 'The update-index action is not available, ensure you have permission to perform this action.' + ); + const indexBase = { name: 'index-name', label: 'test-label' }; + const index = { ...indexBase, assignedContentTypes: [assignedContentType] }; + const webhooks = new Map(); + + await expect( + doCreate(mockHub, index as EnrichedSearchIndex, webhooks, log) + ).rejects.toThrowErrorMatchingSnapshot(); + + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ ...indexBase, assignedContentTypes })); + expect(importModule.enrichIndex).toHaveBeenCalledWith(newIndex, index, webhooks); + expect(log.getData('CREATE')).toEqual([]); + }); + }); + + describe('enrichIndex', () => { + function addWebhookFns(type: AssignedContentType, webhooks: Webhook[] = []): void { + type.related.webhook = jest.fn().mockResolvedValue(webhooks[0]); + type.related.activeContentWebhook = jest.fn().mockResolvedValue(webhooks[1]); + type.related.archivedContentWebhook = jest.fn().mockResolvedValue(webhooks[2]); + type.related.unassign = jest.fn().mockReturnValue(Promise.resolve()); + } + + it('should fetch settings, content types for comparison with the index to be enriched', async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: {}, + assignedContentTypes: [] + }); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.assignedContentTypes.list = jest.fn().mockResolvedValue(new MockPage(AssignedContentType, [])); + index.related.assignedContentTypes.create = jest.fn(); + + await enrichIndex(index, enrichedIndex, undefined); + + expect(enrichedIndex.settings.replicas).toEqual([]); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).not.toHaveBeenCalled(); + }); + + it("should update settings with a union of both source and destination replicas, then update each replica's settings", async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: { + replicas: ['replica-1', 'replica-2'] + }, + assignedContentTypes: [], + replicas: [ + new EnrichedReplica({ + name: 'replica-1', + settings: new SearchIndexSettings({ setting: '1' }) + }), + new EnrichedReplica({ + name: 'replica-2', + settings: new SearchIndexSettings({ setting: '2' }) + }) + ] + }); + + const indexNames = ['replica-1', 'replica-2', 'replica-3']; + const replicas = indexNames.map(name => { + const index = new SearchIndex({ name }); + index.related.update = jest.fn().mockResolvedValue(index); + index.related.settings.update = jest + .fn() + .mockImplementation(settings => Promise.resolve(new SearchIndexSettings(settings))); + return index; + }); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings({ replicas: ['replica-3'] })); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.replicas.list = jest.fn().mockResolvedValue(new MockPage(SearchIndex, replicas)); + index.related.assignedContentTypes.list = jest.fn().mockResolvedValue(new MockPage(AssignedContentType, [])); + index.related.assignedContentTypes.create = jest.fn(); + + await enrichIndex(index, enrichedIndex, undefined); + + expect(enrichedIndex.settings.replicas).toEqual(expect.arrayContaining(['replica-1', 'replica-2', 'replica-3'])); + expect(enrichedIndex.settings.replicas.length).toEqual(3); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.replicas.list).toHaveBeenCalled(); + + for (let i = 0; i < 2; i++) { + expect((replicas[i].related.update as jest.Mock).mock.calls[0][0].name).toEqual(replicas[i].name); + expect(replicas[i].related.settings.update).toHaveBeenCalledWith(enrichedIndex.replicas[i].settings, false); + } + + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).not.toHaveBeenCalled(); + }); + + it('should assign any content types that are not yet assigned on the destination index, removing any that are no longer present', async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: {}, + assignedContentTypes: [ + new AssignedContentType({ contentTypeUri: 'http://toCreate.com' }), + new AssignedContentType({ contentTypeUri: 'http://toUpdate.com' }) + ] + }); + + const existingURIs = ['http://toUpdate.com', 'http://toDelete.com']; + const existingTypes = existingURIs.map(uri => { + const type = new AssignedContentType({ contentTypeUri: uri }); + addWebhookFns(type); + return type; + }); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.assignedContentTypes.list = jest + .fn() + .mockResolvedValue(new MockPage(AssignedContentType, existingTypes)); + + const created: AssignedContentType[] = []; + index.related.assignedContentTypes.create = jest.fn().mockImplementation(type => { + created.push(type); + addWebhookFns(type); + return type; + }); + + await enrichIndex(index, enrichedIndex, undefined); + + expect(created).toEqual([enrichedIndex.assignedContentTypes[0]]); + + expect(existingTypes[0].related.unassign).not.toHaveBeenCalled(); + expect(existingTypes[1].related.unassign).toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).toHaveBeenCalledWith(enrichedIndex.assignedContentTypes[0]); + + expect(enrichedIndex.settings.replicas).toEqual([]); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + }); + + it('should update webhooks for the destination index when they are available', async () => { + const index = new SearchIndex({ + name: 'index-1', + label: 'index-1' + }); + + const enrichedIndex = new EnrichedSearchIndex({ + settings: {}, + assignedContentTypes: [ + new AssignedContentType({ + contentTypeUri: 'http://toUpdate.com', + webhook: 'id-1', + activeContentWebhook: 'id-2' + }) + ] + }); + + const type = new AssignedContentType({ contentTypeUri: 'http://toUpdate.com' }); + const webhooks = [new Webhook({ id: 'id-1' }), new Webhook({ id: 'id-2' }), new Webhook({ id: 'id-3' })]; + addWebhookFns(type, webhooks); + + const enrichedWebhooks = new Map(); + enrichedWebhooks.set('id-1', new Webhook({ id: 'id-1', label: 'updated-1' })); + enrichedWebhooks.set('id-2', new Webhook({ id: 'id-2', label: 'updated-2' })); + + jest.spyOn(importModule, 'updateWebhookIfDifferent').mockReturnValue(Promise.resolve()); + + index.related.settings.get = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.settings.update = jest.fn().mockResolvedValue(new SearchIndexSettings()); + index.related.assignedContentTypes.list = jest.fn().mockResolvedValue(new MockPage(AssignedContentType, [type])); + index.related.assignedContentTypes.create = jest.fn(); + + await enrichIndex(index, enrichedIndex, enrichedWebhooks); + + expect(type.related.unassign).not.toHaveBeenCalled(); + expect(index.related.assignedContentTypes.create).not.toHaveBeenCalled(); + + expect(type.related.webhook).toHaveBeenCalled(); + expect(type.related.activeContentWebhook).toHaveBeenCalled(); + expect(type.related.archivedContentWebhook).toHaveBeenCalled(); + + expect(importModule.updateWebhookIfDifferent).toHaveBeenNthCalledWith( + 1, + webhooks[0], + enrichedWebhooks.get('id-1') + ); + + expect(importModule.updateWebhookIfDifferent).toHaveBeenNthCalledWith( + 2, + webhooks[1], + enrichedWebhooks.get('id-2') + ); + + expect(importModule.updateWebhookIfDifferent).toHaveBeenNthCalledWith(3, webhooks[2], undefined); + + expect(enrichedIndex.settings.replicas).toEqual([]); + expect(index.related.settings.get).toHaveBeenCalled(); + expect(index.related.settings.update).toHaveBeenCalledWith(enrichedIndex.settings, false); + expect(index.related.assignedContentTypes.list).toHaveBeenCalled(); + }); + }); + + describe('doUpdate', () => { + beforeEach(() => { + /* ... */ + }); + + it('should update an index and return report', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + } as EnrichedSearchIndex; + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + const expectedIndex = new SearchIndex({ + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + }); + + const updatedIndex = new SearchIndex(mutatedIndex); + const mockUpdate = jest.fn().mockResolvedValue(updatedIndex); + storedIndex.related.update = mockUpdate; + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + const enrichedStoredIndex = new EnrichedSearchIndex(storedIndex); + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce(enrichedStoredIndex); + jest.spyOn(importModule, 'getIndexProperties').mockReturnValueOnce(mutatedIndex); + jest.spyOn(importModule, 'enrichIndex').mockResolvedValueOnce(); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + const result = await doUpdate(hub, replicas, mutatedIndex, webhooks, log); + + expect(log.getData('UPDATE')).toMatchInlineSnapshot(` + Array [ + "stored-id", + ] + `); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(importModule.enrichIndex).toHaveBeenCalledWith(updatedIndex, mutatedIndex, webhooks); + expect(result).toEqual({ index: updatedIndex, updateStatus: UpdateStatus.UPDATED }); + expect(mockUpdate.mock.calls[0][0].toJSON()).toEqual(expectedIndex.toJSON()); + }); + + it('should skip update when no change to index and return report', async () => { + const mutatedIndex = new EnrichedSearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label', + settings: {}, + assignedContentTypes: [], + replicas: [] + }); + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce( + new EnrichedSearchIndex({ + ...storedIndex, + settings: {}, + assignedContentTypes: [], + replicas: [] + }) + ); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + const result = await doUpdate(hub, replicas, mutatedIndex, webhooks, log); + + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(result).toEqual({ index: storedIndex, updateStatus: UpdateStatus.SKIPPED }); + expect(log.getData('UPDATE')).toEqual([]); + }); + + it('should throw an error when unable to get index during update', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'matched-name', + label: 'label' + } as EnrichedSearchIndex; + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockImplementation(() => { + throw new Error('Error retrieving index'); + }); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + + await expect(doUpdate(hub, replicas, mutatedIndex, webhooks, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(log.getData('UPDATE')).toEqual([]); + }); + + it('should throw an error when unable to update index during update if a string error is returned by sdk', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + } as EnrichedSearchIndex; + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + const expectedIndex = new SearchIndex({ + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + }); + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + const mockUpdate = jest + .fn() + .mockRejectedValue('The update action is not available, ensure you have permission to perform this action.'); + storedIndex.related.update = mockUpdate; + + const enrichedStoredIndex = new EnrichedSearchIndex(storedIndex); + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce(enrichedStoredIndex); + jest.spyOn(importModule, 'getIndexProperties').mockReturnValueOnce(mutatedIndex); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + await expect(doUpdate(hub, replicas, mutatedIndex, webhooks, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(log.getData('UPDATE')).toEqual([]); + expect(mockUpdate.mock.calls[0][0].toJSON()).toEqual(expectedIndex.toJSON()); + }); + + it('should throw an error when unable to update index during update', async () => { + const mutatedIndex = { + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + } as EnrichedSearchIndex; + const storedIndex = new SearchIndex({ + id: 'stored-id', + name: 'matched-name', + label: 'label' + }); + const expectedIndex = new SearchIndex({ + id: 'stored-id', + name: 'not-matched-name', + label: 'mutated-label' + }); + + const hub = new Hub(); + hub.related.searchIndexes.get = jest.fn().mockResolvedValue(storedIndex); + + const mockUpdate = jest.fn().mockRejectedValue(new Error('Error saving index')); + storedIndex.related.update = mockUpdate; + + const enrichedStoredIndex = new EnrichedSearchIndex(storedIndex); + jest.spyOn(exportModule, 'enrichIndex').mockResolvedValueOnce(enrichedStoredIndex); + jest.spyOn(importModule, 'getIndexProperties').mockReturnValueOnce(mutatedIndex); + + const log = new FileLog(); + const webhooks = new Map(); + const replicas = new Map(); + await expect(doUpdate(hub, replicas, mutatedIndex, webhooks, log)).rejects.toThrowErrorMatchingSnapshot(); + expect(hub.related.searchIndexes.get).toHaveBeenCalledWith('stored-id'); + expect(exportModule.enrichIndex).toHaveBeenCalledWith(expect.any(Map), replicas, storedIndex); + expect(log.getData('UPDATE')).toEqual([]); + expect(mockUpdate.mock.calls[0][0].toJSON()).toEqual(expectedIndex.toJSON()); + }); + }); + + describe('loadAndRewriteWebhooks', () => { + it('should create an id to webhook map from the webhooks loaded from the given directory', async () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1', + customPayload: { value: 'a' } + }), + 'directory/webhooks/webhook2.json': new Webhook({ + id: 'id2', + label: 'webhook2', + customPayload: { value: 'b' } + }) + }; + + const hub = new Hub({ + name: 'accountName', + settings: { + virtualStagingEnvironment: { + hostname: 'http://amplience.com' + } + } + }); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + jest.spyOn(webhookRewriter, 'rewriteDeliveryContentItem').mockImplementation(body => { + return body + '-rewrite'; + }); + + const result = await loadAndRewriteWebhooks(hub, 'directory/webhooks'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(2); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(result.get('id2')).toEqual(webhooks['directory/webhooks/webhook2.json']); + expect(webhooks['directory/webhooks/webhook1.json'].customPayload).toEqual({ value: 'a-rewrite' }); + expect(webhooks['directory/webhooks/webhook2.json'].customPayload).toEqual({ value: 'b-rewrite' }); + + expect(webhookRewriter.rewriteDeliveryContentItem).toHaveBeenNthCalledWith( + 1, + 'a', + 'accountName', + 'http://amplience.com' + ); + expect(webhookRewriter.rewriteDeliveryContentItem).toHaveBeenNthCalledWith( + 2, + 'b', + 'accountName', + 'http://amplience.com' + ); + }); + + it('should not rewrite webhook body if none is present', async () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1' + }), + 'directory/webhooks/webhook2.json': new Webhook({ + id: 'id2', + label: 'webhook2' + }) + }; + + const hub = new Hub({ + name: 'accountName', + settings: { + virtualStagingEnvironment: { + hostname: 'http://amplience.com' + } + } + }); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + jest.spyOn(webhookRewriter, 'rewriteDeliveryContentItem'); + + const result = await loadAndRewriteWebhooks(hub, 'directory/webhooks'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(2); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(result.get('id2')).toEqual(webhooks['directory/webhooks/webhook2.json']); + expect(webhooks['directory/webhooks/webhook1.json'].customPayload).toBeUndefined(); + expect(webhooks['directory/webhooks/webhook2.json'].customPayload).toBeUndefined(); + + expect(webhookRewriter.rewriteDeliveryContentItem).not.toHaveBeenCalled(); + }); + + it('should pass an undefined vse to the rewriter if the object is missing from settings', async () => { + const webhooks = { + 'directory/webhooks/webhook1.json': new Webhook({ + id: 'id1', + label: 'webhook1', + customPayload: { value: 'a' } + }) + }; + + const hub = new Hub({ + name: 'accountName', + settings: {} + }); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(webhooks); + jest.spyOn(webhookRewriter, 'rewriteDeliveryContentItem').mockImplementation(body => { + return body + '-rewrite'; + }); + + const result = await loadAndRewriteWebhooks(hub, 'directory/webhooks'); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('directory/webhooks', Webhook); + expect(result.size).toEqual(1); + expect(result.get('id1')).toEqual(webhooks['directory/webhooks/webhook1.json']); + expect(webhooks['directory/webhooks/webhook1.json'].customPayload).toEqual({ value: 'a-rewrite' }); + + expect(webhookRewriter.rewriteDeliveryContentItem).toHaveBeenCalledWith('a', 'accountName', undefined); + }); + }); + + describe('processIndexes', () => { + let mockTable: jest.Mock; + + beforeEach(() => { + mockTable = table as jest.Mock; + mockTable.mockImplementation(jest.requireActual('table').table); + }); + + it('should create and update an index', async () => { + const hub = new Hub(); + const indexesToProcess = [ + new EnrichedSearchIndex({ + name: 'index-name', + label: 'created', + assignedContentTypes: [] + }), + new EnrichedSearchIndex({ + id: 'updated-id', + name: 'index-name-2', + label: 'updated' + }), + new EnrichedSearchIndex({ + id: 'up-to-date-id', + name: 'index-name-3', + label: 'up-to date' + }) + ]; + + const createdIndex = new EnrichedSearchIndex({ + id: 'created-id', + ...indexesToProcess[0].toJSON() + }); + jest.spyOn(importModule, 'doCreate').mockResolvedValueOnce(createdIndex); + const doUpdateResult1 = { + index: indexesToProcess[1], + updateStatus: UpdateStatus.UPDATED + }; + jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult1); + const doUpdateResult2 = { + index: indexesToProcess[2], + updateStatus: UpdateStatus.SKIPPED + }; + jest.spyOn(importModule, 'doUpdate').mockResolvedValueOnce(doUpdateResult2); + const webhooks = new Map(); + const replicas = new Map(); + + await processIndexes(indexesToProcess, replicas, webhooks, hub, new FileLog()); + + expect(importModule.doCreate).toHaveBeenCalledWith(hub, indexesToProcess[0], webhooks, expect.any(FileLog)); + expect(importModule.doUpdate).toHaveBeenCalledWith( + hub, + replicas, + indexesToProcess[1], + webhooks, + expect.any(FileLog) + ); + + expect(mockTable).toHaveBeenCalledTimes(1); + expect(mockTable).toHaveBeenNthCalledWith( + 1, + [ + [chalk.bold('ID'), chalk.bold('Name'), chalk.bold('Result')], + [createdIndex.id, createdIndex.name, 'CREATED'], + [doUpdateResult1.index.id, doUpdateResult1.index.name, 'UPDATED'], + [doUpdateResult2.index.id, doUpdateResult2.index.name, 'UP-TO-DATE'] + ], + streamTableOptions + ); + }); + }); + + describe('validateNoDuplicateIndexNames', function() { + it('should not throw an error when there are no duplicates', () => { + const indexesToProcess = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + name: 'index-name-2' + }) + }; + + expect(() => validateNoDuplicateIndexNames(indexesToProcess)).not.toThrow(); + }); + + it('should throw and error when there are duplicate uris', () => { + const indexesToProcess = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + name: 'index-name-2' + }), + 'file-3': new EnrichedSearchIndex({ + name: 'index-name-2' + }), + 'file-4': new EnrichedSearchIndex({ + name: 'index-name-1' + }) + }; + + expect(() => validateNoDuplicateIndexNames(indexesToProcess)).toThrowErrorMatchingSnapshot(); + }); + }); + + describe('rewriteIndexNames', function() { + it("should rewrite index names to contain the given hub's name", () => { + const indexesToProcess = { + 'file-1': new EnrichedSearchIndex({ + name: 'oldHub.index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + name: 'index-name-2' + }) + }; + + const hub = new Hub({ name: 'newHub' }); + + expect(() => rewriteIndexNames(hub, indexesToProcess)).not.toThrow(); + + expect(indexesToProcess['file-1'].name).toEqual('newHub.index-name-1'); + expect(indexesToProcess['file-2'].name).toEqual('newHub.index-name-2'); + }); + }); + + describe('filterIndexesById', function() { + it('should delete indexes without a matching id', () => { + const indexesToProcess = { + 'file-1': new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2' + }) + }; + + const expectedResult = { 'file-2': indexesToProcess['file-2'] }; + + importModule.filterIndexesById(['index-id-2'], indexesToProcess); + + expect(indexesToProcess).toEqual(expectedResult); + }); + + it('should remove all indexes if no ids are given', () => { + const indexesToProcess = { + 'file-1': new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2' + }) + }; + + importModule.filterIndexesById([], indexesToProcess); + + expect(indexesToProcess).toEqual({}); + }); + }); + + describe('handler tests', () => { + const yargArgs = { + $0: 'test', + _: ['test'] + }; + const config = { + clientId: 'client-id', + clientSecret: 'client-id', + hubId: 'hub-id' + }; + + const mockGetHub = jest.fn(); + const mockList = jest.fn(); + + beforeEach(() => { + (dynamicContentClientFactory as jest.Mock).mockReturnValue({ + hubs: { + get: mockGetHub + } + }); + + mockGetHub.mockResolvedValue({ + id: 'hub-id', + related: { + searchIndexes: { + list: mockList + } + } + }); + + mockList.mockImplementation(() => { + return Promise.resolve( + new MockPage(SearchIndex, [ + new SearchIndex({ + id: 'id', + label: 'label', + name: 'stored-index' + }) + ]) + ); + }); + }); + + it('should use getDefaultLogPath for LOG_FILENAME with process.platform as default', function() { + LOG_FILENAME(); + + expect(getDefaultLogPath).toHaveBeenCalledWith('search-index', 'import', process.platform); + }); + + it('should create an index and update', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; + const fileNamesAndIndexesToImport = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'created' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'content-index-id', + name: 'index-name-2', + label: 'updated' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndexesToImport); + jest + .spyOn(importModule, 'storedIndexMapper') + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-1']) + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); + jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); + jest.spyOn(importModule, 'rewriteIndexNames'); + + await handler(argv); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(rewriteIndexNames).toHaveBeenCalledWith(expect.any(Object), fileNamesAndIndexesToImport); + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalled(); + expect(processIndexes).toHaveBeenCalledWith( + Object.values(fileNamesAndIndexesToImport), + expect.any(Map), + undefined, + expect.any(Object), + expect.any(FileLog) + ); + }); + + it('should call filterIndexesById when a list of ids is provided', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog() }; + const fileNamesAndIndexesToImport = { + 'file-1': new EnrichedSearchIndex({ + id: 'index-id-1', + name: 'index-name-1', + label: 'created' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'index-id-2', + name: 'index-name-2', + label: 'updated' + }) + }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue({ ...fileNamesAndIndexesToImport }); + jest.spyOn(importModule, 'storedIndexMapper').mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); + jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); + jest.spyOn(importModule, 'rewriteIndexNames'); + + await handler(argv, ['index-id-2']); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(rewriteIndexNames).toHaveBeenCalledWith(expect.any(Object), { + 'file-2': fileNamesAndIndexesToImport['file-2'] + }); + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(processIndexes).toHaveBeenCalledWith( + [fileNamesAndIndexesToImport['file-2']], + expect.any(Map), + undefined, + expect.any(Object), + expect.any(FileLog) + ); + }); + + it('should load webhooks when the webhooks argument is provided', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-dir', logFile: new FileLog(), webhooks: true }; + const fileNamesAndIndexesToImport = { + 'file-1': new EnrichedSearchIndex({ + name: 'index-name-1', + label: 'created' + }), + 'file-2': new EnrichedSearchIndex({ + id: 'content-index-id', + name: 'index-name-2', + label: 'updated' + }) + }; + + const webhookMap = new Map(); + + (loadJsonFromDirectory as jest.Mock).mockReturnValue(fileNamesAndIndexesToImport); + jest + .spyOn(importModule, 'storedIndexMapper') + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-1']) + .mockReturnValueOnce(fileNamesAndIndexesToImport['file-2']); + jest.spyOn(importModule, 'processIndexes').mockResolvedValueOnce(); + jest.spyOn(importModule, 'loadAndRewriteWebhooks').mockResolvedValueOnce(webhookMap); + jest.spyOn(importModule, 'rewriteIndexNames'); + + await handler(argv); + + expect(loadJsonFromDirectory).toHaveBeenCalledWith('my-dir', EnrichedSearchIndex); + expect(rewriteIndexNames).toHaveBeenCalledWith(expect.any(Object), fileNamesAndIndexesToImport); + expect(loadAndRewriteWebhooks).toHaveBeenCalledWith(expect.any(Object), join('my-dir', 'webhooks')); + expect(mockGetHub).toHaveBeenCalledWith('hub-id'); + expect(mockList).toHaveBeenCalled(); + expect(processIndexes).toHaveBeenCalledWith( + Object.values(fileNamesAndIndexesToImport), + expect.any(Map), + webhookMap, + expect.any(Object), + expect.any(FileLog) + ); + }); + + it('should throw an error when no content found in import directory', async (): Promise => { + const argv = { ...yargArgs, ...config, dir: 'my-empty-dir', logFile: new FileLog() }; + + (loadJsonFromDirectory as jest.Mock).mockReturnValue([]); + + await expect(handler(argv)).rejects.toThrowErrorMatchingSnapshot(); + }); + }); +}); diff --git a/src/commands/search-index/import.ts b/src/commands/search-index/import.ts new file mode 100644 index 00000000..12dbca03 --- /dev/null +++ b/src/commands/search-index/import.ts @@ -0,0 +1,348 @@ +import chalk from 'chalk'; +import { Hub, Page, Pageable, SearchIndex, Settings, Sortable, Webhook } from 'dc-management-sdk-js'; +import { join } from 'path'; +import { table } from 'table'; +import { Arguments, Argv } from 'yargs'; +import paginator from '../../common/dc-management-sdk-js/paginator'; +import { FileLog } from '../../common/file-log'; +import { createLog, getDefaultLogPath } from '../../common/log-helpers'; +import { streamTableOptions } from '../../common/table/table.consts'; +import { ImportIndexBuilderOptions } from '../../interfaces/import-index-builder-options.interface'; +import dynamicContentClientFactory from '../../services/dynamic-content-client-factory'; +import { ImportResult, loadJsonFromDirectory, UpdateStatus } from '../../services/import.service'; +import { ConfigurationParameters } from '../configure'; +import { EnrichedSearchIndex, equals, enrichIndex as enrichServerIndex, separateReplicas } from './export'; +import { rewriteDeliveryContentItem } from './webhook-rewriter'; + +export const command = 'import '; + +export const desc = 'Import Search Index'; + +export const LOG_FILENAME = (platform: string = process.platform): string => + getDefaultLogPath('search-index', 'import', platform); + +export const builder = (yargs: Argv): void => { + yargs.positional('dir', { + describe: 'Directory containing Search Indexes', + type: 'string' + }); + + yargs.option('logFile', { + type: 'string', + default: LOG_FILENAME, + describe: 'Path to a log file to write to.', + coerce: createLog + }); + + yargs.option('webhooks', { + type: 'boolean', + describe: + 'Import webhooks as well. The command will attempt to rewrite account names and staging environments in the webhook body to match the destination.', + boolean: true + }); +}; + +const searchIndexList = (hub: Hub, parentId?: string, projection?: string) => { + return (options?: Pageable & Sortable): Promise> => + hub.related.searchIndexes.list(parentId, projection, options); +}; + +export const replicaList = (index: SearchIndex, projection?: string) => { + return (options?: Pageable & Sortable): Promise> => + index.related.replicas.list(projection, options); +}; + +type IndexName = string; +type IndexFile = string; + +export const validateNoDuplicateIndexNames = (importedIndexes: { + [filename: string]: EnrichedSearchIndex; +}): void | never => { + const nameToFilenameMap = new Map(); // map: name x filenames + for (const [filename, index] of Object.entries(importedIndexes)) { + if (index.name) { + const otherFilenames: string[] = nameToFilenameMap.get(index.name) || []; + if (filename) { + nameToFilenameMap.set(index.name, [...otherFilenames, filename]); + } + } + } + const uniqueDuplicateNames: [string, IndexFile[]][] = []; + nameToFilenameMap.forEach((filenames, name) => { + if (filenames.length > 1) { + uniqueDuplicateNames.push([name, filenames]); + } + }); + + if (uniqueDuplicateNames.length > 0) { + throw new Error( + `Indexes must have unique name values. Duplicate values found:-\n${uniqueDuplicateNames + .map(([name, filenames]) => ` name: '${name}' in files: [${filenames.map(f => `'${f}'`).join(', ')}]`) + .join('\n')}` + ); + } +}; + +export const rewriteIndexNames = ( + hub: Hub, + importedIndexes: { + [filename: string]: EnrichedSearchIndex; + } +): void | never => { + for (const index of Object.values(importedIndexes)) { + const name = index.name as string; + const firstDot = name.indexOf('.'); + + if (firstDot == -1) { + index.name = `${hub.name}.${name}`; + } else { + index.name = `${hub.name}${name.substring(firstDot)}`; + } + } +}; + +export const filterIndexesById = ( + idFilter: string[], + importedIndexes: { + [filename: string]: SearchIndex; + } +): void | never => { + for (const [filename, index] of Object.entries(importedIndexes)) { + if (idFilter.indexOf(index.id as string) === -1) { + delete importedIndexes[filename]; + } + } +}; + +export const storedIndexMapper = (index: EnrichedSearchIndex, storedIndexes: SearchIndex[]): EnrichedSearchIndex => { + const found = storedIndexes.find(stored => stored.name === index.name); + const mutatedIndex = found ? { ...index, id: found.id } : index; + + return new EnrichedSearchIndex(mutatedIndex); +}; + +export const getIndexProperties = (index: SearchIndex): object => { + return { + label: index.label, + name: index.name, + suffix: index.suffix, + type: index.type + }; +}; + +export const updateWebhookIfDifferent = async (webhook: Webhook, newWebhook: Webhook | undefined): Promise => { + if (newWebhook === undefined) { + return; + } + + await webhook.related.update(newWebhook); +}; + +export const enrichIndex = async ( + index: SearchIndex, + enrichedIndex: EnrichedSearchIndex, + webhooks: Map | undefined +): Promise => { + // Union the replicas on the server and the replicas being imported. + // This avoids replicas being detached from their parents, and thus becoming unusable. + const settings = await index.related.settings.get(); + const replicas = new Set(settings.replicas || []); + if (enrichedIndex.settings.replicas) { + enrichedIndex.settings.replicas.forEach((replica: string) => { + replicas.add(replica); + }); + } + enrichedIndex.settings.replicas = Array.from(replicas); + + // Update the search index settings. + await index.related.settings.update(enrichedIndex.settings, false); + + if (replicas.size) { + // Replica settings must also be updated. The replicas may have changed, so fetch them again. + + const replicas = await paginator(replicaList(index)); + + for (const importReplica of enrichedIndex.replicas) { + let replica = replicas.find(replica => replica.name === importReplica.name); + + if (replica) { + replica = await replica.related.update(new SearchIndex(getIndexProperties(importReplica))); + + replica.related.settings.update(importReplica.settings, false); + } + } + } + + const types = await paginator(index.related.assignedContentTypes.list); + + // Assign any content types that are not assigned. + + const unassigned = new Set(types); + + for (const assignment of enrichedIndex.assignedContentTypes) { + let existing = types.find(type => type.contentTypeUri === assignment.contentTypeUri); + + if (!existing) { + // Need to create a new assignment + existing = await index.related.assignedContentTypes.create(assignment); + } + + unassigned.delete(existing); + + // Update any webhooks if they differ from the ones being imported, if the flag is provided. + // Does the webhook being referenced in the saved index exist in the import? + + if (webhooks) { + await updateWebhookIfDifferent(await existing.related.webhook(), webhooks.get(assignment.webhook)); + await updateWebhookIfDifferent( + await existing.related.activeContentWebhook(), + webhooks.get(assignment.activeContentWebhook) + ); + await updateWebhookIfDifferent( + await existing.related.archivedContentWebhook(), + webhooks.get(assignment.archivedContentWebhook) + ); + } + } + + // Finally, remove any content type assignments that are not present in the imported index. + for (const toRemove of unassigned) { + await toRemove.related.unassign(index.id as string); + } +}; + +export const doCreate = async ( + hub: Hub, + index: EnrichedSearchIndex, + webhooks: Map | undefined, + log: FileLog +): Promise => { + try { + const assignedContentTypes = index.assignedContentTypes.map(type => ({ contentTypeUri: type.contentTypeUri })); + + const toCreate = new SearchIndex({ ...getIndexProperties(index), assignedContentTypes }); + + const createdIndex = await hub.related.searchIndexes.create(toCreate); + + await enrichIndex(createdIndex, index, webhooks); + + log.addAction('CREATE', `${createdIndex.id}`); + + return createdIndex; + } catch (err) { + throw new Error(`Error creating index ${index.name}:\n\n${err}`); + } +}; + +export const doUpdate = async ( + hub: Hub, + allReplicas: Map, + index: EnrichedSearchIndex, + webhooks: Map | undefined, + log: FileLog +): Promise<{ index: SearchIndex; updateStatus: UpdateStatus }> => { + try { + const retrievedIndex: SearchIndex = await hub.related.searchIndexes.get(index.id as string); + + const dstWebhooks = new Map(); + + const enrichedWebhook = await enrichServerIndex(dstWebhooks, allReplicas, retrievedIndex); + + if (equals(enrichedWebhook, index, false, dstWebhooks, webhooks)) { + return { index: retrievedIndex, updateStatus: UpdateStatus.SKIPPED }; + } + + Object.assign(retrievedIndex, getIndexProperties(index)); + + const updatedIndex = await retrievedIndex.related.update(retrievedIndex); + + await enrichIndex(updatedIndex, index, webhooks); + + log.addAction('UPDATE', `${retrievedIndex.id}`); + + return { index: updatedIndex, updateStatus: UpdateStatus.UPDATED }; + } catch (err) { + throw new Error(`Error updating index ${index.name}: ${err.message}`); + } +}; + +export const loadAndRewriteWebhooks = async (hub: Hub, dir: string): Promise> => { + const webhookList = loadJsonFromDirectory(dir, Webhook); + const webhooks = new Map(); + + for (const webhook of Object.values(webhookList)) { + webhooks.set(webhook.id as string, webhook); + } + + // Rewrite webhooks. Load VSE and account name from settings. + const account = hub.name as string; + const settings = hub.settings as Settings; + const vseObj = settings.virtualStagingEnvironment; + const vse = vseObj ? vseObj.hostname : undefined; + + webhooks.forEach(webhook => { + if (webhook.customPayload) { + webhook.customPayload.value = rewriteDeliveryContentItem(webhook.customPayload.value, account, vse); + } + }); + + return webhooks; +}; + +export const processIndexes = async ( + indexesToProcess: EnrichedSearchIndex[], + allReplicas: Map, + webhooks: Map | undefined, + hub: Hub, + log: FileLog +): Promise => { + const data: string[][] = []; + + data.push([chalk.bold('ID'), chalk.bold('Name'), chalk.bold('Result')]); + for (const entry of indexesToProcess) { + let status: ImportResult; + let index: SearchIndex; + if (entry.id) { + const result = await doUpdate(hub, allReplicas, entry, webhooks, log); + index = result.index; + status = result.updateStatus === UpdateStatus.SKIPPED ? 'UP-TO-DATE' : 'UPDATED'; + } else { + index = await doCreate(hub, entry, webhooks, log); + status = 'CREATED'; + } + data.push([index.id as string, index.name as string, status]); + } + + log.appendLine(table(data, streamTableOptions)); +}; + +export const handler = async ( + argv: Arguments, + idFilter?: string[] +): Promise => { + const { dir, logFile } = argv; + const client = dynamicContentClientFactory(argv); + const hub = await client.hubs.get(argv.hubId); + const log = logFile.open(); + const indexes = loadJsonFromDirectory(dir, EnrichedSearchIndex); + if (Object.keys(indexes).length === 0) { + throw new Error(`No indexes found in ${dir}`); + } + + validateNoDuplicateIndexNames(indexes); + rewriteIndexNames(hub, indexes); + + if (idFilter) { + filterIndexesById(idFilter, indexes); + } + + const allStoredIndexes = await paginator(searchIndexList(hub)); + const { storedIndexes, allReplicas } = separateReplicas(allStoredIndexes); + + const indexesToProcess = Object.values(indexes).map(index => storedIndexMapper(index, storedIndexes)); + const webhooks = argv.webhooks ? await loadAndRewriteWebhooks(hub, join(dir, 'webhooks')) : undefined; + + await processIndexes(indexesToProcess, allReplicas, webhooks, hub, log); + + await log.close(); +}; diff --git a/src/commands/search-index/webhook-rewriter.spec.ts b/src/commands/search-index/webhook-rewriter.spec.ts new file mode 100644 index 00000000..57b60b9a --- /dev/null +++ b/src/commands/search-index/webhook-rewriter.spec.ts @@ -0,0 +1,81 @@ +import { rewriteDeliveryContentItem } from './webhook-rewriter'; + +describe('webhook-rewriter tests', function() { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should not rewrite an empty string', async () => { + expect(rewriteDeliveryContentItem('', 'account', 'staging')).toEqual(''); + }); + + it('should not rewrite a simple string', async () => { + expect(rewriteDeliveryContentItem('basic', 'account', 'staging')).toEqual('basic'); + }); + + it('should not rewrite any unrelated tag', async () => { + expect(rewriteDeliveryContentItem('{{#unrelated}}{{/unrelated}}', 'account', 'staging')).toEqual( + '{{#unrelated}}{{/unrelated}}' + ); + }); + + it('should rewrite a simple webhook', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplaced" stagingEnvironment="old"}} + "content" : "{{~#each (first (pluck contentBlocks "content") 4)~}}{{~#each this.values~}}{{#if (eq locale "en-GB")}}{{{truncate value 1000}}}{{/if}}{{~/each~}}{{#unless @last}};{{/unless~}}{{~/each~}}" +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', 'staging')).toMatchSnapshot(); + }); + + it('should rewrite multiple tags within the same webhook', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplaced" stagingEnvironment="old"}} + {{example1}} +{{/withDeliveryContentItem}} + "key": "value", +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplaced2" stagingEnvironment="old2"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', 'staging')).toMatchSnapshot(); + }); + + it('should rewrite only one argument if the other is not present', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="tobereplacednostaging"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', "doesn't appear")).toMatchSnapshot(); + }); + + it('should not rewrite a tag with no matching arguments', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'account', "doesn't appear")).toEqual(example); + }); + + it('should escape quotes and backslashes', async () => { + const example = `{ +{{#withDeliveryContentItem contentItemId=payload.id account="1" stagingEnvironment="2"}} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, '"quotedString\\with\\backslash"', 'back\\slash')).toMatchSnapshot(); + }); + it('should still replace values with unusual whitespace', async () => { + const example = `{ +{{#withDeliveryContentItem + contentItemId=payload.id + account="1" + stagingEnvironment="another"\t }} + {{example1}} +{{/withDeliveryContentItem}} +}`; + expect(rewriteDeliveryContentItem(example, 'replaced', 'whitespace')).toMatchSnapshot(); + }); +}); diff --git a/src/commands/search-index/webhook-rewriter.ts b/src/commands/search-index/webhook-rewriter.ts new file mode 100644 index 00000000..2cc5871a --- /dev/null +++ b/src/commands/search-index/webhook-rewriter.ts @@ -0,0 +1,69 @@ +function matchAll(regex: RegExp, string: string): RegExpExecArray[] { + const result: RegExpExecArray[] = []; + let match: RegExpExecArray | null; + + while ((match = regex.exec(string)) != null) { + result.push(match); + } + + return result; +} + +export function rewriteDeliveryContentItem( + webhookBody: string, + account: string, + stagingEnvironment: string | undefined +): string { + // Current limitations - cannot handle key/value pairs where the value contains space, or a }} within quotes. + // These should not affect the two variables being replaced anyways. + // First, locate the withDeliveryContentItem opening tags. + + const tagRegexG = /\{\{\#withDeliveryContentItem(?\s+)(?.+\=.+(\s+.+\=.+)*\s*)\}\}/g; + const keyValueRegex = /(?.+?)\=(?.+?)\s+/g; + + const tagMatches = matchAll(tagRegexG, webhookBody); + + let replaceOffset = 0; + + for (const tag of tagMatches) { + const tGroups = tag.groups as { [key: string]: string }; + + const matchIndex = tag.index; + const body = tGroups.body + ' '; + const bodyIndex = matchIndex + '{{#withDeliveryContentItem'.length + tGroups.whitespace.length; + + const keyValueMatches = matchAll(keyValueRegex, body); + + for (const pair of keyValueMatches) { + const pGroups = pair.groups as { [key: string]: string }; + + const key = pGroups.key; + const keyIndex = bodyIndex + pair.index + replaceOffset; + + const value = pGroups.value; + const valueIndex = keyIndex + key.length + 1; + + let replaceValue: string | null = null; + + switch (key) { + case 'account': + replaceValue = account; + break; + case 'stagingEnvironment': + if (stagingEnvironment !== undefined) { + replaceValue = stagingEnvironment; + } + break; + } + + if (replaceValue != null) { + replaceValue = `"${replaceValue.replace(/\\/g, '\\\\').replace(/"/g, '"')}"`; + webhookBody = webhookBody.substr(0, valueIndex) + replaceValue + webhookBody.substr(valueIndex + value.length); + + replaceOffset += replaceValue.length - value.length; + } + } + } + + return webhookBody; +} diff --git a/src/interfaces/import-index-builder-options.interface.ts b/src/interfaces/import-index-builder-options.interface.ts new file mode 100644 index 00000000..a1ead6a7 --- /dev/null +++ b/src/interfaces/import-index-builder-options.interface.ts @@ -0,0 +1,5 @@ +import { ImportBuilderOptions } from './import-builder-options.interface'; + +export interface ImportIndexBuilderOptions extends ImportBuilderOptions { + webhooks?: boolean; +}