diff --git a/.talismanrc b/.talismanrc index 7efa0e61f..777a7a83c 100644 --- a/.talismanrc +++ b/.talismanrc @@ -39,4 +39,6 @@ fileignoreconfig: checksum: a64a4d396eddd936a63b799eff58c5c6660b5dcaa3a310fd8b09a027932f1789 - filename: packages/contentstack-migration/README.md checksum: e96006c1a948f766c88ae972b29582fa58eaf8184606bf011eebddc5a06cd7b6 +- filename: packages/contentstack-import/test/unit/utils/asset-helper.test.ts + checksum: da1d476d0a7aaaaee8a355571d85facc5f4de081ede40c13fd263df3bfb5c521 version: "" diff --git a/packages/contentstack-import/src/utils/asset-helper.ts b/packages/contentstack-import/src/utils/asset-helper.ts index e1110656e..7c19cb291 100644 --- a/packages/contentstack-import/src/utils/asset-helper.ts +++ b/packages/contentstack-import/src/utils/asset-helper.ts @@ -1,7 +1,7 @@ import Bluebird from 'bluebird'; import * as url from 'url'; import * as path from 'path'; -import { ContentstackClient, managementSDKClient, validateRegex, log } from '@contentstack/cli-utilities'; +import { ContentstackClient, managementSDKClient, log } from '@contentstack/cli-utilities'; import { ImportConfig } from '../types'; const debug = require('debug')('util:requests'); let _ = require('lodash'); @@ -10,7 +10,8 @@ let helper = require('./file-helper'); const MAX_RETRY_LIMIT = 5; -const escapeRegExp = (str: string) => str.replace(/[*+?^${}()|[\]\\]/g, '\\$&'); +// escapes a value the way it appears inside a JSON string, without the surrounding quotes +const jsonEscape = (str: string) => JSON.stringify(str).slice(1, -1); function validate(req: any) { if (typeof req !== 'object') { @@ -293,9 +294,17 @@ export const lookupAssets = function ( assetUids.forEach(function (assetUid: any) { let uid = mappedAssetUids[assetUid]; if (typeof uid !== 'undefined') { - const escapedAssetUid = assetUid.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - entry = entry.split(escapedAssetUid).join(uid); - matchedUids.push(assetUid); + // split() matches literally, so the UID must not be regex-escaped. It does need JSON + // escaping though, since the search runs on the serialized entry (a UID containing a + // backslash or a quote appears escaped there). + const updatedEntry = entry.split(jsonEscape(assetUid)).join(jsonEscape(uid)); + if (updatedEntry !== entry) { + entry = updatedEntry; + matchedUids.push(assetUid); + } else { + log.debug(`Asset UID ${assetUid} had a mapping but no occurrence in entry ${data.entry?.uid}`); + unmatchedUids.push(assetUid); + } } else { unmatchedUids.push(assetUid); } diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index d01468637..e9ab0e6f5 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -603,13 +603,9 @@ export const restoreJsonRteEntryRefs = ( if (sourceStackEntry[element.uid].indexOf(uid) !== -1) return uid; }); if (element.multiple && Array.isArray(entry[element.uid])) { - for (let i = 0; i < matches.length; i++) { - entry[element.uid] = entry[element.uid].map((el: string) => updateUids(el, matches[i], uidMapper)); - } + entry[element.uid] = entry[element.uid].map((el: string) => updateUids(el, matches, uidMapper)); } else { - for (let i = 0; i < matches.length; i++) { - entry[element.uid] = updateUids(entry[element.uid], matches[i], uidMapper); - } + entry[element.uid] = updateUids(entry[element.uid], matches, uidMapper); } } break; @@ -619,10 +615,12 @@ export const restoreJsonRteEntryRefs = ( return entry; }; -function updateUids(str: string, match: string, uidMapper: Record) { - const sanitizedMatch = escapeRegExp(match); - const replacement = uidMapper[match] ?? sanitizedMatch; - return str.split(sanitizedMatch).join(replacement); +function updateUids(str: string, matches: string[], uidMapper: Record) { + if (!matches.length) return str; + // longest-first so a UID that's a prefix of another (entry.1 vs entry.10) never wins the match; + // single regex pass so a replacement value can never itself get re-scanned/re-replaced + const pattern = [...matches].sort((a, b) => b.length - a.length).map(escapeRegExp).join('|'); + return str.replace(new RegExp(pattern, 'g'), (match) => uidMapper[match] ?? match); } function setDirtyTrue(jsonRteChild: any) { diff --git a/packages/contentstack-import/test/unit/utils/asset-helper.test.ts b/packages/contentstack-import/test/unit/utils/asset-helper.test.ts index 065bceeea..b7ca38235 100644 --- a/packages/contentstack-import/test/unit/utils/asset-helper.test.ts +++ b/packages/contentstack-import/test/unit/utils/asset-helper.test.ts @@ -426,6 +426,117 @@ describe('Asset Helper', () => { expect(result).to.exist; }); + it('should remap JSON RTE asset UIDs containing regex special characters', () => { + const data = { + entry: { + uid: 'entry1', + json_rte: { + children: [ + { + type: 'reference', + attrs: { + type: 'asset', + 'asset-uid': 'safe_6.0_big_picture_sorrento_web', + 'asset-link': '/assets/safe_6.0_big_picture_sorrento_web' + }, + children: [] as any + } + ] + } + }, + content_type: { + uid: 'ct1', + schema: [ + { + uid: 'json_rte', + data_type: 'json', + field_metadata: { rich_text_type: true } + } + ] + } + }; + const mappedAssetUids = { 'safe_6.0_big_picture_sorrento_web': 'bltNewAssetUid' }; + + const result = lookupAssets(data, mappedAssetUids, {}, '/test/mapper', []); + + expect(JSON.stringify(result)).to.not.include('safe_6.0_big_picture_sorrento_web'); + expect(JSON.stringify(result)).to.include('bltNewAssetUid'); + }); + + it('should remap asset UIDs for every regex special character', () => { + const specialCharUids = [ + 'asset.1', + 'asset*1', + 'asset+1', + 'asset?1', + 'asset^1', + 'asset$1', + 'asset{1}', + 'asset(1)', + 'asset|1', + 'asset[1]', + 'asset\\1' + ]; + + specialCharUids.forEach((assetUid, index) => { + const mappedUid = `bltMapped${index}`; + const data = { + entry: { + uid: 'entry1', + json_rte: { + children: [ + { + type: 'reference', + attrs: { type: 'asset', 'asset-uid': assetUid }, + children: [] as any + } + ] + } + }, + content_type: { + uid: 'ct1', + schema: [ + { uid: 'json_rte', data_type: 'json', field_metadata: { rich_text_type: true } } + ] + } + }; + + const result = lookupAssets(data, { [assetUid]: mappedUid }, {}, '/test/mapper', []); + + expect(result.json_rte.children[0].attrs['asset-uid'], `failed for UID: ${assetUid}`).to.equal(mappedUid); + }); + }); + + it('should record a special character asset UID as matched', () => { + const writeFileStub = fileHelper.writeFile as unknown as sinon.SinonStub; + const data = { + entry: { + uid: 'entry1', + json_rte: { + children: [ + { + type: 'reference', + attrs: { type: 'asset', 'asset-uid': 'asset.with.dots' }, + children: [] as any + } + ] + } + }, + content_type: { + uid: 'ct1', + schema: [ + { uid: 'json_rte', data_type: 'json', field_metadata: { rich_text_type: true } } + ] + } + }; + + lookupAssets(data, { 'asset.with.dots': 'bltNewAssetUid' }, {}, '/test/mapper', []); + + const writtenPaths = writeFileStub.getCalls().map((call: any) => call.args[0]); + expect(writtenPaths.some((p: string) => p.includes('matched-asset-uids.json'))).to.be.true; + expect(writtenPaths.some((p: string) => p.includes('unmatched-asset-uids.json'))).to.be.false; + }); + it('should handle JSON custom fields with extensions', () => { const data = { entry: { diff --git a/packages/contentstack-import/test/unit/utils/entries-helper.test.ts b/packages/contentstack-import/test/unit/utils/entries-helper.test.ts index 0d7705092..a6b9e6e88 100644 --- a/packages/contentstack-import/test/unit/utils/entries-helper.test.ts +++ b/packages/contentstack-import/test/unit/utils/entries-helper.test.ts @@ -384,6 +384,110 @@ describe('Entries Helper', () => { expect(restoreJsonRteEntryRefs).to.be.a('function'); }); + it('should remap entry UIDs containing regex special characters in HTML RTE', () => { + const sourceUid = 'source.entry.1'; + const ctSchema = [ + { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true } } + ]; + const sourceStackEntry = { + html_rte_field: `

ref

` + }; + // non-empty: restoreJsonRteEntryRefs skips falsy fields before restoring from source + const entry = { html_rte_field: '

' }; + + const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, { + uidMapper: { [sourceUid]: 'bltTargetEntry1' }, + mappedAssetUids: {}, + mappedAssetUrls: {} + }); + + expect(result.html_rte_field).to.not.include(sourceUid); + expect(result.html_rte_field).to.include('bltTargetEntry1'); + }); + + it('should leave an unmapped special character entry UID untouched in HTML RTE', () => { + const sourceUid = 'source.entry.1'; + const ctSchema = [ + { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true } } + ]; + const sourceStackEntry = { + html_rte_field: `` + }; + // non-empty: restoreJsonRteEntryRefs skips falsy fields before restoring from source + const entry = { html_rte_field: '

' }; + + const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, { + uidMapper: { 'other.entry': 'bltOther' }, + mappedAssetUids: {}, + mappedAssetUrls: {} + }); + + // no mapping applies, so the field must come through byte-for-byte, with no escape characters injected + expect(result.html_rte_field).to.equal(sourceStackEntry.html_rte_field); + }); + + it('should not let a UID that is a prefix of another UID clobber the longer one in HTML RTE', () => { + const ctSchema = [ + { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true } } + ]; + const sourceStackEntry = { + html_rte_field: '' + }; + const entry = { html_rte_field: '

' }; + + const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, { + uidMapper: { 'entry.1': 'bltShort', 'entry.10': 'bltLong' }, + mappedAssetUids: {}, + mappedAssetUrls: {} + }); + + expect(result.html_rte_field).to.equal(''); + }); + + it('should remap regex-special-character entry UIDs in a multiple HTML RTE field', () => { + const sourceUid = 'source.entry.1'; + const ctSchema = [ + { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true }, multiple: true } + ]; + // array-branch matching is exact-element (Array.indexOf), not substring, so the mapped + // UID must be a whole array element here to be found + const sourceStackEntry = { + html_rte_field: [sourceUid, 'unrelated value'] + }; + const entry = { html_rte_field: ['placeholder'] }; + + const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, { + uidMapper: { [sourceUid]: 'bltTargetEntry1' }, + mappedAssetUids: {}, + mappedAssetUrls: {} + }); + + expect(result.html_rte_field[0]).to.equal('bltTargetEntry1'); + expect(result.html_rte_field[1]).to.equal('unrelated value'); + }); + + it('should leave a UID embedded inside a larger string untouched in a multiple HTML RTE field', () => { + const sourceUid = 'source.entry.1'; + const ctSchema = [ + { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true }, multiple: true } + ]; + const sourceStackEntry = { + html_rte_field: [``] + }; + const entry = { html_rte_field: ['placeholder'] }; + + const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, { + uidMapper: { [sourceUid]: 'bltTargetEntry1' }, + mappedAssetUids: {}, + mappedAssetUrls: {} + }); + + // known limitation, not touched by this fix: the multiple branch matches with + // Array.prototype.indexOf (exact element equality), so a UID embedded inside a larger + // string element is never found and the field passes through unchanged + expect(result.html_rte_field[0]).to.equal(sourceStackEntry.html_rte_field[0]); + }); + it('should restore entry references in JSON RTE', () => { const entry = JSON.parse(JSON.stringify(mockEntries.entryWithJsonRteReference)); const sourceStackEntry = mockEntries.sourceStackEntryWithJsonRte;