fix(import): remap asset and entry UIDs containing regex metacharacters

.talismanrc

@@ -39,4 +39,6 @@ fileignoreconfig:
  checksum: a64a4d396eddd936a63b799eff58c5c6660b5dcaa3a310fd8b09a027932f1789
 - filename: packages/contentstack-migration/README.md
  checksum: e96006c1a948f766c88ae972b29582fa58eaf8184606bf011eebddc5a06cd7b6
+- filename: packages/contentstack-import/test/unit/utils/asset-helper.test.ts
+ checksum: da1d476d0a7aaaaee8a355571d85facc5f4de081ede40c13fd263df3bfb5c521
 version: ""

packages/contentstack-import/src/utils/asset-helper.ts

@@ -1,7 +1,7 @@
 import Bluebird from 'bluebird';
 import * as url from 'url';
 import * as path from 'path';
-import { ContentstackClient, managementSDKClient, validateRegex, log } from '@contentstack/cli-utilities';
+import { ContentstackClient, managementSDKClient, log } from '@contentstack/cli-utilities';
 import { ImportConfig } from '../types';
 const debug = require('debug')('util:requests');
 let _ = require('lodash');
@@ -10,7 +10,8 @@ let helper = require('./file-helper');
 
 const MAX_RETRY_LIMIT = 5;
 
-const escapeRegExp = (str: string) => str.replace(/[*+?^${}()|[\]\\]/g, '\\$&');
+// escapes a value the way it appears inside a JSON string, without the surrounding quotes
+const jsonEscape = (str: string) => JSON.stringify(str).slice(1, -1);
 
 function validate(req: any) {
  if (typeof req !== 'object') {
@@ -293,9 +294,17 @@ export const lookupAssets = function (
  assetUids.forEach(function (assetUid: any) {
  let uid = mappedAssetUids[assetUid];
  if (typeof uid !== 'undefined') {
- const escapedAssetUid = assetUid.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
- entry = entry.split(escapedAssetUid).join(uid);
- matchedUids.push(assetUid);
+ // split() matches literally, so the UID must not be regex-escaped. It does need JSON
+ // escaping though, since the search runs on the serialized entry (a UID containing a
+ // backslash or a quote appears escaped there).
+ const updatedEntry = entry.split(jsonEscape(assetUid)).join(jsonEscape(uid));
+ if (updatedEntry !== entry) {
+ entry = updatedEntry;
+ matchedUids.push(assetUid);
+ } else {
+ log.debug(`Asset UID ${assetUid} had a mapping but no occurrence in entry ${data.entry?.uid}`);
+ unmatchedUids.push(assetUid);
+ }
  } else {
  unmatchedUids.push(assetUid);
  }

packages/contentstack-import/src/utils/entries-helper.ts

@@ -603,13 +603,9 @@ export const restoreJsonRteEntryRefs = (
  if (sourceStackEntry[element.uid].indexOf(uid) !== -1) return uid;
  });
  if (element.multiple && Array.isArray(entry[element.uid])) {
- for (let i = 0; i < matches.length; i++) {
- entry[element.uid] = entry[element.uid].map((el: string) => updateUids(el, matches[i], uidMapper));
- }
+ entry[element.uid] = entry[element.uid].map((el: string) => updateUids(el, matches, uidMapper));
  } else {
- for (let i = 0; i < matches.length; i++) {
- entry[element.uid] = updateUids(entry[element.uid], matches[i], uidMapper);
- }
+ entry[element.uid] = updateUids(entry[element.uid], matches, uidMapper);
  }
  }
  break;
@@ -619,10 +615,12 @@ export const restoreJsonRteEntryRefs = (
  return entry;
 };
 
-function updateUids(str: string, match: string, uidMapper: Record<string, string>) {
- const sanitizedMatch = escapeRegExp(match);
- const replacement = uidMapper[match] ?? sanitizedMatch;
- return str.split(sanitizedMatch).join(replacement);
+function updateUids(str: string, matches: string[], uidMapper: Record<string, string>) {
+ if (!matches.length) return str;
+ // longest-first so a UID that's a prefix of another (entry.1 vs entry.10) never wins the match;
+ // single regex pass so a replacement value can never itself get re-scanned/re-replaced
+ const pattern = [...matches].sort((a, b) => b.length - a.length).map(escapeRegExp).join('|');
+ return str.replace(new RegExp(pattern, 'g'), (match) => uidMapper[match] ?? match);
 }
 
 function setDirtyTrue(jsonRteChild: any) {

packages/contentstack-import/test/unit/utils/asset-helper.test.ts

@@ -426,6 +426,117 @@ describe('Asset Helper', () => {
  expect(result).to.exist;
  });
 
+ it('should remap JSON RTE asset UIDs containing regex special characters', () => {
+ const data = {
+ entry: {
+ uid: 'entry1',
+ json_rte: {
+ children: [
+ {
+ type: 'reference',
+ attrs: {
+ type: 'asset',
+ 'asset-uid': 'safe_6.0_big_picture_sorrento_web',
+ 'asset-link': '/assets/safe_6.0_big_picture_sorrento_web'
+ },
+ children: [] as any
+ }
+ ]
+ }
+ },
+ content_type: {
+ uid: 'ct1',
+ schema: [
+ {
+ uid: 'json_rte',
+ data_type: 'json',
+ field_metadata: { rich_text_type: true }
+ }
+ ]
+ }
+ };
+ const mappedAssetUids = { 'safe_6.0_big_picture_sorrento_web': 'bltNewAssetUid' };
+
+ const result = lookupAssets(data, mappedAssetUids, {}, '/test/mapper', []);
+
+ expect(JSON.stringify(result)).to.not.include('safe_6.0_big_picture_sorrento_web');
+ expect(JSON.stringify(result)).to.include('bltNewAssetUid');
+ });
+
+ it('should remap asset UIDs for every regex special character', () => {
+ const specialCharUids = [
+ 'asset.1',
+ 'asset*1',
+ 'asset+1',
+ 'asset?1',
+ 'asset^1',
+ 'asset$1',
+ 'asset{1}',
+ 'asset(1)',
+ 'asset|1',
+ 'asset[1]',
+ 'asset\\1'
+ ];
+
+ specialCharUids.forEach((assetUid, index) => {
+ const mappedUid = `bltMapped${index}`;
+ const data = {
+ entry: {
+ uid: 'entry1',
+ json_rte: {
+ children: [
+ {
+ type: 'reference',
+ attrs: { type: 'asset', 'asset-uid': assetUid },
+ children: [] as any
+ }
+ ]
+ }
+ },
+ content_type: {
+ uid: 'ct1',
+ schema: [
+ { uid: 'json_rte', data_type: 'json', field_metadata: { rich_text_type: true } }
+ ]
+ }
+ };
+
+ const result = lookupAssets(data, { [assetUid]: mappedUid }, {}, '/test/mapper', []);
+
+ expect(result.json_rte.children[0].attrs['asset-uid'], `failed for UID: ${assetUid}`).to.equal(mappedUid);
+ });
+ });
+
+ it('should record a special character asset UID as matched', () => {
+ const writeFileStub = fileHelper.writeFile as unknown as sinon.SinonStub;
+ const data = {
+ entry: {
+ uid: 'entry1',
+ json_rte: {
+ children: [
+ {
+ type: 'reference',
+ attrs: { type: 'asset', 'asset-uid': 'asset.with.dots' },
+ children: [] as any
+ }
+ ]
+ }
+ },
+ content_type: {
+ uid: 'ct1',
+ schema: [
+ { uid: 'json_rte', data_type: 'json', field_metadata: { rich_text_type: true } }
+ ]
+ }
+ };
+
+ lookupAssets(data, { 'asset.with.dots': 'bltNewAssetUid' }, {}, '/test/mapper', []);
+
+ const writtenPaths = writeFileStub.getCalls().map((call: any) => call.args[0]);
+ expect(writtenPaths.some((p: string) => p.includes('matched-asset-uids.json'))).to.be.true;
+ expect(writtenPaths.some((p: string) => p.includes('unmatched-asset-uids.json'))).to.be.false;
+ });
+
  it('should handle JSON custom fields with extensions', () => {
  const data = {
  entry: {

packages/contentstack-import/test/unit/utils/entries-helper.test.ts

@@ -384,6 +384,110 @@ describe('Entries Helper', () => {
  expect(restoreJsonRteEntryRefs).to.be.a('function');
  });
 
+ it('should remap entry UIDs containing regex special characters in HTML RTE', () => {
+ const sourceUid = 'source.entry.1';
+ const ctSchema = [
+ { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true } }
+ ];
+ const sourceStackEntry = {
+ html_rte_field: `<p>ref</p><span data-uid="${sourceUid}"></span>`
+ };
+ // non-empty: restoreJsonRteEntryRefs skips falsy fields before restoring from source
+ const entry = { html_rte_field: '<p></p>' };
+
+ const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, {
+ uidMapper: { [sourceUid]: 'bltTargetEntry1' },
+ mappedAssetUids: {},
+ mappedAssetUrls: {}
+ });
+
+ expect(result.html_rte_field).to.not.include(sourceUid);
+ expect(result.html_rte_field).to.include('bltTargetEntry1');
+ });
+
+ it('should leave an unmapped special character entry UID untouched in HTML RTE', () => {
+ const sourceUid = 'source.entry.1';
+ const ctSchema = [
+ { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true } }
+ ];
+ const sourceStackEntry = {
+ html_rte_field: `<span data-uid="${sourceUid}"></span>`
+ };
+ // non-empty: restoreJsonRteEntryRefs skips falsy fields before restoring from source
+ const entry = { html_rte_field: '<p></p>' };
+
+ const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, {
+ uidMapper: { 'other.entry': 'bltOther' },
+ mappedAssetUids: {},
+ mappedAssetUrls: {}
+ });
+
+ // no mapping applies, so the field must come through byte-for-byte, with no escape characters injected
+ expect(result.html_rte_field).to.equal(sourceStackEntry.html_rte_field);
+ });
+
+ it('should not let a UID that is a prefix of another UID clobber the longer one in HTML RTE', () => {
+ const ctSchema = [
+ { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true } }
+ ];
+ const sourceStackEntry = {
+ html_rte_field: '<span data-uid="entry.10"></span><span data-uid="entry.1"></span>'
+ };
+ const entry = { html_rte_field: '<p></p>' };
+
+ const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, {
+ uidMapper: { 'entry.1': 'bltShort', 'entry.10': 'bltLong' },
+ mappedAssetUids: {},
+ mappedAssetUrls: {}
+ });
+
+ expect(result.html_rte_field).to.equal('<span data-uid="bltLong"></span><span data-uid="bltShort"></span>');
+ });
+
+ it('should remap regex-special-character entry UIDs in a multiple HTML RTE field', () => {
+ const sourceUid = 'source.entry.1';
+ const ctSchema = [
+ { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true }, multiple: true }
+ ];
+ // array-branch matching is exact-element (Array.indexOf), not substring, so the mapped
+ // UID must be a whole array element here to be found
+ const sourceStackEntry = {
+ html_rte_field: [sourceUid, 'unrelated value']
+ };
+ const entry = { html_rte_field: ['placeholder'] };
+
+ const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, {
+ uidMapper: { [sourceUid]: 'bltTargetEntry1' },
+ mappedAssetUids: {},
+ mappedAssetUrls: {}
+ });
+
+ expect(result.html_rte_field[0]).to.equal('bltTargetEntry1');
+ expect(result.html_rte_field[1]).to.equal('unrelated value');
+ });
+
+ it('should leave a UID embedded inside a larger string untouched in a multiple HTML RTE field', () => {
+ const sourceUid = 'source.entry.1';
+ const ctSchema = [
+ { uid: 'html_rte_field', data_type: 'text', field_metadata: { rich_text_type: true }, multiple: true }
+ ];
+ const sourceStackEntry = {
+ html_rte_field: [`<span data-uid="${sourceUid}"></span>`]
+ };
+ const entry = { html_rte_field: ['placeholder'] };
+
+ const result = restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema, {
+ uidMapper: { [sourceUid]: 'bltTargetEntry1' },
+ mappedAssetUids: {},
+ mappedAssetUrls: {}
+ });
+
+ // known limitation, not touched by this fix: the multiple branch matches with
+ // Array.prototype.indexOf (exact element equality), so a UID embedded inside a larger
+ // string element is never found and the field passes through unchanged
+ expect(result.html_rte_field[0]).to.equal(sourceStackEntry.html_rte_field[0]);
+ });
+
  it('should restore entry references in JSON RTE', () => {
  const entry = JSON.parse(JSON.stringify(mockEntries.entryWithJsonRteReference));
  const sourceStackEntry = mockEntries.sourceStackEntryWithJsonRte;