diff --git a/data-migration/personalisation/.eslintignore b/data-migration/personalisation/.eslintignore new file mode 100644 index 000000000..1521c8b76 --- /dev/null +++ b/data-migration/personalisation/.eslintignore @@ -0,0 +1 @@ +dist diff --git a/data-migration/personalisation/.gitignore b/data-migration/personalisation/.gitignore new file mode 100644 index 000000000..9b19292a7 --- /dev/null +++ b/data-migration/personalisation/.gitignore @@ -0,0 +1,4 @@ +.build +coverage +node_modules +dist diff --git a/data-migration/personalisation/package.json b/data-migration/personalisation/package.json new file mode 100644 index 000000000..1f4aedb63 --- /dev/null +++ b/data-migration/personalisation/package.json @@ -0,0 +1,24 @@ +{ + "dependencies": { + "@aws-sdk/client-dynamodb": "3.995.0", + "@aws-sdk/lib-dynamodb": "3.995.0", + "nhs-notify-backend-client": "^0.0.1", + "yargs": "^18.0.0", + "zod": "^4.0.17" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.5", + "@types/yargs": "^17.0.35", + "typescript": "^5.9.3" + }, + "name": "migrate-personalisation", + "private": true, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "migrate": "tsx ./src/index.ts", + "test:unit": "echo none", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/data-migration/personalisation/src/index.ts b/data-migration/personalisation/src/index.ts new file mode 100644 index 000000000..8a74f1276 --- /dev/null +++ b/data-migration/personalisation/src/index.ts @@ -0,0 +1,157 @@ +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { + DynamoDBDocumentClient, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; +import yargs from 'yargs/yargs'; +import { hideBin } from 'yargs/helpers'; +import { $TemplateDto } from 'nhs-notify-backend-client/schemas'; +import { extractTemplatePersonalisation } from './personalisation'; +import z from 'zod/v4'; + +/* + npm -w migrate-personalisation run migrate -- --table-name nhs-notify-alnu1-sbx-api-templates + + after running, you can re-run without --real-run to confirm no more updates required +*/ + +const ddb = DynamoDBDocumentClient.from( + new DynamoDBClient({ region: 'eu-west-2' }), + { + marshallOptions: { removeUndefinedValues: true }, + } +); + +const { tableName, realRun } = yargs(hideBin(process.argv)) + .options({ + tableName: { + type: 'string', + demandOption: true, + }, + realRun: { + type: 'boolean', + default: false, + }, + }) + .parseSync(); + +async function main() { + let lastEvaluatedKey; + + let updatesRequired = 0; + let updatesDone = 0; + + do { + const scanCmd: ScanCommand = new ScanCommand({ + TableName: tableName, + ExclusiveStartKey: lastEvaluatedKey, + FilterExpression: 'templateType <> :letter', + ExpressionAttributeValues: { + ':letter': 'LETTER', + }, + }); + + const response = await ddb.send(scanCmd); + + lastEvaluatedKey = response.LastEvaluatedKey; + + for (const item of response.Items ?? []) { + const template = z + .intersection( + $TemplateDto, + z.object({ + owner: z.string(), + customPersonalisation: z.string().array().optional(), + systemPersonalisation: z.string().array().optional(), + }) + ) + .parse(item); + + if (!template.customPersonalisation || !template.systemPersonalisation) { + updatesRequired += 1; + } else { + console.log( + `template ${template.id} already has personalisation fields` + ); + continue; + } + + let extracted: { custom: string[]; system: string[] }; + + switch (template.templateType) { + case 'SMS': + case 'NHS_APP': { + extracted = extractTemplatePersonalisation( + template.id, + template.message + ); + break; + } + case 'EMAIL': { + const fromMessage = extractTemplatePersonalisation( + template.id, + template.message + ); + const fromSubject = extractTemplatePersonalisation( + template.id, + template.subject + ); + + extracted = { + custom: [ + ...new Set([...fromMessage.custom, ...fromSubject.custom]), + ], + system: [ + ...new Set([...fromMessage.system, ...fromSubject.system]), + ], + }; + break; + } + default: { + throw new Error(`unexpected templateType ${template.templateType}`); + } + } + + console.table({ + id: template.id, + type: template.templateType, + custom: JSON.stringify(extracted.custom), + system: JSON.stringify(extracted.system), + }); + + if (!realRun) continue; + + const updateCmd = new UpdateCommand({ + TableName: tableName, + Key: { + owner: template.owner, + id: template.id, + }, + ExpressionAttributeValues: { + ':systemPersonalisation': extracted.system, + ':customPersonalisation': extracted.custom, + ':expectedLockNumber': template.lockNumber, + }, + UpdateExpression: + 'SET systemPersonalisation = :systemPersonalisation, customPersonalisation = :customPersonalisation', + ConditionExpression: + 'attribute_not_exists(lockNumber) OR lockNumber = :expectedLockNumber', + }); + + await ddb.send(updateCmd); + + console.log(`updated ${template.id}`); + updatesDone += 1; + } + } while (lastEvaluatedKey != null); + + console.table({ updatesRequired, updatesDone }); +} + +// eslint-disable-next-line unicorn/prefer-top-level-await +main().catch((error) => { + console.error(error); + // eslint-disable-next-line unicorn/no-process-exit + process.exit(1); +}); diff --git a/data-migration/personalisation/src/personalisation.ts b/data-migration/personalisation/src/personalisation.ts new file mode 100644 index 000000000..f1bfc96e8 --- /dev/null +++ b/data-migration/personalisation/src/personalisation.ts @@ -0,0 +1,49 @@ +import { DEFAULT_PERSONALISATION_LIST } from 'nhs-notify-backend-client/schemas'; + +const DEFAULT_PERSONALISATION_SET = new Set(DEFAULT_PERSONALISATION_LIST); +const DIGITAL_PERSONALISATION_PATTERN = /\(\((\w+)\)\)/g; + +function dedupePersonalisation(parameters: string[]): string[] { + return [...new Set(parameters)]; +} + +function classifyPersonalisation(parameters: string[]) { + const system: string[] = []; + const custom: string[] = []; + + for (const parameter of dedupePersonalisation(parameters)) { + if (DEFAULT_PERSONALISATION_SET.has(parameter)) { + system.push(parameter); + } else { + custom.push(parameter); + } + } + + return { system, custom }; +} + +export function extractTemplatePersonalisation(id: string, str: string) { + const parameters: string[] = []; + + for (const [, match] of str.matchAll(DIGITAL_PERSONALISATION_PATTERN)) { + if ( + // these would become custom, rather than default personalisation after https://nhsd-jira.digital.nhs.uk/browse/CCM-18022 + // check that they don't exist in prod. If they do, this will require some thought + match === 'clientRef' || + match === 'recipientContactValue' || + match === 'template' || + // these were banned in frontend validation in https://github.com/NHSDigital/nhs-notify-web-template-management/issues/776 + // check that we don't have anything in prod pre-dating the check + match === 'date' || + /^address_line_\d$/.test(match) + ) { + console.warn( + `template ${id} contains problematic personalisation ${match}` + ); + } + + parameters.push(match); + } + + return classifyPersonalisation(dedupePersonalisation(parameters)); +} diff --git a/data-migration/personalisation/tsconfig.json b/data-migration/personalisation/tsconfig.json new file mode 100644 index 000000000..e509ff9ba --- /dev/null +++ b/data-migration/personalisation/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "@tsconfig/node22/tsconfig.json", + "include": [ + "src/**/*" + ] +} diff --git a/package-lock.json b/package-lock.json index d58e7a9d7..4d8e3105f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,8 @@ "utils/entity-update-command-builder", "utils/event-builder", "utils/test-helper-utils", - "utils/utils" + "utils/utils", + "data-migration/personalisation" ], "devDependencies": { "@tsconfig/node22": "^22.0.5", @@ -53,6 +54,141 @@ "typescript": "^5.9.3" } }, + "data-migration/personalisation": { + "name": "migrate-personalisation", + "version": "0.0.1", + "dependencies": { + "@aws-sdk/client-dynamodb": "3.995.0", + "@aws-sdk/lib-dynamodb": "3.995.0", + "nhs-notify-backend-client": "^0.0.1", + "yargs": "^18.0.0", + "zod": "^4.0.17" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.5", + "@types/yargs": "^17.0.35", + "typescript": "^5.9.3" + } + }, + "data-migration/personalisation/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "data-migration/personalisation/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "data-migration/personalisation/node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "data-migration/personalisation/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "data-migration/personalisation/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "data-migration/personalisation/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "data-migration/personalisation/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "data-migration/personalisation/node_modules/yargs": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", + "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^7.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "data-migration/personalisation/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, "frontend": { "name": "nhs-notify-web-template-management-frontend", "version": "0.1.0", @@ -11911,7 +12047,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -13417,6 +13552,18 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -16948,6 +17095,10 @@ "node": ">=8.6" } }, + "node_modules/migrate-personalisation": { + "resolved": "data-migration/personalisation", + "link": true + }, "node_modules/mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -22201,7 +22352,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" diff --git a/package.json b/package.json index 08503e87b..d952d8c12 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "utils/entity-update-command-builder", "utils/event-builder", "utils/test-helper-utils", - "utils/utils" + "utils/utils", + "data-migration/personalisation" ] }