CCM-17899: [not to merge] migrate personalisation fields

data-migration/personalisation/.eslintignore

@@ -0,0 +1 @@
+dist

data-migration/personalisation/.gitignore

@@ -0,0 +1,4 @@
+.build
+coverage
+node_modules
+dist

data-migration/personalisation/package.json

@@ -0,0 +1,24 @@
+{
+  "dependencies": {
+    "@aws-sdk/client-dynamodb": "3.995.0",
+    "@aws-sdk/lib-dynamodb": "3.995.0",
+    "nhs-notify-backend-client": "^0.0.1",
+    "yargs": "^18.0.0",
+    "zod": "^4.0.17"
+  },
+  "devDependencies": {
+    "@tsconfig/node22": "^22.0.5",
+    "@types/yargs": "^17.0.35",
+    "typescript": "^5.9.3"
+  },
+  "name": "migrate-personalisation",
+  "private": true,
+  "scripts": {
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "migrate": "tsx ./src/index.ts",
+    "test:unit": "echo none",
+    "typecheck": "tsc --noEmit"
+  },
+  "version": "0.0.1"
+}

data-migration/personalisation/src/index.ts

@@ -0,0 +1,157 @@
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import {
+  DynamoDBDocumentClient,
+  ScanCommand,
+  UpdateCommand,
+} from '@aws-sdk/lib-dynamodb';
+import yargs from 'yargs/yargs';
+import { hideBin } from 'yargs/helpers';
+import { $TemplateDto } from 'nhs-notify-backend-client/schemas';
+import { extractTemplatePersonalisation } from './personalisation';
+import z from 'zod/v4';
+
+/*
+  npm -w migrate-personalisation run migrate -- --table-name nhs-notify-alnu1-sbx-api-templates
+
+  after running, you can re-run without --real-run to confirm no more updates required
+*/
+
+const ddb = DynamoDBDocumentClient.from(
+  new DynamoDBClient({ region: 'eu-west-2' }),
+  {
+    marshallOptions: { removeUndefinedValues: true },
+  }
+);
+
+const { tableName, realRun } = yargs(hideBin(process.argv))
+  .options({
+    tableName: {
+      type: 'string',
+      demandOption: true,
+    },
+    realRun: {
+      type: 'boolean',
+      default: false,
+    },
+  })
+  .parseSync();
+
+async function main() {
+  let lastEvaluatedKey;
+
+  let updatesRequired = 0;
+  let updatesDone = 0;
+
+  do {
+    const scanCmd: ScanCommand = new ScanCommand({
+      TableName: tableName,
+      ExclusiveStartKey: lastEvaluatedKey,
+      FilterExpression: 'templateType <> :letter',
+      ExpressionAttributeValues: {
+        ':letter': 'LETTER',
+      },
+    });
+
+    const response = await ddb.send(scanCmd);
+
+    lastEvaluatedKey = response.LastEvaluatedKey;
+
+    for (const item of response.Items ?? []) {
+      const template = z
+        .intersection(
+          $TemplateDto,
+          z.object({
+            owner: z.string(),
+            customPersonalisation: z.string().array().optional(),
+            systemPersonalisation: z.string().array().optional(),
+          })
+        )
+        .parse(item);
+
+      if (!template.customPersonalisation || !template.systemPersonalisation) {
+        updatesRequired += 1;
+      } else {
+        console.log(
+          `template ${template.id} already has personalisation fields`
+        );
+        continue;
+      }
+
+      let extracted: { custom: string[]; system: string[] };
+
+      switch (template.templateType) {
+        case 'SMS':
+        case 'NHS_APP': {
+          extracted = extractTemplatePersonalisation(
+            template.id,
+            template.message
+          );
+          break;
+        }
+        case 'EMAIL': {
+          const fromMessage = extractTemplatePersonalisation(
+            template.id,
+            template.message
+          );
+          const fromSubject = extractTemplatePersonalisation(
+            template.id,
+            template.subject
+          );
+
+          extracted = {
+            custom: [
+              ...new Set([...fromMessage.custom, ...fromSubject.custom]),
+            ],
+            system: [
+              ...new Set([...fromMessage.system, ...fromSubject.system]),
+            ],
+          };
+          break;
+        }
+        default: {
+          throw new Error(`unexpected templateType ${template.templateType}`);
+        }
+      }
+
+      console.table({
+        id: template.id,
+        type: template.templateType,
+        custom: JSON.stringify(extracted.custom),
+        system: JSON.stringify(extracted.system),
+      });
+
+      if (!realRun) continue;
+
+      const updateCmd = new UpdateCommand({
+        TableName: tableName,
+        Key: {
+          owner: template.owner,
+          id: template.id,
+        },
+        ExpressionAttributeValues: {
+          ':systemPersonalisation': extracted.system,
+          ':customPersonalisation': extracted.custom,
+          ':expectedLockNumber': template.lockNumber,
+        },
+        UpdateExpression:
+          'SET systemPersonalisation = :systemPersonalisation, customPersonalisation = :customPersonalisation',
+        ConditionExpression:
+          'attribute_not_exists(lockNumber) OR lockNumber = :expectedLockNumber',
+      });
+
+      await ddb.send(updateCmd);
+
+      console.log(`updated ${template.id}`);
+      updatesDone += 1;
+    }
+  } while (lastEvaluatedKey != null);
+
+  console.table({ updatesRequired, updatesDone });
+}
+
+// eslint-disable-next-line unicorn/prefer-top-level-await
+main().catch((error) => {
+  console.error(error);
+  // eslint-disable-next-line unicorn/no-process-exit
+  process.exit(1);
+});

data-migration/personalisation/src/personalisation.ts

@@ -0,0 +1,49 @@
+import { DEFAULT_PERSONALISATION_LIST } from 'nhs-notify-backend-client/schemas';
+
+const DEFAULT_PERSONALISATION_SET = new Set(DEFAULT_PERSONALISATION_LIST);
+const DIGITAL_PERSONALISATION_PATTERN = /\(\((\w+)\)\)/g;
+
+function dedupePersonalisation(parameters: string[]): string[] {
+  return [...new Set(parameters)];
+}
+
+function classifyPersonalisation(parameters: string[]) {
+  const system: string[] = [];
+  const custom: string[] = [];
+
+  for (const parameter of dedupePersonalisation(parameters)) {
+    if (DEFAULT_PERSONALISATION_SET.has(parameter)) {
+      system.push(parameter);
+    } else {
+      custom.push(parameter);
+    }
+  }
+
+  return { system, custom };
+}
+
+export function extractTemplatePersonalisation(id: string, str: string) {
+  const parameters: string[] = [];
+
+  for (const [, match] of str.matchAll(DIGITAL_PERSONALISATION_PATTERN)) {
+    if (
+      // these would become custom, rather than default personalisation after https://nhsd-jira.digital.nhs.uk/browse/CCM-18022
+      // check that they don't exist in prod. If they do, this will require some thought
+      match === 'clientRef' ||
+      match === 'recipientContactValue' ||
+      match === 'template' ||
+      // these were banned in frontend validation in https://github.com/NHSDigital/nhs-notify-web-template-management/issues/776
+      // check that we don't have anything in prod pre-dating the check
+      match === 'date' ||
+      /^address_line_\d$/.test(match)
+    ) {
+      console.warn(
+        `template ${id} contains problematic personalisation ${match}`
+      );
+    }
+
+    parameters.push(match);
+  }
+
+  return classifyPersonalisation(dedupePersonalisation(parameters));
+}

data-migration/personalisation/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "@tsconfig/node22/tsconfig.json",
+  "include": [
+    "src/**/*"
+  ]
+}