diff --git a/infrastructure/terraform/components/dl/data/failure_codes.csv b/infrastructure/terraform/components/dl/data/failure_codes.csv index efb709e96..9c8f0b40e 100644 --- a/infrastructure/terraform/components/dl/data/failure_codes.csv +++ b/infrastructure/terraform/components/dl/data/failure_codes.csv @@ -2,4 +2,6 @@ code,description DL_PDMV_001,Letter rejected by PDM DL_PDMV_002,Timeout waiting for letter storage DL_CLIV_003,Attachment contains a virus +DL_CLIV_004,Duplicate request +DL_CLIV_005,Invalid FHIR resource DL_INTE_001,Request rejected by Core API diff --git a/infrastructure/terraform/components/dl/s3_object_failure_codes.tf b/infrastructure/terraform/components/dl/s3_object_failure_codes.tf index 78c33baef..61a606176 100644 --- a/infrastructure/terraform/components/dl/s3_object_failure_codes.tf +++ b/infrastructure/terraform/components/dl/s3_object_failure_codes.tf @@ -1,6 +1,3 @@ -# Auto-generated CSV containing failure code definitions -# Source: src/digital-letters-events/failure-codes.ts -# Build: make build / make generate (runs generate-dependencies) resource "aws_s3_object" "failure_codes" { bucket = module.s3bucket_reporting.bucket key = "reference-data/failure_codes/failure_codes.csv" diff --git a/lambdas/mesh-download/mesh_download/__tests__/test_processor.py b/lambdas/mesh-download/mesh_download/__tests__/test_processor.py index ecfc997be..078b03da8 100644 --- a/lambdas/mesh-download/mesh_download/__tests__/test_processor.py +++ b/lambdas/mesh-download/mesh_download/__tests__/test_processor.py @@ -68,6 +68,58 @@ def create_sqs_record(cloud_event=None): 'body': json.dumps({'detail': cloud_event}) } +def create_fhir_content(): + """ + Create a mock FHIR JSON content for testing + """ + return json.dumps({ + "resourceType": "DocumentReference", + "id": "82bfb7f3-4889-4e15-b308-bbe4e3cd431f", + "status": "current", + "docStatus": "final", + "type": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "308540004", + "display": "Appointment" + } + ] + }, + "subject": { + "identifier": { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9876543210" + } + }, + "author": [ + { + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "RX809" + }, + "display": "Example NHS Trust" + } + ], + "custodian": { + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "C4L8E" + }, + "display": "NHS ENGLAND: NHS NOTIFY" + }, + "date": "2025-11-19T14:30:00Z", + "description": "Appointment notification letter for outpatient consultation", + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "title": "Appointment Letter - November 2025", + "data": "base64here==" + } + } + ] + }) def create_mesh_message(message_id='test_123', sender='SENDER_001', local_id='ref_001'): """ @@ -80,8 +132,9 @@ def create_mesh_message(message_id='test_123', sender='SENDER_001', local_id='re message.subject = 'test_document.pdf' message.workflow_id = 'TEST_WORKFLOW' message.message_type = 'DATA' - message.read.return_value = b'Test message content' + message.read.return_value = create_fhir_content() message.acknowledge = Mock() + return message @@ -142,7 +195,7 @@ def test_process_sqs_message_success(self, mock_datetime): document_store.store_document.assert_called_once_with( sender_id='TEST_SENDER', message_reference='ref_001', - content=b'Test message content' + content=create_fhir_content() ) mesh_message.acknowledge.assert_called_once() @@ -177,6 +230,75 @@ def test_process_sqs_message_success(self, mock_datetime): assert event_data['messageUri'] == 's3://test-pii-bucket/document-reference/SENDER_001_ref_001' assert set(event_data.keys()) == {'senderId', 'messageReference', 'messageUri', 'meshMessageId'} + @patch('mesh_download.processor.datetime') + def test_process_sqs_message_invalid_fhir_content(self, mock_datetime): + from mesh_download.processor import MeshDownloadProcessor + + config, log, event_publisher, document_store = setup_mocks() + + fixed_time = datetime(2025, 11, 19, 15, 30, 45, tzinfo=timezone.utc) + mock_datetime.now.return_value = fixed_time + + document_store.store_document.return_value = 'document-reference/SENDER_001_ref_001' + + event_publisher.send_events.return_value = [] + + processor = MeshDownloadProcessor( + config=config, + log=log, + mesh_client=config.mesh_client, + download_metric=config.download_metric, + document_store=document_store, + event_publisher=event_publisher + ) + + mesh_message = create_mesh_message() + mesh_message.read.return_value = '{}' # invalid FHIR content (empty JSON)} + config.mesh_client.retrieve_message.return_value = mesh_message + + sqs_record = create_sqs_record() + + processor.process_sqs_message(sqs_record) + + config.mesh_client.retrieve_message.assert_called_once_with('test_message_123') + + mesh_message.read.assert_called_once() + + document_store.store_document.assert_not_called() + + mesh_message.acknowledge.assert_called_once() + + config.download_metric.record.assert_not_called() + + event_publisher.send_events.assert_called_once() + + # Verify the published event content + published_events = event_publisher.send_events.call_args[0][0] + assert len(published_events) == 1 + + published_event = published_events[0] + + # Verify CloudEvent envelope fields + assert published_event['type'] == 'uk.nhs.notify.digital.letters.mesh.inbox.message.invalid.v1' + assert published_event['source'] == '/nhs/england/notify/development/primary/data-plane/digitalletters/mesh' + assert published_event['subject'] == 'customer/00000000-0000-0000-0000-000000000000/recipient/00000000-0000-0000-0000-000000000000' + assert published_event['time'] == '2025-11-19T15:30:45+00:00' + assert 'id' in published_event + assert 'tracestate' not in published_event + assert 'partitionkey' not in published_event + assert 'sequence' not in published_event + assert 'dataclassification' not in published_event + assert 'dataregulation' not in published_event + assert 'datacategory' not in published_event + + # Verify CloudEvent data payload + event_data = published_event['data'] + assert event_data['senderId'] == 'TEST_SENDER' + assert event_data['messageReference'] == 'ref_001' + assert event_data['meshMessageId'] == 'test_message_123' + assert event_data['failureCode'] == 'DL_CLIV_005' + assert set(event_data.keys()) == {'senderId', 'messageReference', 'meshMessageId', 'failureCode'} + def test_process_sqs_message_validation_failure(self): """Malformed CloudEvents should be rejected by pydantic and not trigger downloads""" from mesh_download.processor import MeshDownloadProcessor diff --git a/lambdas/mesh-download/mesh_download/processor.py b/lambdas/mesh-download/mesh_download/processor.py index e51edcedb..1667b2652 100644 --- a/lambdas/mesh-download/mesh_download/processor.py +++ b/lambdas/mesh-download/mesh_download/processor.py @@ -3,8 +3,9 @@ from uuid import uuid4 from pydantic import ValidationError -from digital_letters_events import MESHInboxMessageDownloaded, MESHInboxMessageReceived +from digital_letters_events import MESHInboxMessageDownloaded, MESHInboxMessageReceived, MESHInboxMessageInvalid from mesh_download.errors import MeshMessageNotFound +from nhs_notify_letters_onboarding import validate class MeshDownloadProcessor: @@ -52,6 +53,10 @@ def _parse_and_validate_event(self, sqs_record): ) raise + def _validate_fhir_content(self, content): + json_content = json.loads(content) + validate(json_content) + def _handle_download(self, event, logger): data = event.data @@ -72,6 +77,18 @@ def _handle_download(self, event, logger): content = message.read() logger.info("Downloaded MESH message content") + try: + self._validate_fhir_content(content) + except Exception as e: + logger.error("FHIR content is invalid", error=str(e)) + + self._publish_message_invalid_event(incoming_event=event) + + message.acknowledge() + logger.info("Acknowledged message") + + return + uri = self._store_message_content( sender_id=data.senderId, message_reference=data.messageReference, @@ -139,3 +156,39 @@ def _publish_downloaded_event(self, incoming_event, message_uri): message_uri=message_uri, message_reference=incoming_event.data.messageReference ) + + def _publish_message_invalid_event(self, incoming_event): + """ + Publishes a MESHInboxMessageInvalid event. + """ + now = datetime.now(timezone.utc).isoformat() + + cloud_event = { + **incoming_event.model_dump(exclude_none=True), + 'id': str(uuid4()), + 'time': now, + 'recordedtime': now, + 'type': 'uk.nhs.notify.digital.letters.mesh.inbox.message.invalid.v1', + 'dataschema': ( + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/' + 'digital-letters-mesh-inbox-message-invalid-data.schema.json' + ), + 'data': { + 'senderId': incoming_event.data.senderId, + 'meshMessageId': incoming_event.data.meshMessageId, + 'failureCode': 'DL_CLIV_005', + 'messageReference': incoming_event.data.messageReference, + } + } + + failed = self.__event_publisher.send_events([cloud_event], MESHInboxMessageInvalid) + if failed: + msg = f"Failed to publish MESHInboxMessageInvalid event: {failed}" + self.__log.error(msg, failed_count=len(failed)) + raise RuntimeError(msg) + + self.__log.info( + "Published MESHInboxMessageInvalid event", + sender_id=incoming_event.data.senderId, + message_reference=incoming_event.data.messageReference + ) diff --git a/lambdas/mesh-download/requirements.txt b/lambdas/mesh-download/requirements.txt index b9af3fe0f..b3fac7e06 100644 --- a/lambdas/mesh-download/requirements.txt +++ b/lambdas/mesh-download/requirements.txt @@ -8,6 +8,7 @@ urllib3>=1.26.19,<2.0.0 idna>=3.7 requests>=2.32.0 pyopenssl>=24.2.1 +nhs-notify-digital-letters-onboarding @ git+https://github.com/NHSDigital/nhs-notify-digital-letters-onboarding@75362ff36814a0b355f95ad8b6834e400c49c161 -e ../../src/digital-letters-events -e ../../utils/py-mock-mesh -e ../../utils/py-utils diff --git a/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-invalid-data.schema.yaml b/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-invalid-data.schema.yaml index f94885a6c..666708021 100644 --- a/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-invalid-data.schema.yaml +++ b/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-invalid-data.schema.yaml @@ -10,6 +10,8 @@ properties: $ref: ../defs/requests.schema.yaml#/properties/senderId failureCode: $ref: ../defs/requests.schema.yaml#/properties/failureCode + messageReference: + $ref: ../defs/requests.schema.yaml#/properties/messageReference required: - meshMessageId - senderId diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index b3f6b9f6f..0ca126e33 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -69,6 +69,7 @@ export const PRINT_STATUS_HANDLER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-pr export const PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-analyser`; export const PRINT_SENDER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-sender`; export const MOVE_SCANNED_FILES_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-move-scanned-files`; +export const MESH_DOWNLOAD_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-mesh-download`; // Data Firehose export const FIREHOSE_STREAM_NAME = `${CSI}-to-s3-reporting`; diff --git a/tests/playwright/digital-letters-component-tests/mesh-poll-download.component.spec.ts b/tests/playwright/digital-letters-component-tests/mesh-poll-download.component.spec.ts index 7fde42e51..1fbb4002f 100644 --- a/tests/playwright/digital-letters-component-tests/mesh-poll-download.component.spec.ts +++ b/tests/playwright/digital-letters-component-tests/mesh-poll-download.component.spec.ts @@ -2,6 +2,7 @@ import { expect, test } from '@playwright/test'; import { ENV, MESH_DOWNLOAD_DLQ_NAME, + MESH_DOWNLOAD_LAMBDA_LOG_GROUP_NAME, MESH_POLL_LAMBDA_NAME, NON_PII_S3_BUCKET_NAME, PII_S3_BUCKET_NAME, @@ -16,6 +17,56 @@ import { v4 as uuidv4 } from 'uuid'; import { SENDER_ID_SKIPS_NOTIFY } from 'constants/tests-constants'; import { validateMESHInboxMessageReceived } from 'digital-letters-events'; +const validPdmRequest = { + resourceType: 'DocumentReference', + id: '82bfb7f3-4889-4e15-b308-bbe4e3cd431f', + status: 'current', + docStatus: 'final', + type: { + coding: [ + { + // eslint-disable-next-line sonarjs/no-clear-text-protocols + system: 'http://snomed.info/sct', + code: '308540004', + display: 'Appointment', + }, + ], + }, + subject: { + identifier: { + system: 'https://fhir.nhs.uk/Id/nhs-number', + value: '9876543210', + }, + }, + author: [ + { + identifier: { + system: 'https://fhir.nhs.uk/Id/ods-organization-code', + value: 'RX809', + }, + display: 'Example NHS Trust', + }, + ], + custodian: { + identifier: { + system: 'https://fhir.nhs.uk/Id/ods-organization-code', + value: 'C4L8E', + }, + display: 'NHS ENGLAND: NHS NOTIFY', + }, + date: '2025-11-19T14:30:00Z', + description: 'Appointment notification letter for outpatient consultation', + content: [ + { + attachment: { + contentType: 'application/pdf', + title: 'Appointment Letter - November 2025', + data: 'base64here==', + }, + }, + ], +}; + test.describe('Digital Letters - MESH Poll and Download', () => { const senderId = SENDER_ID_SKIPS_NOTIFY; const sendersMeshMailboxId = 'test-mesh-sender-1'; @@ -75,15 +126,31 @@ test.describe('Digital Letters - MESH Poll and Download', () => { }, 180_000); } + async function expectMeshInboxMessageInvalidEvent( + meshMessageId: string, + messageReference: string, + ): Promise { + await expectToPassEventually(async () => { + const eventLogEntry = await getLogsFromCloudwatch( + `/aws/vendedlogs/events/event-bus/nhs-${ENV}-dl`, + [ + '$.message_type = "EVENT_RECEIPT"', + '$.details.detail_type = "uk.nhs.notify.digital.letters.mesh.inbox.message.invalid.v1"', + `$.details.event_detail = "*\\"messageReference\\":\\"${messageReference}\\"*"`, + `$.details.event_detail = "*\\"senderId\\":\\"${senderId}\\"*"`, + `$.details.event_detail = "*\\"meshMessageId\\":\\"${meshMessageId}\\"*"`, + `$.details.event_detail = "*\\"failureCode\\":\\"DL_CLIV_005\\"*"`, + ], + ); + + expect(eventLogEntry.length).toBeGreaterThanOrEqual(1); + }, 180_000); + } + test('should poll message from MESH inbox, publish received event, download message, and publish downloaded event', async () => { const meshMessageId = `${Date.now()}_TEST_${uuidv4().slice(0, 8)}`; const messageReference = uuidv4(); - const messageContent = JSON.stringify({ - senderId, - messageReference, - testData: 'This is a test letter content', - timestamp: new Date().toISOString(), - }); + const messageContent = JSON.stringify(validPdmRequest); await uploadMeshMessage(meshMessageId, messageReference, messageContent); @@ -111,6 +178,43 @@ test.describe('Digital Letters - MESH Poll and Download', () => { }, 60_000); }); + test('given invalid PDM request should publish invalid event, log an error, acknowledge message', async () => { + const meshMessageId = `${Date.now()}_TEST_${uuidv4().slice(0, 8)}`; + const messageReference = uuidv4(); + const invalidPdmRequest = { ...validPdmRequest, id: undefined }; + + const messageContent = JSON.stringify(invalidPdmRequest); + + await uploadMeshMessage(meshMessageId, messageReference, messageContent); + + await invokeLambda(MESH_POLL_LAMBDA_NAME); + + await expectMeshInboxMessageReceivedEvent(meshMessageId); + await expectMeshInboxMessageInvalidEvent(meshMessageId, messageReference); + + await expectToPassEventually(async () => { + const filteredLogs = await getLogsFromCloudwatch( + MESH_DOWNLOAD_LAMBDA_LOG_GROUP_NAME, + [ + '$.event = "FHIR content is invalid"', + `$.mesh_message_id = "${meshMessageId}"`, + '$.error = "\'id\' is a required property*"', + ], + ); + + expect(filteredLogs.length).toEqual(1); + }, 120); + + await expectToPassEventually(async () => { + await expect(async () => { + await downloadFromS3( + NON_PII_S3_BUCKET_NAME, + `mock-mesh/${meshMailboxId}/in/${meshMessageId}`, + ); + }).rejects.toThrow('No objects found'); + }, 60_000); + }); + test('should send message to mesh-download DLQ when download fails', async () => { test.setTimeout(160_000);