CCM-15342: Configure python lambda to output level

lambdas/file-scanner-lambda/src/__tests__/apis/sqs-handler.test.ts

@@ -112,6 +112,7 @@ describe('SQS Handler', () => {
           description: 'Processing ItemDequeued event',
           messageReference: 'ref-001',
           senderId: 'SENDER_001',
+          messageId: 'msg-001',
         }),
       );
 
@@ -347,6 +348,7 @@ describe('SQS Handler', () => {
       expect(mockLogger.warn).toHaveBeenCalledWith(
         expect.objectContaining({
           description: 'Error parsing SQS record',
+          messageId: 'msg-001',
         }),
       );
     });

lambdas/file-scanner-lambda/src/apis/sqs-handler.ts

@@ -33,6 +33,7 @@ function validateRecord(
     logger.warn({
       err: error,
       description: 'Error parsing SQS record',
+      messageId,
     });
     return null;
   }
@@ -49,6 +50,7 @@ async function processRecord(
     eventId: event.id,
     messageReference: event.data.messageReference,
     senderId: event.data.senderId,
+    messageId: validatedRecord.messageId,
   });
 
   const result = await fileScanner.scanFile(event.data.messageUri, {

lambdas/mesh-acknowledge/mesh_acknowledge/acknowledger.py

@@ -59,7 +59,7 @@ def acknowledge_message(self,
                 "Acknowledged MESH message",
                 mesh_mailbox_id=mailbox_id,
                 mesh_message_id=message_id,
-                mesh_message_reference=message_reference,
+                message_reference=message_reference,
                 ack_message_id=ack_message_id
             )
 
@@ -70,7 +70,7 @@ def acknowledge_message(self,
                 "Failed to acknowledge MESH message",
                 mesh_mailbox_id=mailbox_id,
                 mesh_message_id=message_id,
-                mesh_message_reference=message_reference,
+                message_reference=message_reference,
                 error=str(e)
             )
 

lambdas/mesh-download/mesh_download/processor.py

@@ -26,9 +26,13 @@ def __init__(self, **kwargs):
         self.__storage_bucket = self.__config.transactional_data_bucket
 
     def process_sqs_message(self, sqs_record):
+        sqs_message_id = sqs_record.get('messageId')
         try:
-            validated_event = self._parse_and_validate_event(sqs_record)
-            logger = self.__log.bind(mesh_message_id=validated_event.data.meshMessageId)
+            validated_event = self._parse_and_validate_event(sqs_record, sqs_message_id)
+            logger = self.__log.bind(mesh_message_id=validated_event.data.meshMessageId,
+                                    message_reference=validated_event.data.messageReference,
+                                    sender_id=validated_event.data.senderId,
+                                    sqs_message_id=sqs_message_id)
 
             logger.info("Processing MESH download request")
             return self._handle_download(validated_event, logger)
@@ -41,7 +45,7 @@ def process_sqs_message(self, sqs_record):
             )
             raise
 
-    def _parse_and_validate_event(self, sqs_record):
+    def _parse_and_validate_event(self, sqs_record, sqs_message_id) -> MESHInboxMessageReceived:
         message_body = json.loads(sqs_record['body'])
         event_detail = message_body.get('detail', {})
 
@@ -53,6 +57,7 @@ def _parse_and_validate_event(self, sqs_record):
             self.__log.error(
                 "CloudEvent validation failed",
                 validation_errors=str(e),
+                sqs_message_id=sqs_message_id,
                 event_detail=event_detail
             )
             raise
@@ -88,25 +93,29 @@ def _handle_download(self, event, logger):
 
         logger.info(
             "Retrieved MESH message",
-            sender=getattr(message, 'sender', ''),
             local_id=getattr(message, 'local_id', ''),
             workflow_id=getattr(message, 'workflow_id', ''),
             subject=getattr(message, 'subject', ''),
             message_type=getattr(message, 'message_type', '')
         )
 
         content = message.read()
-        logger.info("Downloaded MESH message content")
+        logger.info(
+            "Downloaded MESH message content",
+            local_id=getattr(message, 'local_id', ''),
+            workflow_id=getattr(message, 'workflow_id', ''),
+            subject=getattr(message, 'subject', ''),
+            message_type=getattr(message, 'message_type', '')
+        )
 
         try:
             self._validate_fhir_content(content)
         except Exception as e:
             logger.error(
                 "FHIR content is invalid",
-                error=str(e),
-                mesh_message_id=data.meshMessageId
+                error=str(e)
             )
-            self._publish_message_invalid_event(incoming_event=event, failure_code='DL_CLIV_005')
+            self._publish_message_invalid_event(incoming_event=event, failure_code='DL_CLIV_005', logger=logger)
             message.acknowledge()
             logger.info("Acknowledged message")
             return
@@ -121,29 +130,26 @@ def _handle_download(self, event, logger):
             )
         except DocumentAlreadyExistsInternalRetryError:
             logger.warning(
-                "Internal retry detected. Message already stored with same meshMessageId, skipping",
-                mesh_message_id=data.meshMessageId,
-                message_reference=data.messageReference
+                "Internal retry detected. Message already stored with same meshMessageId, skipping"
             )
             self.__internal_duplicate_download_metric.record(1)
             message.acknowledge()
             logger.info("Acknowledged message")
             return 'skipped'
         except DocumentAlreadyExistsError:
             logger.warning(
-                "Trust duplicate detected. Same senderId + messageReference but different meshMessageId",
-                mesh_message_id=data.meshMessageId,
-                message_reference=data.messageReference
+                "Trust duplicate detected. Same senderId + messageReference but different meshMessageId"
             )
             self.__trust_duplicate_download_metric.record(1)
-            self._publish_message_invalid_event(incoming_event=event, failure_code='DL_CLIV_004')
+            self._publish_message_invalid_event(incoming_event=event, failure_code='DL_CLIV_004', logger=logger)
             message.acknowledge()
             logger.info("Acknowledged message")
             return 'duplicate'
 
         self._publish_downloaded_event(
             incoming_event=event,
-            message_uri=uri
+            message_uri=uri,
+            logger=logger
         )
         self.__download_metric.record(1)
         message.acknowledge()
@@ -165,7 +171,7 @@ def _store_message_content(self, sender_id, message_reference, mesh_message_id,
 
         return message_uri
 
-    def _publish_downloaded_event(self, incoming_event, message_uri):
+    def _publish_downloaded_event(self, incoming_event, message_uri, logger):
         """
         Publishes a MESHInboxMessageDownloaded event.
         """
@@ -194,17 +200,15 @@ def _publish_downloaded_event(self, incoming_event, message_uri):
         failed = self.__event_publisher.send_events([cloud_event], MESHInboxMessageDownloaded)
         if failed:
             msg = f"Failed to publish MESHInboxMessageDownloaded event: {failed}"
-            self.__log.error(msg, failed_count=len(failed))
+            logger.error(msg, failed_count=len(failed))
             raise RuntimeError(msg)
 
-        self.__log.info(
+        logger.info(
             "Published MESHInboxMessageDownloaded event",
-            sender_id=incoming_event.data.senderId,
             message_uri=message_uri,
-            message_reference=incoming_event.data.messageReference
         )
 
-    def _publish_message_invalid_event(self, incoming_event, failure_code: str):
+    def _publish_message_invalid_event(self, incoming_event, failure_code: str, logger):
         """
         Publishes a MESHInboxMessageInvalid event.
         """
@@ -231,11 +235,9 @@ def _publish_message_invalid_event(self, incoming_event, failure_code: str):
         failed = self.__event_publisher.send_events([cloud_event], MESHInboxMessageInvalid)
         if failed:
             msg = f"Failed to publish MESHInboxMessageInvalid event: {failed}"
-            self.__log.error(msg, failed_count=len(failed))
+            logger.error(msg, failed_count=len(failed))
             raise RuntimeError(msg)
 
-        self.__log.info(
-            "Published MESHInboxMessageInvalid event",
-            sender_id=incoming_event.data.senderId,
-            message_reference=incoming_event.data.messageReference
+        logger.info(
+            "Published MESHInboxMessageInvalid event"
         )

lambdas/mesh-poll/mesh_poll/processor.py

@@ -197,7 +197,8 @@ def _publish_mesh_inbox_message_received_event(self, event_detail):
 
         self.__log.info("Published MESHInboxMessageReceived event",
                         mesh_message_id=event_detail["data"]["meshMessageId"],
-                        sender_id=event_detail["data"]["senderId"])
+                        sender_id=event_detail["data"]["senderId"],
+                        message_reference=event_detail["data"]["messageReference"])
 
     def _publish_mesh_inbox_message_invalid_event(self, event_detail):
         """

lambdas/move-scanned-files-lambda/src/apis/sqs-handler.ts

@@ -55,6 +55,10 @@ export const createHandler = ({
             }
           } else {
             // there was something wrong with the event
+            logger.warn({
+              description: 'Failed to process event',
+              messageId: sqsRecord.messageId,
+            });
             batchItemFailures.push({ itemIdentifier: sqsRecord.messageId });
           }
         } catch (error: any) {

lambdas/pdm-poll-lambda/src/__tests__/apis/sqs-handler.test.ts

@@ -254,6 +254,10 @@ describe('SQS Handler', () => {
       expect(logger.warn).toHaveBeenCalledWith({
         err: 'PDM error',
         description: 'Failed processing message',
+        messageId: '1',
+        messageReference: pdmResourceSubmittedEvent.data.messageReference,
+        senderId: pdmResourceSubmittedEvent.data.senderId,
+        resourceId: pdmResourceSubmittedEvent.data.resourceId,
       });
 
       expect(logger.info).toHaveBeenCalledWith(
@@ -276,6 +280,7 @@ describe('SQS Handler', () => {
           `Unexpected token 'o', "not-json" is not valid JSON`,
         ),
         description: 'Error parsing SQS record',
+        messageId: '1',
       });
 
       expect(logger.info).toHaveBeenCalledWith(

lambdas/pdm-poll-lambda/src/apis/sqs-handler.ts

@@ -55,6 +55,7 @@ function validateRecord(
     logger.warn({
       err: error,
       description: 'Error parsing SQS record',
+      messageId,
     });
     return null;
   }
@@ -184,6 +185,10 @@ export const createHandler = ({
           logger.warn({
             err: error.message,
             description: 'Failed processing message',
+            messageId: validatedRecord.messageId,
+            messageReference: validatedRecord.event.data.messageReference,
+            senderId: validatedRecord.event.data.senderId,
+            resourceId: validatedRecord.event.data.resourceId,
           });
           batchItemFailures.push({ itemIdentifier: validatedRecord.messageId });
         }

lambdas/pdm-poll-lambda/src/app/pdm.ts

@@ -32,7 +32,13 @@ export class Pdm {
 
   async poll(item: any): Promise<PdmPollResult> {
     try {
-      this.logger.info(item);
+      this.logger.info({
+        description: 'Polling PDM for document resource',
+        messageId: item.messageId,
+        senderId: item.data.senderId,
+        messageReference: item.data.messageReference,
+        resourceId: item.data.resourceId,
+      });
 
       const response = await this.pdmClient.getDocumentReference(
         item.data.resourceId,
@@ -59,6 +65,10 @@ export class Pdm {
     } catch (error) {
       this.logger.error({
         description: 'Error getting document resource from PDM',
+        messageId: item.messageId,
+        senderId: item.data.senderId,
+        messageReference: item.data.messageReference,
+        resourceId: item.data.resourceId,
         err: error,
       });
 

lambdas/pdm-uploader-lambda/src/__tests__/app/upload-to-pdm.test.ts

@@ -96,6 +96,7 @@ describe('UploadToPdm', () => {
           name: error.name,
         }),
         messageReference: mockEvent.data.messageReference,
+        senderId: mockEvent.data.senderId,
       });
       expect(mockPdmClient.createDocumentReference).not.toHaveBeenCalled();
     });
@@ -115,6 +116,7 @@ describe('UploadToPdm', () => {
           message: error.message,
         }),
         messageReference: mockEvent.data.messageReference,
+        senderId: mockEvent.data.senderId,
       });
       expect(mockPdmClient.createDocumentReference).not.toHaveBeenCalled();
     });
@@ -134,6 +136,7 @@ describe('UploadToPdm', () => {
           name: error.name,
         }),
         messageReference: mockEvent.data.messageReference,
+        senderId: mockEvent.data.senderId,
       });
     });
   });

lambdas/pdm-uploader-lambda/src/app/upload-to-pdm.ts

@@ -28,6 +28,7 @@ export class UploadToPdm {
         description: 'Successfully sent request to PDM',
         eventId: event.id,
         messageReference,
+        senderId: event.data.senderId,
         resourceId: response.id,
       });
 
@@ -36,6 +37,7 @@ export class UploadToPdm {
       this.logger.error({
         description: 'Error sending request to PDM',
         messageReference: event.data.messageReference,
+        senderId: event.data.senderId,
         err:
           error instanceof Error
             ? { message: error.message, name: error.name, stack: error.stack }

lambdas/print-analyser/src/__tests__/apis/sqs-handler.test.ts

@@ -125,6 +125,8 @@ describe('SQS Handler', () => {
 
       expect(logger.child).toHaveBeenCalledWith({
         messageReference: fileSafeEvent.data.messageReference,
+        senderId: fileSafeEvent.data.senderId,
+        messageId: '1',
       });
       expect(mockChildLogger.error).toHaveBeenCalledWith({
         err: expect.arrayContaining([

lambdas/print-analyser/src/apis/sqs-handler.ts

@@ -40,7 +40,8 @@ function validateRecord(
 
     const messageReference =
       sqsEventDetail?.data?.messageReference || 'not present';
-    const childLogger = logger.child({ messageReference });
+    const senderId = sqsEventDetail?.data?.senderId || 'not present';
+    const childLogger = logger.child({ messageReference, senderId, messageId });
 
     validateFileSafe(sqsEventDetail, childLogger);
 

utils/py-utils/dl_utils/log_config.py

@@ -1,4 +1,4 @@
 import structlog
 
-structlog.configure(processors=[structlog.processors.JSONRenderer()])
+structlog.configure(processors=[structlog.stdlib.add_log_level, structlog.processors.JSONRenderer()])
 log = structlog.get_logger()