From e968b41ae07f904288e051dbd5245265505912e2 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 20 Nov 2025 16:54:07 -0600 Subject: [PATCH 001/137] Add VPC stack with tests --- .../compact-connect/pipeline/backend_stage.py | 11 + .../stacks/vpc_stack/__init__.py | 117 +++++++++++ backend/compact-connect/tests/app/test_vpc.py | 189 ++++++++++++++++++ 3 files changed, 317 insertions(+) create mode 100644 backend/compact-connect/stacks/vpc_stack/__init__.py create mode 100644 backend/compact-connect/tests/app/test_vpc.py diff --git a/backend/compact-connect/pipeline/backend_stage.py b/backend/compact-connect/pipeline/backend_stage.py index 6a83de8a9..243590be8 100644 --- a/backend/compact-connect/pipeline/backend_stage.py +++ b/backend/compact-connect/pipeline/backend_stage.py @@ -18,6 +18,7 @@ from stacks.state_api_stack import StateApiStack from stacks.state_auth import StateAuthStack from stacks.transaction_monitoring_stack import TransactionMonitoringStack +from stacks.vpc_stack import VpcStack class BackendStage(Stage): @@ -38,6 +39,16 @@ def __init__( environment = Environment(account=environment_context['account_id'], region=environment_context['region']) + # VPC Stack - provides networking infrastructure for OpenSearch and Lambda functions + self.vpc_stack = VpcStack( + self, + 'VpcStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + ) + self.persistent_stack = PersistentStack( self, 'PersistentStack', diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py new file mode 100644 index 000000000..8170b56c3 --- /dev/null +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -0,0 +1,117 @@ +from aws_cdk import RemovalPolicy +from aws_cdk.aws_ec2 import ( + FlowLogDestination, + FlowLogTrafficType, + GatewayVpcEndpointAwsService, + InterfaceVpcEndpointAwsService, + IpAddresses, + Port, + SecurityGroup, + SubnetConfiguration, + SubnetType, + Vpc, +) +from aws_cdk.aws_logs import LogGroup, RetentionDays +from common_constructs.stack import AppStack +from constructs import Construct + + +class VpcStack(AppStack): + """ + Stack for VPC resources needed for OpenSearch Domain and Lambda functions. + + This stack provides network infrastructure including: + - VPC with private subnets across multiple availability zones + - VPC endpoints for AWS services (CloudWatch Logs, S3, etc.) + - Security groups for OpenSearch and Lambda functions + - VPC Flow Logs for network monitoring + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + *, + environment_name: str, + environment_context: dict, + **kwargs, + ): + super().__init__( + scope, construct_id, environment_context=environment_context, environment_name=environment_name, **kwargs + ) + + # Determine removal policy based on environment + removal_policy = RemovalPolicy.RETAIN if environment_name == 'prod' else RemovalPolicy.DESTROY + + # Create VPC with private subnets across multiple availability zones + self.vpc = Vpc( + self, + 'CompactConnectVpc', + # No Internet or NAT Gateway needed - using VPC endpoints for AWS service access + create_internet_gateway=False, + nat_gateways=0, + ip_addresses=IpAddresses.cidr('10.0.0.0/16'), + max_azs=3, # Use up to 3 availability zones for high availability + subnet_configuration=[ + SubnetConfiguration( + name='private_subnet', + subnet_type=SubnetType.PRIVATE_WITH_EGRESS, + ), + ], + enable_dns_hostnames=True, + enable_dns_support=True, + ) + + # Create VPC Flow Logs for monitoring network traffic + flow_log_group = LogGroup( + self, + 'VpcFlowLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + ) + + self.vpc.add_flow_log( + 'VpcFlowLog', + destination=FlowLogDestination.to_cloud_watch_logs(flow_log_group), + traffic_type=FlowLogTrafficType.ALL, + ) + + # VPC Endpoint for CloudWatch Logs + # This allows Lambda functions in the VPC to send logs to CloudWatch without internet access + self.logs_vpc_endpoint = self.vpc.add_interface_endpoint( + 'LogsVpcEndpoint', + service=InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, + ) + + # VPC Endpoint for DynamoDB + # This allows Lambda functions to access DynamoDB without internet access + self.dynamodb_vpc_endpoint = self.vpc.add_gateway_endpoint( + 'DynamoDbVpcEndpoint', + service=GatewayVpcEndpointAwsService.DYNAMODB, + ) + + # Security Group for Lambda Functions + # This will control inbound and outbound traffic for Lambda functions that interact with OpenSearch + self.lambda_security_group = SecurityGroup( + self, + 'LambdaSecurityGroup', + vpc=self.vpc, + description='Security group for Lambda functions within VPC', + allow_all_outbound=True, # Allow Lambda to make outbound connections + ) + + # Security Group for OpenSearch Domain + # This will control inbound and outbound traffic for the OpenSearch cluster + self.opensearch_security_group = SecurityGroup( + self, + 'OpenSearchSecurityGroup', + vpc=self.vpc, + description='Security group for OpenSearch Domain', + allow_all_outbound=True, # Allow OpenSearch to make outbound connections + ) + # Allow Lambda functions to communicate with OpenSearch on port 443 (HTTPS) + self.opensearch_security_group.add_ingress_rule( + peer=self.lambda_security_group, + connection=Port.tcp(443), + description='Security group for OpenSearch Domain', + ) diff --git a/backend/compact-connect/tests/app/test_vpc.py b/backend/compact-connect/tests/app/test_vpc.py new file mode 100644 index 000000000..5dc2257b9 --- /dev/null +++ b/backend/compact-connect/tests/app/test_vpc.py @@ -0,0 +1,189 @@ +import json +from unittest import TestCase + +from aws_cdk.assertions import Match, Template + +from tests.app.base import TstAppABC + + +class TestVpcStack(TstAppABC, TestCase): + """ + Test cases for the VpcStack to ensure proper VPC configuration + for OpenSearch Domain and Lambda functions. + """ + + @classmethod + def get_context(cls): + with open('cdk.json') as f: + context = json.load(f)['context'] + with open('cdk.context.sandbox-example.json') as f: + context.update(json.load(f)) + + # Suppresses lambda bundling for tests + context['aws:cdk:bundling-stacks'] = [] + return context + + def test_vpc_configuration(self): + """ + Test that the VPC is created with the correct configuration for OpenSearch and Lambda functions. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify exactly one VPC is created + vpc_template.resource_count_is('AWS::EC2::VPC', 1) + + # Verify VPC has the correct configuration + vpc_template.has_resource_properties( + 'AWS::EC2::VPC', + { + 'CidrBlock': '10.0.0.0/16', + 'EnableDnsHostnames': True, + 'EnableDnsSupport': True, + }, + ) + + def test_subnets_configuration(self): + """ + Test that subnets are created across multiple availability zones. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify at least 3 subnets are created (one per AZ, max 3 AZs) + # The actual number depends on the region's available AZs + subnet_resources = vpc_template.find_resources('AWS::EC2::Subnet') + subnet_count = len(subnet_resources) + self.assertEqual(subnet_count, 3, 'The VPC should have 3 subnets for OpenSearch high availability') + + def test_no_internet_gateway(self): + """ + Test that no Internet Gateway is created, as we're using VPC endpoints for AWS service access. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify no Internet Gateway is created + vpc_template.resource_count_is('AWS::EC2::InternetGateway', 0) + + def test_no_nat_gateway(self): + """ + Test that no NAT Gateway is created, as we're using VPC endpoints for AWS service access. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify no NAT Gateway is created + vpc_template.resource_count_is('AWS::EC2::NatGateway', 0) + + def test_vpc_flow_logs(self): + """ + Test that VPC Flow Logs are configured to monitor network traffic. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify Flow Log is created + vpc_template.resource_count_is('AWS::EC2::FlowLog', 1) + + # Verify Flow Log is configured correctly + vpc_template.has_resource_properties( + 'AWS::EC2::FlowLog', + { + 'ResourceType': 'VPC', + 'TrafficType': 'ALL', + }, + ) + + # Verify CloudWatch Log Group for Flow Logs exists + vpc_template.resource_count_is('AWS::Logs::LogGroup', 1) + + def test_cloudwatch_logs_vpc_endpoint(self): + """ + Test that CloudWatch Logs VPC endpoint is created to allow Lambda functions to send logs. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify VPC endpoint for CloudWatch Logs is created + vpc_template.has_resource_properties( + 'AWS::EC2::VPCEndpoint', + { + 'ServiceName': Match.string_like_regexp('.*logs.*'), + 'VpcEndpointType': 'Interface', + }, + ) + + def test_dynamodb_vpc_endpoint(self): + """ + Test that DynamoDB VPC endpoint is created for Lambda functions to access DynamoDB. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify VPC gateway endpoint for DynamoDB is created + vpc_template.has_resource_properties( + 'AWS::EC2::VPCEndpoint', + { + 'VpcEndpointType': 'Gateway', + }, + ) + + def test_security_groups_created(self): + """ + Test that security groups are created for OpenSearch and Lambda functions. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Verify security groups are created (2 for our services + default VPC security group) + security_groups = vpc_template.find_resources('AWS::EC2::SecurityGroup') + + # Verify OpenSearch security group exists with correct description + opensearch_sg_logical_id = vpc_stack.get_logical_id(vpc_stack.opensearch_security_group.node.default_child) + opensearch_sg = TestVpcStack.get_resource_properties_by_logical_id(opensearch_sg_logical_id, security_groups) + self.assertEqual( + { + 'GroupDescription': 'Security group for OpenSearch Domain', + 'SecurityGroupEgress': [ + {'CidrIp': '0.0.0.0/0', 'Description': 'Allow all outbound traffic by default', 'IpProtocol': '-1'} + ], + 'VpcId': {'Ref': 'CompactConnectVpcF5956695'}, + }, + opensearch_sg, + ) + + # Verify Lambda security group exists with correct description + lambda_sg_logical_id = vpc_stack.get_logical_id(vpc_stack.lambda_security_group.node.default_child) + lambda_sg = TestVpcStack.get_resource_properties_by_logical_id(lambda_sg_logical_id, security_groups) + self.assertEqual( + { + 'GroupDescription': 'Security group for Lambda functions within VPC', + 'SecurityGroupEgress': [ + {'CidrIp': '0.0.0.0/0', 'Description': 'Allow all outbound traffic by default', 'IpProtocol': '-1'} + ], + 'VpcId': {'Ref': 'CompactConnectVpcF5956695'}, + }, + lambda_sg, + ) + + def test_opensearch_ingress_rule(self): + """ + Test that the OpenSearch security group allows ingress from Lambda security group on port 443. + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Get the logical IDs for both security groups + lambda_sg_logical_id = vpc_stack.get_logical_id(vpc_stack.lambda_security_group.node.default_child) + + # Verify ingress rule exists allowing Lambda to access OpenSearch on port 443 + vpc_template.has_resource_properties( + 'AWS::EC2::SecurityGroupIngress', + { + 'IpProtocol': 'tcp', + 'FromPort': 443, + 'ToPort': 443, + 'SourceSecurityGroupId': {'Fn::GetAtt': [lambda_sg_logical_id, 'GroupId']}, + }, + ) From c7a6fd02e46aceb8fce8f4f3b82d995ab3a32d5c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 21 Nov 2025 11:30:00 -0600 Subject: [PATCH 002/137] PR feedback --- backend/compact-connect/stacks/vpc_stack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index 8170b56c3..7d6ede0fa 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -55,7 +55,7 @@ def __init__( subnet_configuration=[ SubnetConfiguration( name='private_subnet', - subnet_type=SubnetType.PRIVATE_WITH_EGRESS, + subnet_type=SubnetType.PRIVATE_ISOLATED, ), ], enable_dns_hostnames=True, @@ -113,5 +113,5 @@ def __init__( self.opensearch_security_group.add_ingress_rule( peer=self.lambda_security_group, connection=Port.tcp(443), - description='Security group for OpenSearch Domain', + description='Allow HTTPS traffic from Lambda functions', ) From 100b1257ad24c266642922bdb6825695b0cf1779 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 21 Nov 2025 12:21:08 -0600 Subject: [PATCH 003/137] Add encryption to VPC flow logs --- .../stacks/vpc_stack/__init__.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index 7d6ede0fa..d47df7b21 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -11,7 +11,10 @@ SubnetType, Vpc, ) +from aws_cdk.aws_iam import ServicePrincipal +from aws_cdk.aws_kms import Key from aws_cdk.aws_logs import LogGroup, RetentionDays +from cdk_nag import NagSuppressions from common_constructs.stack import AppStack from constructs import Construct @@ -43,6 +46,14 @@ def __init__( # Determine removal policy based on environment removal_policy = RemovalPolicy.RETAIN if environment_name == 'prod' else RemovalPolicy.DESTROY + self.vpc_encryption_key = Key( + self, + 'VpcEncryptionKey', + enable_key_rotation=True, + alias=f'{self.stack_name}-vpc-encryption-key', + removal_policy=removal_policy, + ) + # Create VPC with private subnets across multiple availability zones self.vpc = Vpc( self, @@ -62,12 +73,17 @@ def __init__( enable_dns_support=True, ) + # grant access to Cloudwatch logs for vpc encryption key + logs_principal = ServicePrincipal('logs.amazonaws.com') + self.vpc_encryption_key.grant_encrypt_decrypt(logs_principal) + # Create VPC Flow Logs for monitoring network traffic flow_log_group = LogGroup( self, 'VpcFlowLogGroup', retention=RetentionDays.ONE_MONTH, removal_policy=removal_policy, + encryption_key=self.vpc_encryption_key ) self.vpc.add_flow_log( @@ -83,6 +99,32 @@ def __init__( service=InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, ) + # Suppress CdkNag warnings for the auto-generated VPC endpoint security group + # These warnings occur because CDK creates security group rules with intrinsic functions + # that CdkNag cannot fully evaluate at synthesis time + NagSuppressions.add_resource_suppressions_by_path( + self, + path=self.logs_vpc_endpoint.node.path, + suppressions=[ + { + 'id': 'AwsSolutions-EC23', + 'reason': 'VPC endpoint security groups are automatically managed by CDK. Inbound rules are ' + 'appropriately restricted to HTTPS (port 443) from VPC CIDR block.', + }, + { + 'id': 'HIPAA.Security-EC2RestrictedCommonPorts', + 'reason': 'VPC endpoint security groups are automatically managed by CDK. Only HTTPS (port 443) ' + 'is allowed for CloudWatch Logs communication.', + }, + { + 'id': 'HIPAA.Security-EC2RestrictedSSH', + 'reason': 'VPC endpoint security groups are automatically managed by CDK. SSH is not enabled on ' + 'this security group.', + }, + ], + apply_to_children=True, + ) + # VPC Endpoint for DynamoDB # This allows Lambda functions to access DynamoDB without internet access self.dynamodb_vpc_endpoint = self.vpc.add_gateway_endpoint( From 3d4054a15a0bd4848e25db49ff5bb8173b65aacf Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 21 Nov 2025 12:27:02 -0600 Subject: [PATCH 004/137] PR feedback - fix docs --- backend/compact-connect/stacks/vpc_stack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index d47df7b21..43dc4061b 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -25,7 +25,7 @@ class VpcStack(AppStack): This stack provides network infrastructure including: - VPC with private subnets across multiple availability zones - - VPC endpoints for AWS services (CloudWatch Logs, S3, etc.) + - VPC endpoints for AWS services (CloudWatch Logs, DynamoDB) - Security groups for OpenSearch and Lambda functions - VPC Flow Logs for network monitoring """ From f5134309795c04436556314759e707f474dde658 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 21 Nov 2025 17:07:20 -0600 Subject: [PATCH 005/137] Add search persistent stack with OpenSearch Domain --- .../common_constructs/constants.py | 2 + .../compact-connect/pipeline/backend_stage.py | 15 + .../search_persistent_stack/__init__.py | 261 ++++++++++++++++++ .../tests/app/test_search_persistent_stack.py | 167 +++++++++++ 4 files changed, 445 insertions(+) create mode 100644 backend/compact-connect/common_constructs/constants.py create mode 100644 backend/compact-connect/stacks/search_persistent_stack/__init__.py create mode 100644 backend/compact-connect/tests/app/test_search_persistent_stack.py diff --git a/backend/compact-connect/common_constructs/constants.py b/backend/compact-connect/common_constructs/constants.py new file mode 100644 index 000000000..26201949e --- /dev/null +++ b/backend/compact-connect/common_constructs/constants.py @@ -0,0 +1,2 @@ +PROD_ENV_NAME = 'prod' +BETA_ENV_NAME = 'beta' diff --git a/backend/compact-connect/pipeline/backend_stage.py b/backend/compact-connect/pipeline/backend_stage.py index 243590be8..fe46abfb8 100644 --- a/backend/compact-connect/pipeline/backend_stage.py +++ b/backend/compact-connect/pipeline/backend_stage.py @@ -15,6 +15,7 @@ from stacks.persistent_stack import PersistentStack from stacks.provider_users import ProviderUsersStack from stacks.reporting_stack import ReportingStack +from stacks.search_persistent_stack import SearchPersistentStack from stacks.state_api_stack import StateApiStack from stacks.state_auth import StateAuthStack from stacks.transaction_monitoring_stack import TransactionMonitoringStack @@ -49,6 +50,20 @@ def __init__( environment_name=environment_name, ) + # Search Persistent Stack - OpenSearch Domain for advanced provider search + # currently not deploying to prod or beta to reduce costs until search api functionality is completed + # to reduce costs + if environment_name != 'prod' and environment_name != 'beta': + self.search_persistent_stack = SearchPersistentStack( + self, + 'SearchPersistentStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + vpc_stack=self.vpc_stack, + ) + self.persistent_stack = PersistentStack( self, 'PersistentStack', diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py new file mode 100644 index 000000000..238c457ee --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -0,0 +1,261 @@ +from aws_cdk import RemovalPolicy +from aws_cdk.aws_ec2 import SubnetSelection, SubnetType +from aws_cdk.aws_iam import Effect, PolicyStatement, ServicePrincipal +from aws_cdk.aws_kms import Key +from aws_cdk.aws_logs import LogGroup, ResourcePolicy, RetentionDays +from aws_cdk.aws_opensearchservice import ( + CapacityConfig, + Domain, + EbsOptions, + EncryptionAtRestOptions, + EngineVersion, + LoggingOptions, + TLSSecurityPolicy, + ZoneAwarenessConfig, +) +from cdk_nag import NagSuppressions +from common_constructs.stack import AppStack +from constructs import Construct + +from common_constructs.constants import PROD_ENV_NAME +from stacks.vpc_stack import VpcStack + + +class SearchPersistentStack(AppStack): + """ + Stack for OpenSearch Domain and related search infrastructure. + + This stack provides the search capabilities for the advanced provider search feature: + - OpenSearch Domain deployed in VPC for network isolation + - KMS encryption for data at rest + - Node-to-node encryption and HTTPS enforcement + - Environment-specific instance sizing and cluster configuration + + Instance sizing by environment: + - Non-prod (sandbox/test/beta): t3.small.search, 1 node + - Prod: m7g.medium.search, 3 master + 3 data nodes (with standby) + + Note: Prod deployment is currently conditional and will not deploy until the full + advanced search API is implemented. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + *, + environment_name: str, + environment_context: dict, + vpc_stack: VpcStack, + **kwargs, + ): + super().__init__( + scope, construct_id, environment_context=environment_context, environment_name=environment_name, **kwargs + ) + + # Determine removal policy based on environment + removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY + + # Create dedicated KMS key for OpenSearch domain encryption + self.opensearch_encryption_key = Key( + self, + 'OpenSearchEncryptionKey', + enable_key_rotation=True, + alias=f'{self.stack_name}-opensearch-encryption-key', + removal_policy=removal_policy, + ) + + # Grant OpenSearch service principal permission to use the key + opensearch_principal = ServicePrincipal('es.amazonaws.com') + self.opensearch_encryption_key.grant_encrypt_decrypt(opensearch_principal) + + # Grant cloudwatch service principal permission to use the key + log_principal = ServicePrincipal('logs.amazonaws.com') + self.opensearch_encryption_key.grant_encrypt_decrypt(log_principal) + + # Determine instance type and capacity based on environment + capacity_config = self._get_capacity_config(environment_name) + # determine AZ awareness based on environment + zone_awareness_config = self._get_zone_awareness_config(environment_name) + + opensearch_app_log_group = LogGroup( + self, + 'OpensearchAppLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + encryption_key=self.opensearch_encryption_key, + ) + slow_search_log_group = LogGroup( + self, + 'SlowSearchLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + encryption_key=self.opensearch_encryption_key, + ) + slow_index_log_group = LogGroup( + self, + 'SlowIndexLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + encryption_key=self.opensearch_encryption_key, + ) + + # Create CloudWatch Logs resource policy to allow OpenSearch to write logs + # This is done manually to avoid CDK creating an auto-generated Lambda function + ResourcePolicy( + self, + 'OpenSearchLogsResourcePolicy', + policy_statements=[ + PolicyStatement( + effect=Effect.ALLOW, + principals=[ServicePrincipal('es.amazonaws.com')], + actions=[ + 'logs:PutLogEvents', + 'logs:CreateLogStream', + ], + resources=[ + opensearch_app_log_group.log_group_arn, + slow_search_log_group.log_group_arn, + slow_index_log_group.log_group_arn, + ], + ), + ], + ) + + # Create OpenSearch Domain + self.domain = Domain( + self, + 'ProviderSearchDomain', + # TODO - set this to OPENSEARCH_3_1 after runtime migration PR is merged + version=EngineVersion.OPENSEARCH_2_19, + capacity=capacity_config, + # VPC configuration for network isolation + vpc=vpc_stack.vpc, + vpc_subnets=[SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED)], + security_groups=[vpc_stack.opensearch_security_group], + # EBS volume configuration + ebs=EbsOptions(enabled=True, volume_size=20 if environment_name == 'prod' else 10), + # Encryption settings + encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.opensearch_encryption_key), + node_to_node_encryption=True, + enforce_https=True, + tls_security_policy=TLSSecurityPolicy.TLS_1_2, + logging=LoggingOptions( + app_log_enabled=True, + app_log_group=opensearch_app_log_group, + slow_search_log_enabled=True, + slow_search_log_group=slow_search_log_group, + slow_index_log_enabled=True, + slow_index_log_group=slow_index_log_group, + ), + # Suppress auto-generated Lambda for log resource policy (we created it manually above) + suppress_logs_resource_policy=True, + # Domain removal policy + removal_policy=removal_policy, + zone_awareness=zone_awareness_config, + ) + + # Add CDK Nag suppressions for OpenSearch Domain + self._add_opensearch_suppressions(environment_name) + + def _get_capacity_config(self, environment_name: str) -> CapacityConfig: + """ + Determine OpenSearch cluster capacity configuration based on environment. + + Non-prod (sandbox, test, beta, etc.): Single t3.small.search node + Prod: 3 dedicated master (m7g.medium.search) + 3 data nodes (m7g.medium.search) with standby + + param environment_name: The deployment environment name + + return: CapacityConfig with appropriate instance types and counts + """ + if environment_name == PROD_ENV_NAME: + # Production configuration with high availability + # 3 dedicated master nodes + 3 data nodes across 3 AZs with standby + # Multi-AZ with standby does not support t3 instance types + return CapacityConfig( + # Data nodes - m7g.medium provides 4 vCPUs and 8GB RAM + data_node_instance_type='m7g.medium.search', + data_nodes=3, + # Dedicated master nodes for cluster management + master_node_instance_type='m7g.medium.search', + master_nodes=3, + # Multi-AZ with standby for high availability + multi_az_with_standby_enabled=True, + ) + + # Single node configuration for all non-prod environments + # (test, beta, and developer sandboxes) + return CapacityConfig( + data_node_instance_type='t3.small.search', + data_nodes=1, + # No dedicated master nodes for single-node clusters + master_nodes=None, + # No multi-AZ for single node + multi_az_with_standby_enabled=False, + ) + + def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConfig: + """ + Determine OpenSearch cluster availability zone awareness based on environment. + + 3 for production, not enabled for all other non-prod environments + + param environment_name: The deployment environment name + + return: ZoneAwarenessConfig with appropriate settings + """ + if environment_name == PROD_ENV_NAME: + return ZoneAwarenessConfig(enabled=True, availability_zone_count=3) + + # non-prod environments only use one data node, hence we don't enable zone awareness + return ZoneAwarenessConfig(enabled=False) + + def _add_opensearch_suppressions(self, environment_name: str): + """ + Add CDK Nag suppressions for OpenSearch Domain configuration. + + Some security best practices are not applicable or will be implemented later: + - Fine-grained access control: Will be added with full API implementation + - Access policies: Will be configured when Lambda functions are added + - Dedicated master nodes: Only needed for prod (>3 nodes) + """ + NagSuppressions.add_resource_suppressions( + self.domain, + suppressions=[ + { + 'id': 'AwsSolutions-OS3', + 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups.' + 'The data in the domain is only accessible by the ingest lambda which indexes the' + 'documents and the search API lambda which can only be accessed by authenticated staff' + 'users in CompactConnect.', + }, + { + 'id': 'AwsSolutions-OS5', + 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups.' + 'The data in the domain is only accessible by the ingest lambda which indexes the' + 'documents and the search API lambda which can only be accessed by authenticated staff' + 'users in CompactConnect.', + }, + ], + apply_to_children=True, + ) + if environment_name != PROD_ENV_NAME: + NagSuppressions.add_resource_suppressions( + self.domain, + suppressions=[ + { + 'id': 'AwsSolutions-OS4', + 'reason': 'Dedicated master nodes are only used in production environments with multiple data ' + 'nodes. Single-node non-prod environments do not require dedicated master nodes.', + }, + { + 'id': 'AwsSolutions-OS7', + 'reason': 'Zone awareness with standby is only enabled for production environments with ' + 'multiple nodes. Single-node test environments do not require multi-AZ ' + 'configuration.', + }, + ], + apply_to_children=True, + ) + diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py new file mode 100644 index 000000000..b21584ba6 --- /dev/null +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -0,0 +1,167 @@ +import json +from unittest import TestCase + +from aws_cdk.assertions import Match, Template + +from tests.app.base import TstAppABC + + +class TestSearchPersistentStack(TstAppABC, TestCase): + """ + Test cases for the SearchPersistentStack to ensure proper OpenSearch Domain configuration + for advanced provider search functionality. + """ + + @classmethod + def get_context(cls): + with open('cdk.json') as f: + context = json.load(f)['context'] + with open('cdk.context.sandbox-example.json') as f: + context.update(json.load(f)) + + # Suppresses lambda bundling for tests + context['aws:cdk:bundling-stacks'] = [] + return context + + def test_opensearch_domain_created(self): + """ + Test that the OpenSearch Domain is created with the correct basic configuration. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify exactly one OpenSearch Domain is created + search_template.resource_count_is('AWS::OpenSearchService::Domain', 1) + + def test_opensearch_version(self): + """ + Test that OpenSearch uses the correct version. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify OpenSearch version + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'EngineVersion': 'OpenSearch_2.19', + }, + ) + + def test_vpc_configuration(self): + """ + Test that the OpenSearch Domain is deployed within the VPC for network isolation. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify VPC configuration is present + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'VPCOptions': { + 'SubnetIds': Match.any_value(), + 'SecurityGroupIds': Match.any_value(), + }, + }, + ) + + def test_node_to_node_encryption(self): + """ + Test that node-to-node encryption is enabled. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify node-to-node encryption is enabled + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'NodeToNodeEncryptionOptions': { + 'Enabled': True, + }, + }, + ) + + def test_https_enforcement(self): + """ + Test that HTTPS is enforced for all traffic to the domain. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify HTTPS is required + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'DomainEndpointOptions': { + 'EnforceHTTPS': True, + 'TLSSecurityPolicy': 'Policy-Min-TLS-1-2-2019-07', + }, + }, + ) + + def test_ebs_encryption(self): + """ + Test that EBS volumes are encrypted. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + encryption_key_logical_id = search_stack.get_logical_id( + search_stack.opensearch_encryption_key.node.default_child + ) + + # Verify EBS volumes are encrypted + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'EBSOptions': { + 'EBSEnabled': True, + 'VolumeSize': 10, + }, + 'EncryptionAtRestOptions': { + 'Enabled': True, + 'KmsKeyId': {"Ref": encryption_key_logical_id, } + }, + }, + ) + + def test_sandbox_instance_type(self): + """ + Test that sandbox environment uses t3.small.search instance type for cost optimization. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify sandbox uses t3.small.search with single node + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'ClusterConfig': { + 'InstanceType': 't3.small.search', + 'InstanceCount': 1, + 'DedicatedMasterEnabled': False, + 'MultiAZWithStandbyEnabled': False, + }, + }, + ) + + def test_logging_configuration(self): + """ + Test that appropriate logging is enabled for monitoring and troubleshooting. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify logging configuration + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'LogPublishingOptions': { + 'ES_APPLICATION_LOGS': Match.object_like({'Enabled': True}), + }, + }, + ) + + From 90b57f89ce2cd6dbb447885aee4b7248a86314f5 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 24 Nov 2025 12:19:42 -0600 Subject: [PATCH 006/137] remove 'beta release' from the IT docs --- .../docs/it_staff_onboarding_instructions.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/docs/it_staff_onboarding_instructions.md b/backend/compact-connect/docs/it_staff_onboarding_instructions.md index 45fe9642d..184d99b40 100644 --- a/backend/compact-connect/docs/it_staff_onboarding_instructions.md +++ b/backend/compact-connect/docs/it_staff_onboarding_instructions.md @@ -1,4 +1,4 @@ -# CompactConnect Automated License Data Upload Instructions (Beta Release) +# CompactConnect Automated License Data Upload Instructions ## Overview @@ -61,8 +61,7 @@ Follow these steps to obtain an access token and make requests to the CompactCon ### Step 1: Generate an Access Token You must first obtain an access token to authenticate your API requests. The access token will be used in the -Authorization header of subsequent API calls. While the following curl command demonstrates how to generate a token for -the **beta** environment, you should implement this authentication flow in your application's programming language using +Authorization header of subsequent API calls. While the following curl command demonstrates how to generate a token, you should implement this authentication flow in your application's programming language using appropriate HTTPS request libraries: > **Note**: When copying commands, be careful of line breaks. You may need to remove any extra spaces or @@ -104,10 +103,10 @@ AWS documentation: https://docs.aws.amazon.com/cognito/latest/developerguide/tok - Your application should request a new token before the current one expires - Store the `access_token` value for use in API requests -### Step 2: Upload License Data to the Beta Environment (JSON POST Endpoint) +### Step 2: Upload License Data (JSON POST Endpoint) The CompactConnect License API can be called through a POST REST endpoint which takes in a list of license record -objects. The following curl command example demonstrates how to upload license data into the **beta** environment, but +objects. The following curl command example demonstrates how to upload license data, but you should implement this API call in your application's programming language using appropriate HTTPS request libraries. You will need to replace the example payload with valid license data that includes the correct license types for your specific compact. See the From 6097f5cb8fe005541ab48be15d458875407d9652 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 24 Nov 2025 16:22:04 -0600 Subject: [PATCH 007/137] Set explicit CIDR blocks --- .../stacks/vpc_stack/__init__.py | 61 ++++++++++++++++++- backend/compact-connect/tests/app/test_vpc.py | 48 +++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index 43dc4061b..868f6e87b 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -28,6 +28,23 @@ class VpcStack(AppStack): - VPC endpoints for AWS services (CloudWatch Logs, DynamoDB) - Security groups for OpenSearch and Lambda functions - VPC Flow Logs for network monitoring + + IMPORTANT - VPC Subnet CIDR Allocation Strategy: + ================================================= + This VPC uses explicit CIDR block overrides to prevent conflicts when expanding. + Each subnet CIDR is locked in using CloudFormation property overrides, which + allows safe addition of more AZs/subnets in the future without deployment failures. + + Current allocation from 10.0.0.0/16 VPC CIDR: + - Private subnets (3 AZs): 10.0.0.0/20, 10.0.16.0/20, 10.0.32.0/20 (4096 IPs each) + - Reserved for future expansion: 10.0.48.0/20, 10.0.64.0/20, etc. + + To add more subnets in the future: + 1. Increase max_azs (e.g., from 3 to 4) + 2. Add new CIDR blocks to the private_cidrs list (e.g., '10.0.48.0/20') + 3. Deploy - existing subnets won't be modified due to explicit CIDR overrides + + Solution reference: https://github.com/aws/aws-cdk/issues/24708#issuecomment-1665795316 """ def __init__( @@ -55,6 +72,7 @@ def __init__( ) # Create VPC with private subnets across multiple availability zones + # Using explicit CIDR allocation to allow future expansion without conflicts self.vpc = Vpc( self, 'CompactConnectVpc', @@ -62,17 +80,31 @@ def __init__( create_internet_gateway=False, nat_gateways=0, ip_addresses=IpAddresses.cidr('10.0.0.0/16'), - max_azs=3, # Use up to 3 availability zones for high availability + # Use 3 AZs for high availability + # CDK will automatically select 3 AZs from the region + max_azs=3, subnet_configuration=[ SubnetConfiguration( - name='private_subnet', + name='private', subnet_type=SubnetType.PRIVATE_ISOLATED, + # cidr_mask is set to 20 to provide /20 subnets (4096 IPs each) + # However, we explicitly override the CIDR blocks below to lock them in + cidr_mask=20, ), ], enable_dns_hostnames=True, enable_dns_support=True, ) + # Explicitly set CIDR blocks for each subnet to prevent conflicts when expanding VPC + # This follows the solution from: https://github.com/aws/aws-cdk/issues/24708#issuecomment-1665795316 + # By locking in the CIDR blocks, we can safely add more AZs or public subnets in the future without + # CloudFormation errors. + private_cidrs = ['10.0.0.0/20', '10.0.16.0/20', '10.0.32.0/20'] + self._assign_subnet_cidr('privateSubnet1', private_cidrs[0]) + self._assign_subnet_cidr('privateSubnet2', private_cidrs[1]) + self._assign_subnet_cidr('privateSubnet3', private_cidrs[2]) + # grant access to Cloudwatch logs for vpc encryption key logs_principal = ServicePrincipal('logs.amazonaws.com') self.vpc_encryption_key.grant_encrypt_decrypt(logs_principal) @@ -157,3 +189,28 @@ def __init__( connection=Port.tcp(443), description='Allow HTTPS traffic from Lambda functions', ) + + def _assign_subnet_cidr(self, subnet_name: str, cidr: str): + """ + Explicitly assign a CIDR block to a subnet by overriding the CloudFormation property. + + This prevents CIDR conflicts when adding more AZs to the VPC in the future. + Without this override, CloudFormation attempts to reassign CIDR blocks when subnets/AZs are added, + causing deployment failures with "CIDR conflict" errors. See https://github.com/aws/aws-cdk/issues/24708 + + param subnet_name: The logical name of the subnet (e.g., 'privateSubnet1') + param cidr: The CIDR block to assign (e.g., '10.0.0.0/20') + """ + + # Navigate the construct tree to find the subnet + subnet_construct = self.vpc.node.try_find_child(subnet_name) + if subnet_construct is None: + raise ValueError(f'Subnet {subnet_name} not found in VPC') + + # Get the underlying CloudFormation subnet resource + cfn_subnet = subnet_construct.node.try_find_child('Subnet') + if cfn_subnet is None: + raise ValueError(f'CloudFormation Subnet resource not found for {subnet_name}') + + # Override the CIDR block property + cfn_subnet.add_property_override('CidrBlock', cidr) diff --git a/backend/compact-connect/tests/app/test_vpc.py b/backend/compact-connect/tests/app/test_vpc.py index 5dc2257b9..17ec09c86 100644 --- a/backend/compact-connect/tests/app/test_vpc.py +++ b/backend/compact-connect/tests/app/test_vpc.py @@ -187,3 +187,51 @@ def test_opensearch_ingress_rule(self): 'SourceSecurityGroupId': {'Fn::GetAtt': [lambda_sg_logical_id, 'GroupId']}, }, ) + + def test_explicit_subnet_cidr_blocks(self): + """ + Test that subnet CIDR blocks are explicitly set to allow future VPC expansion. + + This verifies that each subnet has its CIDR block locked in via CloudFormation + property overrides. This prevents CIDR conflicts when adding more AZs in the future. + + CIDR allocation from 10.0.0.0/16 VPC: + - Subnet 1 (AZ 1): 10.0.0.0/20 (10.0.0.0 - 10.0.15.255, 4096 IPs) + - Subnet 2 (AZ 2): 10.0.16.0/20 (10.0.16.0 - 10.0.31.255, 4096 IPs) + - Subnet 3 (AZ 3): 10.0.32.0/20 (10.0.32.0 - 10.0.47.255, 4096 IPs) + - Reserved for future: 10.0.48.0/20 and beyond + + Reference: https://github.com/aws/aws-cdk/issues/24708#issuecomment-1665795316 + """ + vpc_stack = self.app.sandbox_backend_stage.vpc_stack + vpc_template = Template.from_stack(vpc_stack) + + # Get all subnet resources + subnet_resources = vpc_template.find_resources('AWS::EC2::Subnet') + + # Filter to only private subnets (those without MapPublicIpOnLaunch) + private_subnets = [] + for logical_id, subnet in subnet_resources.items(): + properties = subnet.get('Properties', {}) + # Private subnets don't have MapPublicIpOnLaunch or it's set to false + if not properties.get('MapPublicIpOnLaunch', False): + private_subnets.append((logical_id, properties)) + + # Verify we have exactly 3 private subnets + self.assertEqual( + 3, len(private_subnets), f'Expected exactly 3 private subnets, found {len(private_subnets)}' + ) + + # Expected CIDR blocks for the 3 private subnets + expected_cidr_blocks = ['10.0.0.0/20', '10.0.16.0/20', '10.0.32.0/20'] + + # Extract and sort the CIDR blocks from the subnets + actual_cidr_blocks = sorted([subnet[1]['CidrBlock'] for subnet in private_subnets]) + + # Verify the CIDR blocks match our expected explicit allocation + self.assertEqual( + expected_cidr_blocks, + actual_cidr_blocks, + 'Subnet CIDR blocks do not match expected explicit allocation. ' + 'This is critical for preventing conflicts when expanding the VPC.', + ) From d72fea49c232b5868f091a917937e528b0566ee2 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 24 Nov 2025 16:58:09 -0600 Subject: [PATCH 008/137] Set explicit subnet for non-prod OpenSearch domain --- .../search_persistent_stack/__init__.py | 55 ++++++++++++++++++- .../stacks/vpc_stack/__init__.py | 11 +++- .../tests/app/test_search_persistent_stack.py | 38 +++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 238c457ee..1efa0733f 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -14,11 +14,11 @@ ZoneAwarenessConfig, ) from cdk_nag import NagSuppressions +from common_constructs.constants import PROD_ENV_NAME from common_constructs.stack import AppStack from constructs import Construct -from common_constructs.constants import PROD_ENV_NAME -from stacks.vpc_stack import VpcStack +from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack class SearchPersistentStack(AppStack): @@ -77,6 +77,8 @@ def __init__( capacity_config = self._get_capacity_config(environment_name) # determine AZ awareness based on environment zone_awareness_config = self._get_zone_awareness_config(environment_name) + # Determine subnet selection based on environment + vpc_subnets = self._get_vpc_subnets(environment_name, vpc_stack) opensearch_app_log_group = LogGroup( self, @@ -131,7 +133,7 @@ def __init__( capacity=capacity_config, # VPC configuration for network isolation vpc=vpc_stack.vpc, - vpc_subnets=[SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED)], + vpc_subnets=vpc_subnets, security_groups=[vpc_stack.opensearch_security_group], # EBS volume configuration ebs=EbsOptions(enabled=True, volume_size=20 if environment_name == 'prod' else 10), @@ -211,6 +213,53 @@ def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConf # non-prod environments only use one data node, hence we don't enable zone awareness return ZoneAwarenessConfig(enabled=False) + def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> list[SubnetSelection]: + """ + Determine VPC subnet selection based on environment. + + Production: All private isolated subnets (3 AZs) for zone awareness and high availability + Non-prod: Single subnet (privateSubnet1 with CIDR 10.0.0.0/20) for single-node deployment + + param environment_name: The deployment environment name + param vpc_stack: The VPC stack containing the private subnets + + return: List of SubnetSelection with appropriate subnet configuration + """ + if environment_name == PROD_ENV_NAME: + # Production: Use all private isolated subnets from the VPC. + # VPC is configured with max_azs=3, so this will select exactly 3 subnets + return [SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED)] + + # Non-prod: Single-node deployment explicitly uses privateSubnet1 (CIDR 10.0.0.0/20) + # OpenSearch requires exactly one subnet for single-node deployments + # We explicitly find the subnet by its construct name to guarantee consistency + private_subnet1 = self._find_subnet_by_name(vpc_stack.vpc, PRIVATE_SUBNET_ONE_NAME) + return [SubnetSelection(subnets=[private_subnet1])] + + def _find_subnet_by_name(self, vpc, subnet_name: str): + """ + Find a specific subnet by its logical construct name in the VPC. + + This provides a guaranteed, explicit reference to a specific subnet regardless of + CDK's internal list ordering, which is critical for stateful resources like OpenSearch. + + param vpc: The VPC construct containing the subnet + param subnet_name: The logical name of the subnet (e.g., 'privateSubnet1') + + return: The ISubnet instance + + raises ValueError: If the subnet cannot be found + """ + # Navigate the construct tree to find the subnet by name + subnet_construct = vpc.node.try_find_child(subnet_name) + if subnet_construct is None: + raise ValueError( + f'Subnet {subnet_name} not found in VPC construct tree. ' + f'Available children: {[c.node.id for c in vpc.node.children]}' + ) + + return subnet_construct + def _add_opensearch_suppressions(self, environment_name: str): """ Add CDK Nag suppressions for OpenSearch Domain configuration. diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index 868f6e87b..cb7f7fdb2 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -19,6 +19,11 @@ from constructs import Construct +PRIVATE_SUBNET_ONE_NAME = 'privateSubnet1' +PRIVATE_SUBNET_TWO_NAME = 'privateSubnet2' +PRIVATE_SUBNET_THREE_NAME = 'privateSubnet3' + + class VpcStack(AppStack): """ Stack for VPC resources needed for OpenSearch Domain and Lambda functions. @@ -101,9 +106,9 @@ def __init__( # By locking in the CIDR blocks, we can safely add more AZs or public subnets in the future without # CloudFormation errors. private_cidrs = ['10.0.0.0/20', '10.0.16.0/20', '10.0.32.0/20'] - self._assign_subnet_cidr('privateSubnet1', private_cidrs[0]) - self._assign_subnet_cidr('privateSubnet2', private_cidrs[1]) - self._assign_subnet_cidr('privateSubnet3', private_cidrs[2]) + self._assign_subnet_cidr(PRIVATE_SUBNET_ONE_NAME, private_cidrs[0]) + self._assign_subnet_cidr(PRIVATE_SUBNET_TWO_NAME, private_cidrs[1]) + self._assign_subnet_cidr(PRIVATE_SUBNET_THREE_NAME, private_cidrs[2]) # grant access to Cloudwatch logs for vpc encryption key logs_principal = ServicePrincipal('logs.amazonaws.com') diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index b21584ba6..f5f4a47a9 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -164,4 +164,42 @@ def test_logging_configuration(self): }, ) + def test_sandbox_uses_expected_private_subnet(self): + """ + Test that the OpenSearch Domain in sandbox uses expected private Subnet. + + For non-prod single-node deployments, OpenSearch must use exactly one subnet. + We explicitly select privateSubnet1 (CIDR 10.0.0.0/20) to ensure deterministic + placement across deployments. + + This test verifies that OpenSearch references the specific subnet we expect, + not just any arbitrary subnet from the VPC. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Get the OpenSearch Domain's subnet configuration + opensearch_resources = search_template.find_resources('AWS::OpenSearchService::Domain') + opensearch_properties = list(opensearch_resources.values())[0]['Properties'] + vpc_options = opensearch_properties['VPCOptions'] + subnet_ids = vpc_options['SubnetIds'] + + # For sandbox (non-prod), should use exactly one subnet + self.assertEqual(len(subnet_ids), 1, 'Sandbox OpenSearch should use exactly one subnet') + + # Get the subnet reference from OpenSearch + opensearch_subnet_ref = subnet_ids[0] + # Extract the export name that OpenSearch is importing + import_value = opensearch_subnet_ref['Fn::ImportValue'] + + # Verify OpenSearch is importing the correct subnet (privateSubnet1) + # The import_value should reference the export name of privateSubnet1 + # The export name contains the construct name, which includes 'privateSubnet1' + self.assertIn( + 'privateSubnet1', + str(import_value), + f'OpenSearch should import privateSubnet1, but is importing: {import_value}. ' + 'This is critical for deterministic subnet placement in non-prod environments.' + ) + From fad5fb0da1ab37dd88180572dc9f22d54482e845 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 24 Nov 2025 21:44:51 -0600 Subject: [PATCH 009/137] Fix cloudwatch policy --- .../stacks/search_persistent_stack/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 1efa0733f..82369a119 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -104,6 +104,7 @@ def __init__( # Create CloudWatch Logs resource policy to allow OpenSearch to write logs # This is done manually to avoid CDK creating an auto-generated Lambda function + # The resource ARNs must include ':*' to grant permissions on log streams within the log groups ResourcePolicy( self, 'OpenSearchLogsResourcePolicy', @@ -116,9 +117,9 @@ def __init__( 'logs:CreateLogStream', ], resources=[ - opensearch_app_log_group.log_group_arn, - slow_search_log_group.log_group_arn, - slow_index_log_group.log_group_arn, + f'{opensearch_app_log_group.log_group_arn}:*', + f'{slow_search_log_group.log_group_arn}:*', + f'{slow_index_log_group.log_group_arn}:*', ], ), ], From 7c6fb81dbdba15685f2d7b6611b67c22a92232c2 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 24 Nov 2025 22:54:29 -0600 Subject: [PATCH 010/137] Add alarms for domain monitoring --- .../search_persistent_stack/__init__.py | 115 +++++++++++++++++- .../tests/app/test_search_persistent_stack.py | 51 ++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 82369a119..28c914aeb 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,4 +1,6 @@ -from aws_cdk import RemovalPolicy +from aws_cdk import Duration, RemovalPolicy +from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Metric, TreatMissingData +from aws_cdk.aws_cloudwatch_actions import SnsAction from aws_cdk.aws_ec2 import SubnetSelection, SubnetType from aws_cdk.aws_iam import Effect, PolicyStatement, ServicePrincipal from aws_cdk.aws_kms import Key @@ -14,6 +16,7 @@ ZoneAwarenessConfig, ) from cdk_nag import NagSuppressions +from common_constructs.alarm_topic import AlarmTopic from common_constructs.constants import PROD_ENV_NAME from common_constructs.stack import AppStack from constructs import Construct @@ -73,6 +76,25 @@ def __init__( log_principal = ServicePrincipal('logs.amazonaws.com') self.opensearch_encryption_key.grant_encrypt_decrypt(log_principal) + # Create dedicated KMS key for alarm topic encryption + search_alarm_encryption_key = Key( + self, + 'SearchAlarmEncryptionKey', + enable_key_rotation=True, + alias=f'{self.stack_name}-search-alarm-encryption-key', + removal_policy=removal_policy, + ) + + # Create alarm topic for OpenSearch capacity and health monitoring + notifications = environment_context.get('notifications', {}) + self.alarm_topic = AlarmTopic( + self, + 'SearchAlarmTopic', + master_key=search_alarm_encryption_key, + email_subscriptions=notifications.get('email', []), + slack_subscriptions=notifications.get('slack', []), + ) + # Determine instance type and capacity based on environment capacity_config = self._get_capacity_config(environment_name) # determine AZ awareness based on environment @@ -161,6 +183,9 @@ def __init__( # Add CDK Nag suppressions for OpenSearch Domain self._add_opensearch_suppressions(environment_name) + # Add capacity monitoring alarms for proactive scaling + self._add_capacity_alarms(environment_name) + def _get_capacity_config(self, environment_name: str) -> CapacityConfig: """ Determine OpenSearch cluster capacity configuration based on environment. @@ -261,6 +286,94 @@ def _find_subnet_by_name(self, vpc, subnet_name: str): return subnet_construct + def _add_capacity_alarms(self, environment_name: str): + """ + Add CloudWatch alarms to monitor OpenSearch capacity and alert before hitting limits. + + These proactive thresholds give the DevOps team time to plan scaling activities: + - Free Storage Space < 50% of allocated capacity + - JVM Memory Pressure > 60% + - CPU Utilization > 60% + + param environment_name: The deployment environment name + """ + # Get the volume size for calculating storage threshold + volume_size_gb = 20 if environment_name == PROD_ENV_NAME else 10 + # 50% threshold in MB (FreeStorageSpace metric is reported in megabytes) + # Formula: GB * 1024 MB/GB * 0.5 for 50% threshold + storage_threshold_mb = volume_size_gb * 1024 * 0.5 + + # Alarm: Free Storage Space < 50% + # This gives ample time to plan capacity increases before hitting critical levels + # Note: FreeStorageSpace metric is reported in megabytes (MB) + Alarm( + self, + 'FreeStorageSpaceAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='FreeStorageSpace', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, + # check every day to avoid alerting on spikes + period=Duration.days(1), + statistic='Minimum', + ), + evaluation_periods=1, # 1 day + threshold=storage_threshold_mb, + comparison_operator=ComparisonOperator.LESS_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} free storage space has dropped below 50% ' + f'({storage_threshold_mb}MB of {volume_size_gb * 1024}MB allocated EBS volume). ' + 'Consider planning to increase EBS volume size or scaling the cluster.' + ), + ).add_alarm_action(SnsAction(self.alarm_topic)) + + # Alarm: JVM Memory Pressure > 60% + # Sustained high memory pressure indicates need for instance scaling + Alarm( + self, + 'JVMMemoryPressureAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='JVMMemoryPressure', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, + period=Duration.minutes(5), + statistic='Maximum', + ), + evaluation_periods=3, # 15 minutes sustained + threshold=60, + comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 60%. ' + 'This indicates the cluster is using a significant portion of its heap memory. ' + 'Consider scaling to larger instance types if pressure continues to increase.' + ), + ).add_alarm_action(SnsAction(self.alarm_topic)) + + # Alarm: CPU Utilization > 60% + # Sustained high CPU indicates need for more compute capacity + Alarm( + self, + 'CPUUtilizationAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='CPUUtilization', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, + period=Duration.minutes(5), + statistic='Average', + ), + evaluation_periods=3, # 15 minutes sustained + threshold=60, + comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} CPU utilization has been above 60% for 15 minutes. ' + 'This indicates sustained high load. Consider scaling to larger instance types ' + 'or adding more data nodes to distribute the load.' + ), + ).add_alarm_action(SnsAction(self.alarm_topic)) + def _add_opensearch_suppressions(self, environment_name: str): """ Add CDK Nag suppressions for OpenSearch Domain configuration. diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index f5f4a47a9..9e5141d5c 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -164,6 +164,57 @@ def test_logging_configuration(self): }, ) + def test_capacity_alarms_configured(self): + """ + Test that capacity monitoring alarms are configured for proactive scaling. + + Verifies three critical alarms: + 1. Free Storage Space < 50% threshold + 2. JVM Memory Pressure > 60% threshold + 3. CPU Utilization > 60% threshold + + These alarms give DevOps team time to plan scaling activities before hitting limits. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify Free Storage Space Alarm + # Note: FreeStorageSpace is reported in megabytes (MB), not bytes + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'FreeStorageSpace', + 'Namespace': 'AWS/ES', + 'Threshold': 5120, # 5GB in MB (50% of 10GB = 5GB = 5120MB for sandbox) + 'ComparisonOperator': 'LessThanThreshold', + 'EvaluationPeriods': 1, + }, + ) + + # Verify JVM Memory Pressure Alarm + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'JVMMemoryPressure', + 'Namespace': 'AWS/ES', + 'Threshold': 60, + 'ComparisonOperator': 'GreaterThanThreshold', + 'EvaluationPeriods': 3, + }, + ) + + # Verify CPU Utilization Alarm + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'CPUUtilization', + 'Namespace': 'AWS/ES', + 'Threshold': 60, + 'ComparisonOperator': 'GreaterThanThreshold', + 'EvaluationPeriods': 3, # 15 minutes sustained + }, + ) + def test_sandbox_uses_expected_private_subnet(self): """ Test that the OpenSearch Domain in sandbox uses expected private Subnet. From e1bcd0eb1f6f0c53e40e811cdf5acd8cc6373b85 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 09:24:21 -0600 Subject: [PATCH 011/137] Tweak alarms thresholds --- .../stacks/search_persistent_stack/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 28c914aeb..dca424991 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -313,11 +313,11 @@ def _add_capacity_alarms(self, environment_name: str): namespace='AWS/ES', metric_name='FreeStorageSpace', dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, - # check every day to avoid alerting on spikes - period=Duration.days(1), + # check twice a day + period=Duration.hours(12), statistic='Minimum', ), - evaluation_periods=1, # 1 day + evaluation_periods=1, # Notify the moment the storage space is less than 50% threshold=storage_threshold_mb, comparison_operator=ComparisonOperator.LESS_THAN_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, @@ -340,12 +340,12 @@ def _add_capacity_alarms(self, environment_name: str): period=Duration.minutes(5), statistic='Maximum', ), - evaluation_periods=3, # 15 minutes sustained - threshold=60, + evaluation_periods=6, # 30 minutes sustained + threshold=70, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, alarm_description=( - f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 60%. ' + f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 70%. ' 'This indicates the cluster is using a significant portion of its heap memory. ' 'Consider scaling to larger instance types if pressure continues to increase.' ), @@ -369,7 +369,7 @@ def _add_capacity_alarms(self, environment_name: str): treat_missing_data=TreatMissingData.NOT_BREACHING, alarm_description=( f'OpenSearch Domain {self.domain.domain_name} CPU utilization has been above 60% for 15 minutes. ' - 'This indicates sustained high load. Consider scaling to larger instance types ' + 'This indicates sustained high load. Review metrics and consider scaling to larger instance types ' 'or adding more data nodes to distribute the load.' ), ).add_alarm_action(SnsAction(self.alarm_topic)) From ae50c5aea1fa999785b15ccef8c519be5fe51d78 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 09:54:20 -0600 Subject: [PATCH 012/137] Add advanced option to prevent specifying index in queries for security --- .../search_persistent_stack/__init__.py | 16 ++++++++++++-- .../tests/app/test_search_persistent_stack.py | 21 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index dca424991..78fa3d52e 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -23,6 +23,9 @@ from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack +PROD_EBS_VOLUME_SIZE = 25 +NON_PROD_EBS_VOLUME_SIZE = 10 + class SearchPersistentStack(AppStack): """ @@ -159,12 +162,21 @@ def __init__( vpc_subnets=vpc_subnets, security_groups=[vpc_stack.opensearch_security_group], # EBS volume configuration - ebs=EbsOptions(enabled=True, volume_size=20 if environment_name == 'prod' else 10), + ebs=EbsOptions( + enabled=True, + volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE + ), # Encryption settings encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.opensearch_encryption_key), node_to_node_encryption=True, enforce_https=True, tls_security_policy=TLSSecurityPolicy.TLS_1_2, + # Advanced security options + advanced_options={ + # Prevent queries from accessing multiple indices in a single request + # This is a security control to ensure queries are scoped to a single index + 'rest.action.multi.allow_explicit_index': 'false', + }, logging=LoggingOptions( app_log_enabled=True, app_log_group=opensearch_app_log_group, @@ -298,7 +310,7 @@ def _add_capacity_alarms(self, environment_name: str): param environment_name: The deployment environment name """ # Get the volume size for calculating storage threshold - volume_size_gb = 20 if environment_name == PROD_ENV_NAME else 10 + volume_size_gb = PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE # 50% threshold in MB (FreeStorageSpace metric is reported in megabytes) # Formula: GB * 1024 MB/GB * 0.5 for 50% threshold storage_threshold_mb = volume_size_gb * 1024 * 0.5 diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 9e5141d5c..350fc359e 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -215,6 +215,27 @@ def test_capacity_alarms_configured(self): }, ) + def test_multi_index_queries_disabled(self): + """ + Test that multi-index queries are disabled for security. + + This verifies that the advanced option 'rest.action.multi.allow_explicit_index' is set to 'false', + which prevents queries from targeting multiple indices in a single request. + This is a security control to ensure queries remain scoped to a single index. + """ + search_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify the advanced option is set to prevent multi-index queries + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'AdvancedOptions': { + 'rest.action.multi.allow_explicit_index': 'false', + }, + }, + ) + def test_sandbox_uses_expected_private_subnet(self): """ Test that the OpenSearch Domain in sandbox uses expected private Subnet. From 5bfbfde36bb728382964e311a93cd17b61252611 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 14:55:08 -0600 Subject: [PATCH 013/137] Add custom resource to manage OpenSearch indices --- .../bin/compile_requirements.sh | 4 + backend/compact-connect/bin/sync_deps.sh | 4 + .../common_constructs/python_function.py | 20 +- .../lambdas/python/common/cc_common/config.py | 14 ++ .../python/search/custom_resource_handler.py | 124 ++++++++++ .../python/search/handlers/__init__.py | 0 .../handlers/manage_opensearch_indices.py | 232 ++++++++++++++++++ .../python/search/opensearch_client.py | 26 ++ .../lambdas/python/search/requirements-dev.in | 1 + .../python/search/requirements-dev.txt | 0 .../lambdas/python/search/requirements.in | 2 + .../lambdas/python/search/requirements.txt | 0 .../lambdas/python/search/tests/__init__.py | 87 +++++++ .../python/search/tests/function/__init__.py | 16 ++ .../test_manage_opensearch_indices.py | 11 + .../compact-connect/pipeline/backend_stage.py | 28 +-- .../search_persistent_stack/__init__.py | 19 +- .../search_persistent_stack/index_manager.py | 160 ++++++++++++ .../stacks/vpc_stack/__init__.py | 9 + .../tests/app/test_search_persistent_stack.py | 5 +- 20 files changed, 740 insertions(+), 22 deletions(-) create mode 100644 backend/compact-connect/lambdas/python/search/custom_resource_handler.py create mode 100644 backend/compact-connect/lambdas/python/search/handlers/__init__.py create mode 100644 backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py create mode 100644 backend/compact-connect/lambdas/python/search/opensearch_client.py create mode 100644 backend/compact-connect/lambdas/python/search/requirements-dev.in create mode 100644 backend/compact-connect/lambdas/python/search/requirements-dev.txt create mode 100644 backend/compact-connect/lambdas/python/search/requirements.in create mode 100644 backend/compact-connect/lambdas/python/search/requirements.txt create mode 100644 backend/compact-connect/lambdas/python/search/tests/__init__.py create mode 100644 backend/compact-connect/lambdas/python/search/tests/function/__init__.py create mode 100644 backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py create mode 100644 backend/compact-connect/stacks/search_persistent_stack/index_manager.py diff --git a/backend/compact-connect/bin/compile_requirements.sh b/backend/compact-connect/bin/compile_requirements.sh index 0706a9062..7ee49131a 100755 --- a/backend/compact-connect/bin/compile_requirements.sh +++ b/backend/compact-connect/bin/compile_requirements.sh @@ -21,6 +21,10 @@ pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/provi # avoid installation failures # pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements-dev.in # pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements.in +pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements-dev.in +pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements.in +pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/search/requirements-dev.in +pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/search/requirements.in pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/staff-user-pre-token/requirements-dev.in pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/staff-user-pre-token/requirements.in pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/staff-users/requirements-dev.in diff --git a/backend/compact-connect/bin/sync_deps.sh b/backend/compact-connect/bin/sync_deps.sh index 1656a985a..8801bdcf6 100755 --- a/backend/compact-connect/bin/sync_deps.sh +++ b/backend/compact-connect/bin/sync_deps.sh @@ -20,6 +20,10 @@ pip-sync \ lambdas/python/disaster-recovery/requirements.txt \ lambdas/python/provider-data-v1/requirements-dev.txt \ lambdas/python/provider-data-v1/requirements.txt \ + lambdas/python/purchases/requirements-dev.txt \ + lambdas/python/purchases/requirements.txt \ + lambdas/python/search/requirements-dev.txt \ + lambdas/python/search/requirements.txt \ lambdas/python/staff-user-pre-token/requirements-dev.txt \ lambdas/python/staff-user-pre-token/requirements.txt \ lambdas/python/staff-users/requirements-dev.txt \ diff --git a/backend/compact-connect/common_constructs/python_function.py b/backend/compact-connect/common_constructs/python_function.py index 0dbcf7828..625eb60ef 100644 --- a/backend/compact-connect/common_constructs/python_function.py +++ b/backend/compact-connect/common_constructs/python_function.py @@ -5,7 +5,7 @@ from aws_cdk import Duration from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Stats, TreatMissingData from aws_cdk.aws_cloudwatch_actions import SnsAction -from aws_cdk.aws_iam import IRole, Role, ServicePrincipal +from aws_cdk.aws_iam import IRole, Role, ServicePrincipal, ManagedPolicy from aws_cdk.aws_lambda import ILayerVersion, Runtime from aws_cdk.aws_lambda_python_alpha import PythonFunction as CdkPythonFunction from aws_cdk.aws_logs import ILogGroup, LogGroup, RetentionDays @@ -81,6 +81,24 @@ def __init__( assumed_by=ServicePrincipal('lambda.amazonaws.com'), ) log_group.grant_write(role) + if 'vpc' in kwargs: + # if the function is being created in a VPC, add the AWSLambdaVPCAccessExecutionRole policy to the role + role.add_managed_policy( + ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaVPCAccessExecutionRole') + ) + NagSuppressions.add_resource_suppressions( + role, + suppressions=[ + { + 'id': 'AwsSolutions-IAM4', + 'appliesTo': [ + 'Policy::arn::iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole' + ], + 'reason': "Lambdas deployed within a VPC require this policy to access the VPC.", + }, + ], + ) + # We can't directly grant a provided role permission to log to our log group, since that could create a # circular dependency with the stack the role came from. The role creator will have to be responsible for # setting its permissions. diff --git a/backend/compact-connect/lambdas/python/common/cc_common/config.py b/backend/compact-connect/lambdas/python/common/cc_common/config.py index 79fd69b75..7ab9b803f 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/config.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/config.py @@ -22,6 +22,20 @@ class _Config: presigned_post_ttl_seconds = 3600 default_page_size = 100 + @property + def environment_region(self): + """ + Returns the region name of the region the lambda is running in. + """ + return os.environ['AWS_REGION'] + + @property + def opensearch_host_endpoint(self): + """ + Returns the region name of the region the lambda is running in. + """ + return os.environ['OPENSEARCH_HOST_ENDPOINT'] + @cached_property def cognito_client(self): return boto3.client('cognito-idp') diff --git a/backend/compact-connect/lambdas/python/search/custom_resource_handler.py b/backend/compact-connect/lambdas/python/search/custom_resource_handler.py new file mode 100644 index 000000000..cb46deb96 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/custom_resource_handler.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +from abc import ABC, abstractmethod +from typing import TypedDict + +from aws_lambda_powertools.logging.lambda_context import build_lambda_context_model +from aws_lambda_powertools.utilities.typing import LambdaContext +from cc_common.config import logger + + +class CustomResourceResponse(TypedDict, total=False): + """Return body for the custom resource handler.""" + + PhysicalResourceId: str + Data: dict + NoEcho: bool + + +class CustomResourceHandler(ABC): + """Base class for custom resource migrations. + + This class provides a framework for implementing CloudFormation custom resources. + It handles the routing of CloudFormation events to appropriate methods and provides a consistent + logging pattern. + + Subclasses must implement the on_create, on_update, and on_delete methods. + + Instances of this class are callable and can be used directly as Lambda handlers. + """ + + def __init__(self, handler_name: str): + """Initialize the custom resource handler. + + :type handler_name: str + """ + self.handler_name = handler_name + + def __call__(self, event: dict, _context: LambdaContext) -> CustomResourceResponse | None: + return self._on_event(event, _context) + + def _on_event(self, event: dict, _context: LambdaContext) -> CustomResourceResponse | None: + """CloudFormation event handler using the CDK provider framework. + See: https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.custom_resources/README.html + + This method routes the event to the appropriate handler method based on the request type. + + :param event: The lambda event with properties in ResourceProperties + :type event: dict + :param _context: Lambda context + :type _context: LambdaContext + :return: Optional result from the handler method + :rtype: Optional[CustomResourceResponse] + :raises ValueError: If the request type is not supported + """ + + # @logger.inject_lambda_context doesn't work on instance methods, so we'll build the context manually + lambda_context = build_lambda_context_model(_context) + logger.structure_logs(**lambda_context.__dict__) + + logger.info(f'{self.handler_name} handler started') + + properties = event.get('ResourceProperties', {}) + request_type = event['RequestType'] + + match request_type: + case 'Create': + try: + resp = self.on_create(properties) + except Exception as e: + logger.error(f'Error in {self.handler_name} creation', exc_info=e) + raise + case 'Update': + try: + resp = self.on_update(properties) + except Exception as e: + logger.error(f'Error in {self.handler_name} update', exc_info=e) + raise + case 'Delete': + try: + resp = self.on_delete(properties) + except Exception as e: + logger.error(f'Error in {self.handler_name} delete', exc_info=e) + raise + case _: + raise ValueError(f'Unexpected request type: {request_type}') + + logger.info(f'{self.handler_name} handler complete') + return resp + + @abstractmethod + def on_create(self, properties: dict) -> CustomResourceResponse | None: + """Handle Create events. + + This method should be implemented by subclasses to perform the migration when a resource is being created. + + :param properties: The ResourceProperties from the CloudFormation event + :type properties: dict + :return: Any result to be returned to CloudFormation + :rtype: Optional[CustomResourceResponse] + """ + + @abstractmethod + def on_update(self, properties: dict) -> CustomResourceResponse | None: + """Handle Update events. + + This method should be implemented by subclasses to perform the migration when a resource is being updated. + + :param properties: The ResourceProperties from the CloudFormation event + :type properties: dict + :return: Any result to be returned to CloudFormation + :rtype: Optional[CustomResourceResponse] + """ + + @abstractmethod + def on_delete(self, properties: dict) -> CustomResourceResponse | None: + """Handle Delete events. + + This method should be implemented by subclasses to handle deletion of the migration. In many cases, this can + be a no-op as the migration is temporary and deletion should have no effect. + + :param properties: The ResourceProperties from the CloudFormation event + :type properties: dict + :return: Any result to be returned to CloudFormation + :rtype: Optional[CustomResourceResponse] + """ diff --git a/backend/compact-connect/lambdas/python/search/handlers/__init__.py b/backend/compact-connect/lambdas/python/search/handlers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py new file mode 100644 index 000000000..c2b04ba1f --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -0,0 +1,232 @@ +from cc_common.config import logger, config +from custom_resource_handler import CustomResourceHandler, CustomResourceResponse +from opensearch_client import OpenSearchClient + + +class OpenSearchIndexManager(CustomResourceHandler): + """ + Custom resource handler to create OpenSearch indices for compacts. + """ + + def on_create(self) -> None: + """ + Create the indices on creation. + """ + logger.info('Connecting to OpenSearch domain') + client = OpenSearchClient() + + compacts = config.compacts + for compact in compacts: + index_name = f'compact_{compact}_providers' + self._create_provider_index(client, index_name) + + def on_update(self, properties: dict) -> None: + """ + No-op on update. + """ + + def on_delete(self, _properties: dict) -> CustomResourceResponse | None: + """ + No-op on delete. + """ + + def _create_provider_index(self, client: OpenSearchClient, index_name: str) -> None: + """ + Create the provider index in OpenSearch if it doesn't exist. + """ + if client.index_exists(index_name): + logger.info(f"Index '{index_name}' already exists. Skipping creation.") + return + logger.info(f"Creating index '{index_name}'...") + client.create_index(index_name, self._get_provider_index_mapping()) + logger.info(f"Index '{index_name}' created successfully.") + + def _get_provider_index_mapping(self) -> dict: + """ + Define the index mapping for provider documents. + """ + # Nested schema for AttestationVersion + attestation_version_properties = { + 'attestationId': {'type': 'keyword'}, + 'version': {'type': 'keyword'}, + } + + # Nested schema for AdverseAction + adverse_action_properties = { + 'type': {'type': 'keyword'}, + 'adverseActionId': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'licenseTypeAbbreviation': {'type': 'keyword'}, + 'actionAgainst': {'type': 'keyword'}, + 'effectiveStartDate': {'type': 'date'}, + 'creationDate': {'type': 'date'}, + 'effectiveLiftDate': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'encumbranceType': {'type': 'keyword'}, + 'clinicalPrivilegeActionCategories': {'type': 'keyword'}, + 'clinicalPrivilegeActionCategory': {'type': 'keyword'}, + 'submittingUser': {'type': 'keyword'}, + 'liftingUser': {'type': 'keyword'}, + } + + # Nested schema for Investigation + investigation_properties = { + 'type': {'type': 'keyword'}, + 'investigationId': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'status': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + } + + # Nested schema for License + license_properties = { + 'providerId': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + 'compact': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'licenseStatusName': {'type': 'keyword'}, + 'licenseStatus': {'type': 'keyword'}, + 'jurisdictionUploadedLicenseStatus': {'type': 'keyword'}, + 'compactEligibility': {'type': 'keyword'}, + 'jurisdictionUploadedCompactEligibility': {'type': 'keyword'}, + 'npi': {'type': 'keyword'}, + 'licenseNumber': {'type': 'keyword'}, + 'givenName': { + 'type': 'text', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'middleName': { + 'type': 'text', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'familyName': { + 'type': 'text', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'suffix': {'type': 'keyword'}, + 'dateOfIssuance': {'type': 'date'}, + 'dateOfRenewal': {'type': 'date'}, + 'dateOfExpiration': {'type': 'date'}, + 'homeAddressStreet1': {'type': 'text'}, + 'homeAddressStreet2': {'type': 'text'}, + 'homeAddressCity': { + 'type': 'text', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'homeAddressState': {'type': 'keyword'}, + 'homeAddressPostalCode': {'type': 'keyword'}, + 'emailAddress': {'type': 'keyword'}, + 'phoneNumber': {'type': 'keyword'}, + 'adverseActions': {'type': 'nested', 'properties': adverse_action_properties}, + 'investigations': {'type': 'nested', 'properties': investigation_properties}, + 'investigationStatus': {'type': 'keyword'}, + } + + # Nested schema for Privilege + privilege_properties = { + 'type': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseJurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'dateOfIssuance': {'type': 'date'}, + 'dateOfRenewal': {'type': 'date'}, + 'dateOfExpiration': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'adverseActions': {'type': 'nested', 'properties': adverse_action_properties}, + 'investigations': {'type': 'nested', 'properties': investigation_properties}, + 'administratorSetStatus': {'type': 'keyword'}, + 'compactTransactionId': {'type': 'keyword'}, + 'attestations': {'type': 'nested', 'properties': attestation_version_properties}, + 'privilegeId': {'type': 'keyword'}, + 'status': {'type': 'keyword'}, + 'activeSince': {'type': 'date'}, + 'investigationStatus': {'type': 'keyword'}, + } + + # Nested schema for MilitaryAffiliation + military_affiliation_properties = { + 'type': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + 'providerId': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'fileNames': {'type': 'keyword'}, + 'affiliationType': {'type': 'keyword'}, + 'dateOfUpload': {'type': 'date'}, + 'status': {'type': 'keyword'}, + } + + return { + 'settings': { + 'index': { + 'number_of_shards': 1, + 'number_of_replicas': 0, + 'refresh_interval': '1s', + }, + 'analysis': { + 'analyzer': { + 'name_analyzer': { + 'type': 'custom', + 'tokenizer': 'standard', + 'filter': ['lowercase', 'asciifolding'], + } + } + }, + }, + 'mappings': { + 'properties': { + # Top-level provider fields + 'providerId': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + 'compact': {'type': 'keyword'}, + 'licenseJurisdiction': {'type': 'keyword'}, + 'currentHomeJurisdiction': {'type': 'keyword'}, + 'licenseStatus': {'type': 'keyword'}, + 'compactEligibility': {'type': 'keyword'}, + 'npi': {'type': 'keyword'}, + 'givenName': { + 'type': 'text', + 'analyzer': 'name_analyzer', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'middleName': { + 'type': 'text', + 'analyzer': 'name_analyzer', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'familyName': { + 'type': 'text', + 'analyzer': 'name_analyzer', + 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, + }, + 'suffix': {'type': 'keyword'}, + 'dateOfExpiration': {'type': 'date'}, + 'compactConnectRegisteredEmailAddress': {'type': 'keyword'}, + 'jurisdictionUploadedLicenseStatus': {'type': 'keyword'}, + 'jurisdictionUploadedCompactEligibility': {'type': 'keyword'}, + 'privilegeJurisdictions': {'type': 'keyword'}, + 'providerFamGivMid': {'type': 'text', 'analyzer': 'name_analyzer'}, + 'providerDateOfUpdate': {'type': 'date'}, + 'birthMonthDay': {'type': 'keyword'}, + # Nested arrays + 'licenses': {'type': 'nested', 'properties': license_properties}, + 'privileges': {'type': 'nested', 'properties': privilege_properties}, + 'militaryAffiliations': { + 'type': 'nested', + 'properties': military_affiliation_properties, + }, + } + }, + } + + +on_event = OpenSearchIndexManager('opensearch-index-manager') diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py new file mode 100644 index 000000000..c598aa159 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -0,0 +1,26 @@ +from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth +from cc_common.config import config +import boto3 + +class OpenSearchClient: + def __init__(self): + lambda_credentials = boto3.Session().get_credentials() + auth = AWSV4SignerAuth( + credentials=lambda_credentials, + region=config.environment_region, + service='es' + ) + self._client = OpenSearch( + hosts = [{'host': config.opensearch_host_endpoint, 'port': 443}], + http_auth = auth, + use_ssl = True, + verify_certs = True, + connection_class = RequestsHttpConnection, + pool_maxsize = 20 + ) + + def create_index(self, index_name: str, index_mapping: dict) -> None: + self._client.indices.create(index=index_name, body=index_mapping) + + def index_exists(self, index_name: str) -> bool: + return self._client.indices.exists(index=index_name) diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.in b/backend/compact-connect/lambdas/python/search/requirements-dev.in new file mode 100644 index 000000000..e0c3124af --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/requirements-dev.in @@ -0,0 +1 @@ +moto[dynamodb]>=5.0.12, <6 diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.txt b/backend/compact-connect/lambdas/python/search/requirements-dev.txt new file mode 100644 index 000000000..e69de29bb diff --git a/backend/compact-connect/lambdas/python/search/requirements.in b/backend/compact-connect/lambdas/python/search/requirements.in new file mode 100644 index 000000000..0c9e7c499 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/requirements.in @@ -0,0 +1,2 @@ +# common requirements are managed in the common requirements.in file +opensearch-py>=3.1.0, <4.0.0 diff --git a/backend/compact-connect/lambdas/python/search/requirements.txt b/backend/compact-connect/lambdas/python/search/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/backend/compact-connect/lambdas/python/search/tests/__init__.py b/backend/compact-connect/lambdas/python/search/tests/__init__.py new file mode 100644 index 000000000..214a66839 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/__init__.py @@ -0,0 +1,87 @@ +import json +import os +from unittest import TestCase +from unittest.mock import MagicMock + +from aws_lambda_powertools.utilities.typing import LambdaContext + + +class TstLambdas(TestCase): + @classmethod + def setUpClass(cls): + os.environ.update( + { + # Set to 'true' to enable debug logging + 'DEBUG': 'true', + 'ALLOWED_ORIGINS': '["https://example.org"]', + 'AWS_DEFAULT_REGION': 'us-east-1', + 'AWS_REGION': 'us-east-1', + 'ENVIRONMENT_NAME': 'test', + 'COMPACTS': '["aslp", "octp", "coun"]', + 'OPENSEARCH_HOST_ENDPOINT': 'vpc-providersearchd-5bzuqxhpxffk-w6dkpddu.us-east-1.es.amazonaws.com', + 'JURISDICTIONS': json.dumps( + [ + 'al', + 'ak', + 'az', + 'ar', + 'ca', + 'co', + 'ct', + 'de', + 'dc', + 'fl', + 'ga', + 'hi', + 'id', + 'il', + 'in', + 'ia', + 'ks', + 'ky', + 'la', + 'me', + 'md', + 'ma', + 'mi', + 'mn', + 'ms', + 'mo', + 'mt', + 'ne', + 'nv', + 'nh', + 'nj', + 'nm', + 'ny', + 'nc', + 'nd', + 'oh', + 'ok', + 'or', + 'pa', + 'pr', + 'ri', + 'sc', + 'sd', + 'tn', + 'tx', + 'ut', + 'vt', + 'va', + 'vi', + 'wa', + 'wv', + 'wi', + 'wy', + ] + ), + }, + ) + # Monkey-patch config object to be sure we have it based + # on the env vars we set above + import cc_common.config + + cls.config = cc_common.config._Config() # noqa: SLF001 protected-access + cc_common.config.config = cls.config + cls.mock_context = MagicMock(name='MockLambdaContext', spec=LambdaContext) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py new file mode 100644 index 000000000..1d127059a --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -0,0 +1,16 @@ +from moto import mock_aws + +from tests import TstLambdas + + +@mock_aws +class TstFunction(TstLambdas): + """Base class to set up Moto mocking and create mock AWS resources for functional testing""" + + def setUp(self): # noqa: N801 invalid-name + super().setUp() + # This must be imported within the tests, since they import modules which require + # environment variables that are not set until the TstLambdas class is initialized + from common_test.test_data_generator import TestDataGenerator + + self.test_data_generator = TestDataGenerator diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py new file mode 100644 index 000000000..c28011510 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -0,0 +1,11 @@ +from moto import mock_aws + +from . import TstFunction + + +@mock_aws +class TestOpenSearchIndexManager(TstFunction): + """Test suite for ManageFeatureFlagHandler custom resource.""" + + def setUp(self): + super().setUp() diff --git a/backend/compact-connect/pipeline/backend_stage.py b/backend/compact-connect/pipeline/backend_stage.py index fe46abfb8..d7a828739 100644 --- a/backend/compact-connect/pipeline/backend_stage.py +++ b/backend/compact-connect/pipeline/backend_stage.py @@ -50,20 +50,6 @@ def __init__( environment_name=environment_name, ) - # Search Persistent Stack - OpenSearch Domain for advanced provider search - # currently not deploying to prod or beta to reduce costs until search api functionality is completed - # to reduce costs - if environment_name != 'prod' and environment_name != 'beta': - self.search_persistent_stack = SearchPersistentStack( - self, - 'SearchPersistentStack', - env=environment, - environment_context=environment_context, - standard_tags=standard_tags, - environment_name=environment_name, - vpc_stack=self.vpc_stack, - ) - self.persistent_stack = PersistentStack( self, 'PersistentStack', @@ -246,3 +232,17 @@ def __init__( # Explicitly declare the dependency to ensure proper deployment order self.data_migration_stack.add_dependency(self.api_stack) self.data_migration_stack.add_dependency(self.event_listener_stack) + + # Search Persistent Stack - OpenSearch Domain for advanced provider search + # currently not deploying to prod or beta to reduce costs until search api functionality is completed + # to reduce costs + if environment_name != 'prod' and environment_name != 'beta': + self.search_persistent_stack = SearchPersistentStack( + self, + 'SearchPersistentStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + vpc_stack=self.vpc_stack, + ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 78fa3d52e..059b33822 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -17,10 +17,11 @@ ) from cdk_nag import NagSuppressions from common_constructs.alarm_topic import AlarmTopic -from common_constructs.constants import PROD_ENV_NAME from common_constructs.stack import AppStack from constructs import Construct +from common_constructs.constants import PROD_ENV_NAME +from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack PROD_EBS_VOLUME_SIZE = 25 @@ -159,7 +160,7 @@ def __init__( capacity=capacity_config, # VPC configuration for network isolation vpc=vpc_stack.vpc, - vpc_subnets=vpc_subnets, + vpc_subnets=[vpc_subnets], security_groups=[vpc_stack.opensearch_security_group], # EBS volume configuration ebs=EbsOptions( @@ -192,6 +193,14 @@ def __init__( zone_awareness=zone_awareness_config, ) + self.index_manager_custom_resource = IndexManagerCustomResource( + self, + construct_id='indexManager', + opensearch_domain=self.domain, + vpc_stack=vpc_stack, + vpc_subnets=vpc_subnets, + ) + # Add CDK Nag suppressions for OpenSearch Domain self._add_opensearch_suppressions(environment_name) @@ -251,7 +260,7 @@ def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConf # non-prod environments only use one data node, hence we don't enable zone awareness return ZoneAwarenessConfig(enabled=False) - def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> list[SubnetSelection]: + def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> SubnetSelection: """ Determine VPC subnet selection based on environment. @@ -266,13 +275,13 @@ def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> list[S if environment_name == PROD_ENV_NAME: # Production: Use all private isolated subnets from the VPC. # VPC is configured with max_azs=3, so this will select exactly 3 subnets - return [SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED)] + return SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED) # Non-prod: Single-node deployment explicitly uses privateSubnet1 (CIDR 10.0.0.0/20) # OpenSearch requires exactly one subnet for single-node deployments # We explicitly find the subnet by its construct name to guarantee consistency private_subnet1 = self._find_subnet_by_name(vpc_stack.vpc, PRIVATE_SUBNET_ONE_NAME) - return [SubnetSelection(subnets=[private_subnet1])] + return SubnetSelection(subnets=[private_subnet1]) def _find_subnet_by_name(self, vpc, subnet_name: str): """ diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py new file mode 100644 index 000000000..46ffa97dc --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -0,0 +1,160 @@ +import os + +from aws_cdk import CustomResource, Duration +from aws_cdk.aws_logs import LogGroup, RetentionDays +from aws_cdk.aws_opensearchservice import Domain +from aws_cdk.custom_resources import Provider +from cdk_nag import NagSuppressions +from common_constructs.stack import Stack +from constructs import Construct +from moto.ec2.models.subnets import Subnet + +from common_constructs.python_function import PythonFunction +from stacks.vpc_stack import VpcStack + + +class IndexManagerCustomResource(Construct): + """ + Custom resource for managing OpenSearch indices. + + This construct creates a CloudFormation custom resource that populates the OpenSearch Domain with the needed + provider indices. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + opensearch_domain: Domain, + vpc_stack: VpcStack, + vpc_subnets: list[Subnet], + ): + """ + Initialize the IndexManagerCustomResource construct. + + :param scope: The scope of the construct + :param construct_id: The id of the construct + :param opensearch_domain: The reference to the OpenSearch domain resource + :param vpc_stack: The VPC stack + :param vpc_subnets: The VPC subnets + """ + super().__init__(scope, construct_id) + stack = Stack.of(scope) + + # Create Lambda function for managing OpenSearch indices + # This function is reused across all FeatureFlagResource instances + self.manage_function = PythonFunction( + self, + 'IndexManagerFunction', + index=os.path.join('handlers', 'manage_opensearch_indices.py'), + lambda_dir='search', + handler='on_event', + log_retention=RetentionDays.ONE_MONTH, + environment={ + 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, + **stack.common_env_vars, + }, + timeout=Duration.minutes(5), + memory_size=256, + vpc=vpc_stack.vpc, + vpc_subnets=vpc_subnets, + security_groups=[vpc_stack.lambda_security_group] + ) + # grant resource ability to create and check indices + opensearch_domain.grant_read_write(self.manage_function) + + # Add CDK Nag suppressions for the Lambda function's IAM role + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{self.manage_function.role.node.path}/DefaultPolicy/Resource', + [ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The grant_read_write method requires wildcard permissions on the OpenSearch domain to ' + 'create, read, and manage indices. This is appropriate for an index management function ' + 'that needs to operate on all indices in the domain.', + }, + ], + ) + + provider_log_group = LogGroup( + self, + 'ProviderLogGroup', + retention=RetentionDays.ONE_DAY, + ) + NagSuppressions.add_resource_suppressions( + provider_log_group, + suppressions=[ + { + 'id': 'HIPAA.Security-CloudWatchLogGroupEncrypted', + 'reason': 'We do not log sensitive data to CloudWatch, and operational visibility of system' + ' logs to operators with credentials for the AWS account is desired. Encryption is not' + ' appropriate here.', + }, + ], + ) + + # Create custom resource provider + # Note: Provider framework Lambda does NOT need VPC access - it only needs to: + # 1. Invoke the Lambda (via Lambda service API, no VPC needed) + # 2. Respond to CloudFormation + provider = Provider( + self, + 'Provider', + on_event_handler=self.manage_function, + log_group=provider_log_group, + ) + + # Add CDK Nag suppressions for the provider framework + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{provider.node.path}/framework-onEvent/Resource', + [ + {'id': 'AwsSolutions-L1', 'reason': 'We do not control this runtime'}, + { + 'id': 'HIPAA.Security-LambdaConcurrency', + 'reason': 'This function is only run at deploy time, by CloudFormation and has no need for ' + 'concurrency limits.', + }, + { + 'id': 'HIPAA.Security-LambdaDLQ', + 'reason': 'This is a synchronous function that runs at deploy time. It does not need a DLQ', + }, + ], + ) + + # Add CDK Nag suppressions for the provider framework's IAM role + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{provider.node.path}/framework-onEvent/ServiceRole/Resource', + [ + { + 'id': 'AwsSolutions-IAM4', + 'reason': 'The Provider framework requires AWS managed policies (AWSLambdaBasicExecutionRole) ' + 'for its service role. We do not control these policies.', + }, + ], + ) + + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{provider.node.path}/framework-onEvent/ServiceRole/DefaultPolicy/Resource', + [ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The Provider framework requires wildcard permissions to invoke the Lambda function. ' + 'This is a standard pattern for custom resource providers and is necessary for the ' + 'framework to manage the custom resource lifecycle.', + }, + ], + ) + + # Create custom resource for managing indices + # This custom resource will create the 'compact_{compact}_providers' indices + # with the appropriate mappings once the domain is ready + self.index_manager = CustomResource( + self, + 'IndexManagerCustomResource', + resource_type='Custom::IndexManager', + service_token=provider.service_token, + ) diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index cb7f7fdb2..121e28681 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -169,6 +169,15 @@ def __init__( service=GatewayVpcEndpointAwsService.DYNAMODB, ) + # VPC Endpoint for S3 + # This is needed for our custom resource which manages OpenSearch indices to access + # the CloudFormation S3 bucket without internet access + self.s3_vpc_endpoint = self.vpc.add_gateway_endpoint( + 'S3VpcEndpoint', + service=GatewayVpcEndpointAwsService.S3, + ) + + # Security Group for Lambda Functions # This will control inbound and outbound traffic for Lambda functions that interact with OpenSearch self.lambda_security_group = SecurityGroup( diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 350fc359e..93207e8f0 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -30,6 +30,7 @@ def test_opensearch_domain_created(self): search_stack = self.app.sandbox_backend_stage.search_persistent_stack search_template = Template.from_stack(search_stack) + # Verify exactly one OpenSearch Domain is created search_template.resource_count_is('AWS::OpenSearchService::Domain', 1) @@ -197,9 +198,9 @@ def test_capacity_alarms_configured(self): { 'MetricName': 'JVMMemoryPressure', 'Namespace': 'AWS/ES', - 'Threshold': 60, + 'Threshold': 70, 'ComparisonOperator': 'GreaterThanThreshold', - 'EvaluationPeriods': 3, + 'EvaluationPeriods': 6, }, ) From 9028af014815595517eeef5dee367c91cf3b581c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 15:07:23 -0600 Subject: [PATCH 014/137] Add requirements for search lambda directory --- .../python/search/requirements-dev.txt | 68 +++++++++++++++++++ .../lambdas/python/search/requirements.txt | 36 ++++++++++ 2 files changed, 104 insertions(+) diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.txt b/backend/compact-connect/lambdas/python/search/requirements-dev.txt index e69de29bb..aba8ea2b7 100644 --- a/backend/compact-connect/lambdas/python/search/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/search/requirements-dev.txt @@ -0,0 +1,68 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements-dev.in +# +boto3==1.41.4 + # via moto +botocore==1.41.4 + # via + # boto3 + # moto + # s3transfer +certifi==2025.11.12 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.4 + # via requests +cryptography==46.0.3 + # via moto +docker==7.1.0 + # via moto +idna==3.11 + # via requests +jinja2==3.1.6 + # via moto +jmespath==1.0.1 + # via + # boto3 + # botocore +markupsafe==3.0.3 + # via + # jinja2 + # werkzeug +moto[dynamodb]==5.1.17 + # via -r lambdas/python/search/requirements-dev.in +py-partiql-parser==0.6.3 + # via moto +pycparser==2.23 + # via cffi +python-dateutil==2.9.0.post0 + # via + # botocore + # moto +pyyaml==6.0.3 + # via responses +requests==2.32.5 + # via + # docker + # moto + # responses +responses==0.25.8 + # via moto +s3transfer==0.15.0 + # via boto3 +six==1.17.0 + # via python-dateutil +urllib3==2.5.0 + # via + # botocore + # docker + # requests + # responses +werkzeug==3.1.3 + # via moto +xmltodict==1.0.2 + # via moto diff --git a/backend/compact-connect/lambdas/python/search/requirements.txt b/backend/compact-connect/lambdas/python/search/requirements.txt index e69de29bb..ed72ac2ec 100644 --- a/backend/compact-connect/lambdas/python/search/requirements.txt +++ b/backend/compact-connect/lambdas/python/search/requirements.txt @@ -0,0 +1,36 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements.in +# +certifi==2025.11.12 + # via + # opensearch-py + # requests +charset-normalizer==3.4.4 + # via requests +events==0.5 + # via opensearch-py +grpcio==1.76.0 + # via opensearch-protobufs +idna==3.11 + # via requests +opensearch-protobufs==0.19.0 + # via opensearch-py +opensearch-py==3.1.0 + # via -r lambdas/python/search/requirements.in +protobuf==6.33.1 + # via opensearch-protobufs +python-dateutil==2.9.0.post0 + # via opensearch-py +requests==2.32.5 + # via opensearch-py +six==1.17.0 + # via python-dateutil +typing-extensions==4.15.0 + # via grpcio +urllib3==2.5.0 + # via + # opensearch-py + # requests From 56f6ff8eeb199f0978dfeb2874dbb4b734340654 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 16:56:51 -0600 Subject: [PATCH 015/137] Add domain access policy to restrict access to lambda roles --- .../search_persistent_stack/__init__.py | 68 ++++++++++++++++++- .../search_persistent_stack/index_manager.py | 3 + 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 059b33822..31bf84dca 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,8 +1,8 @@ -from aws_cdk import Duration, RemovalPolicy +from aws_cdk import Duration, Fn, RemovalPolicy from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Metric, TreatMissingData from aws_cdk.aws_cloudwatch_actions import SnsAction from aws_cdk.aws_ec2 import SubnetSelection, SubnetType -from aws_cdk.aws_iam import Effect, PolicyStatement, ServicePrincipal +from aws_cdk.aws_iam import Effect, PolicyStatement, Role, ServicePrincipal from aws_cdk.aws_kms import Key from aws_cdk.aws_logs import LogGroup, ResourcePolicy, RetentionDays from aws_cdk.aws_opensearchservice import ( @@ -80,6 +80,30 @@ def __init__( log_principal = ServicePrincipal('logs.amazonaws.com') self.opensearch_encryption_key.grant_encrypt_decrypt(log_principal) + # Create IAM roles for Lambda functions that need OpenSearch access + self.opensearch_ingest_lambda_role = Role( + self, + 'OpenSearchIngestLambdaRole', + assumed_by=ServicePrincipal('lambda.amazonaws.com'), + description='IAM role for Ingest Lambda function that needs write access to OpenSearch Domain', + ) + + self.opensearch_index_manager_lambda_role = Role( + self, + 'OpenSearchIndexManagerLambdaRole', + assumed_by=ServicePrincipal('lambda.amazonaws.com'), + description='IAM role for index manager Lambda function that needs read/write access to OpenSearch Domain', + ) + + # Create IAM role for Lambda functions access OpenSearch through API + # this role only needs read access + self.search_api_lambda_role = Role( + self, + 'SearchApiLambdaRole', + assumed_by=ServicePrincipal('lambda.amazonaws.com'), + description='IAM role for Search API Lambda functions that need read access to OpenSearch Domain', + ) + # Create dedicated KMS key for alarm topic encryption search_alarm_encryption_key = Key( self, @@ -193,12 +217,52 @@ def __init__( zone_awareness=zone_awareness_config, ) + opensearch_ingest_access_policy = PolicyStatement( + effect=Effect.ALLOW, + principals=[self.opensearch_ingest_lambda_role], + actions=[ + 'es:ESHttpPost', + 'es:ESHttpPut', + ], + resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + ) + opensearch_index_manager_access_policy = PolicyStatement( + effect=Effect.ALLOW, + principals=[self.opensearch_index_manager_lambda_role], + actions=[ + 'es:ESHttpGet', + 'es:ESHttpPost', + 'es:ESHttpPut', + ], + resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + ) + opensearch_search_api_access_policy = PolicyStatement( + effect=Effect.ALLOW, + principals=[self.search_api_lambda_role], + actions=[ + 'es:ESHttpGet', + ], + resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + ) + # add access policy to restrict access to set of roles + self.domain.add_access_policies( + opensearch_ingest_access_policy, + opensearch_index_manager_access_policy, + opensearch_search_api_access_policy + ) + # grant lambda roles access to domain + self.domain.grant_read(self.search_api_lambda_role) + self.domain.grant_write(self.opensearch_ingest_lambda_role) + self.domain.grant_read_write(self.opensearch_index_manager_lambda_role) + + self.index_manager_custom_resource = IndexManagerCustomResource( self, construct_id='indexManager', opensearch_domain=self.domain, vpc_stack=vpc_stack, vpc_subnets=vpc_subnets, + lambda_role=self.opensearch_index_manager_lambda_role ) # Add CDK Nag suppressions for OpenSearch Domain diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index 46ffa97dc..ae3049cdf 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -1,6 +1,7 @@ import os from aws_cdk import CustomResource, Duration +from aws_cdk.aws_iam import IRole from aws_cdk.aws_logs import LogGroup, RetentionDays from aws_cdk.aws_opensearchservice import Domain from aws_cdk.custom_resources import Provider @@ -28,6 +29,7 @@ def __init__( opensearch_domain: Domain, vpc_stack: VpcStack, vpc_subnets: list[Subnet], + lambda_role: IRole ): """ Initialize the IndexManagerCustomResource construct. @@ -49,6 +51,7 @@ def __init__( index=os.path.join('handlers', 'manage_opensearch_indices.py'), lambda_dir='search', handler='on_event', + role=lambda_role, log_retention=RetentionDays.ONE_MONTH, environment={ 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, From 2d066ceed3dd86172f9a0d761c25dbcb56a49335 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 17:01:04 -0600 Subject: [PATCH 016/137] formatting --- .../handlers/manage_opensearch_indices.py | 2 +- .../python/search/opensearch_client.py | 25 +++++------ .../search_persistent_stack/__init__.py | 44 +++++++++---------- .../search_persistent_stack/index_manager.py | 32 +++++++------- .../stacks/vpc_stack/__init__.py | 4 +- .../tests/app/test_search_persistent_stack.py | 9 ++-- backend/compact-connect/tests/app/test_vpc.py | 4 +- 7 files changed, 54 insertions(+), 66 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index c2b04ba1f..b031a7ac2 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -1,4 +1,4 @@ -from cc_common.config import logger, config +from cc_common.config import config, logger from custom_resource_handler import CustomResourceHandler, CustomResourceResponse from opensearch_client import OpenSearchClient diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index c598aa159..fdcd30fd9 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -1,23 +1,20 @@ -from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth -from cc_common.config import config import boto3 +from cc_common.config import config +from opensearchpy import AWSV4SignerAuth, OpenSearch, RequestsHttpConnection + class OpenSearchClient: def __init__(self): lambda_credentials = boto3.Session().get_credentials() - auth = AWSV4SignerAuth( - credentials=lambda_credentials, - region=config.environment_region, - service='es' - ) + auth = AWSV4SignerAuth(credentials=lambda_credentials, region=config.environment_region, service='es') self._client = OpenSearch( - hosts = [{'host': config.opensearch_host_endpoint, 'port': 443}], - http_auth = auth, - use_ssl = True, - verify_certs = True, - connection_class = RequestsHttpConnection, - pool_maxsize = 20 - ) + hosts=[{'host': config.opensearch_host_endpoint, 'port': 443}], + http_auth=auth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + pool_maxsize=20, + ) def create_index(self, index_name: str, index_mapping: dict) -> None: self._client.indices.create(index=index_name, body=index_mapping) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 31bf84dca..9b18cd007 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -189,7 +189,7 @@ def __init__( # EBS volume configuration ebs=EbsOptions( enabled=True, - volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE + volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, ), # Encryption settings encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.opensearch_encryption_key), @@ -218,14 +218,14 @@ def __init__( ) opensearch_ingest_access_policy = PolicyStatement( - effect=Effect.ALLOW, - principals=[self.opensearch_ingest_lambda_role], - actions=[ - 'es:ESHttpPost', - 'es:ESHttpPut', - ], - resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], - ) + effect=Effect.ALLOW, + principals=[self.opensearch_ingest_lambda_role], + actions=[ + 'es:ESHttpPost', + 'es:ESHttpPut', + ], + resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + ) opensearch_index_manager_access_policy = PolicyStatement( effect=Effect.ALLOW, principals=[self.opensearch_index_manager_lambda_role], @@ -246,23 +246,20 @@ def __init__( ) # add access policy to restrict access to set of roles self.domain.add_access_policies( - opensearch_ingest_access_policy, - opensearch_index_manager_access_policy, - opensearch_search_api_access_policy + opensearch_ingest_access_policy, opensearch_index_manager_access_policy, opensearch_search_api_access_policy ) # grant lambda roles access to domain self.domain.grant_read(self.search_api_lambda_role) self.domain.grant_write(self.opensearch_ingest_lambda_role) self.domain.grant_read_write(self.opensearch_index_manager_lambda_role) - self.index_manager_custom_resource = IndexManagerCustomResource( self, construct_id='indexManager', opensearch_domain=self.domain, vpc_stack=vpc_stack, vpc_subnets=vpc_subnets, - lambda_role=self.opensearch_index_manager_lambda_role + lambda_role=self.opensearch_index_manager_lambda_role, ) # Add CDK Nag suppressions for OpenSearch Domain @@ -474,16 +471,16 @@ def _add_opensearch_suppressions(self, environment_name: str): { 'id': 'AwsSolutions-OS3', 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups.' - 'The data in the domain is only accessible by the ingest lambda which indexes the' - 'documents and the search API lambda which can only be accessed by authenticated staff' - 'users in CompactConnect.', + 'The data in the domain is only accessible by the ingest lambda which indexes the' + 'documents and the search API lambda which can only be accessed by authenticated staff' + 'users in CompactConnect.', }, { 'id': 'AwsSolutions-OS5', 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups.' - 'The data in the domain is only accessible by the ingest lambda which indexes the' - 'documents and the search API lambda which can only be accessed by authenticated staff' - 'users in CompactConnect.', + 'The data in the domain is only accessible by the ingest lambda which indexes the' + 'documents and the search API lambda which can only be accessed by authenticated staff' + 'users in CompactConnect.', }, ], apply_to_children=True, @@ -495,15 +492,14 @@ def _add_opensearch_suppressions(self, environment_name: str): { 'id': 'AwsSolutions-OS4', 'reason': 'Dedicated master nodes are only used in production environments with multiple data ' - 'nodes. Single-node non-prod environments do not require dedicated master nodes.', + 'nodes. Single-node non-prod environments do not require dedicated master nodes.', }, { 'id': 'AwsSolutions-OS7', 'reason': 'Zone awareness with standby is only enabled for production environments with ' - 'multiple nodes. Single-node test environments do not require multi-AZ ' - 'configuration.', + 'multiple nodes. Single-node test environments do not require multi-AZ ' + 'configuration.', }, ], apply_to_children=True, ) - diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index ae3049cdf..9785d27bd 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -23,13 +23,13 @@ class IndexManagerCustomResource(Construct): """ def __init__( - self, - scope: Construct, - construct_id: str, - opensearch_domain: Domain, - vpc_stack: VpcStack, - vpc_subnets: list[Subnet], - lambda_role: IRole + self, + scope: Construct, + construct_id: str, + opensearch_domain: Domain, + vpc_stack: VpcStack, + vpc_subnets: list[Subnet], + lambda_role: IRole, ): """ Initialize the IndexManagerCustomResource construct. @@ -61,7 +61,7 @@ def __init__( memory_size=256, vpc=vpc_stack.vpc, vpc_subnets=vpc_subnets, - security_groups=[vpc_stack.lambda_security_group] + security_groups=[vpc_stack.lambda_security_group], ) # grant resource ability to create and check indices opensearch_domain.grant_read_write(self.manage_function) @@ -74,8 +74,8 @@ def __init__( { 'id': 'AwsSolutions-IAM5', 'reason': 'The grant_read_write method requires wildcard permissions on the OpenSearch domain to ' - 'create, read, and manage indices. This is appropriate for an index management function ' - 'that needs to operate on all indices in the domain.', + 'create, read, and manage indices. This is appropriate for an index management function ' + 'that needs to operate on all indices in the domain.', }, ], ) @@ -91,8 +91,8 @@ def __init__( { 'id': 'HIPAA.Security-CloudWatchLogGroupEncrypted', 'reason': 'We do not log sensitive data to CloudWatch, and operational visibility of system' - ' logs to operators with credentials for the AWS account is desired. Encryption is not' - ' appropriate here.', + ' logs to operators with credentials for the AWS account is desired. Encryption is not' + ' appropriate here.', }, ], ) @@ -117,7 +117,7 @@ def __init__( { 'id': 'HIPAA.Security-LambdaConcurrency', 'reason': 'This function is only run at deploy time, by CloudFormation and has no need for ' - 'concurrency limits.', + 'concurrency limits.', }, { 'id': 'HIPAA.Security-LambdaDLQ', @@ -134,7 +134,7 @@ def __init__( { 'id': 'AwsSolutions-IAM4', 'reason': 'The Provider framework requires AWS managed policies (AWSLambdaBasicExecutionRole) ' - 'for its service role. We do not control these policies.', + 'for its service role. We do not control these policies.', }, ], ) @@ -146,8 +146,8 @@ def __init__( { 'id': 'AwsSolutions-IAM5', 'reason': 'The Provider framework requires wildcard permissions to invoke the Lambda function. ' - 'This is a standard pattern for custom resource providers and is necessary for the ' - 'framework to manage the custom resource lifecycle.', + 'This is a standard pattern for custom resource providers and is necessary for the ' + 'framework to manage the custom resource lifecycle.', }, ], ) diff --git a/backend/compact-connect/stacks/vpc_stack/__init__.py b/backend/compact-connect/stacks/vpc_stack/__init__.py index 121e28681..650889e9f 100644 --- a/backend/compact-connect/stacks/vpc_stack/__init__.py +++ b/backend/compact-connect/stacks/vpc_stack/__init__.py @@ -18,7 +18,6 @@ from common_constructs.stack import AppStack from constructs import Construct - PRIVATE_SUBNET_ONE_NAME = 'privateSubnet1' PRIVATE_SUBNET_TWO_NAME = 'privateSubnet2' PRIVATE_SUBNET_THREE_NAME = 'privateSubnet3' @@ -120,7 +119,7 @@ def __init__( 'VpcFlowLogGroup', retention=RetentionDays.ONE_MONTH, removal_policy=removal_policy, - encryption_key=self.vpc_encryption_key + encryption_key=self.vpc_encryption_key, ) self.vpc.add_flow_log( @@ -177,7 +176,6 @@ def __init__( service=GatewayVpcEndpointAwsService.S3, ) - # Security Group for Lambda Functions # This will control inbound and outbound traffic for Lambda functions that interact with OpenSearch self.lambda_security_group = SecurityGroup( diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 93207e8f0..62b1d5702 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -30,7 +30,6 @@ def test_opensearch_domain_created(self): search_stack = self.app.sandbox_backend_stage.search_persistent_stack search_template = Template.from_stack(search_stack) - # Verify exactly one OpenSearch Domain is created search_template.resource_count_is('AWS::OpenSearchService::Domain', 1) @@ -123,7 +122,9 @@ def test_ebs_encryption(self): }, 'EncryptionAtRestOptions': { 'Enabled': True, - 'KmsKeyId': {"Ref": encryption_key_logical_id, } + 'KmsKeyId': { + 'Ref': encryption_key_logical_id, + }, }, }, ) @@ -272,7 +273,5 @@ def test_sandbox_uses_expected_private_subnet(self): 'privateSubnet1', str(import_value), f'OpenSearch should import privateSubnet1, but is importing: {import_value}. ' - 'This is critical for deterministic subnet placement in non-prod environments.' + 'This is critical for deterministic subnet placement in non-prod environments.', ) - - diff --git a/backend/compact-connect/tests/app/test_vpc.py b/backend/compact-connect/tests/app/test_vpc.py index 17ec09c86..b268a7d9e 100644 --- a/backend/compact-connect/tests/app/test_vpc.py +++ b/backend/compact-connect/tests/app/test_vpc.py @@ -218,9 +218,7 @@ def test_explicit_subnet_cidr_blocks(self): private_subnets.append((logical_id, properties)) # Verify we have exactly 3 private subnets - self.assertEqual( - 3, len(private_subnets), f'Expected exactly 3 private subnets, found {len(private_subnets)}' - ) + self.assertEqual(3, len(private_subnets), f'Expected exactly 3 private subnets, found {len(private_subnets)}') # Expected CIDR blocks for the 3 private subnets expected_cidr_blocks = ['10.0.0.0/20', '10.0.16.0/20', '10.0.32.0/20'] From 8c143b4d6111d24a8929510f3607eeddf712e751 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 17:01:17 -0600 Subject: [PATCH 017/137] apply vpc policy to all lambdas --- .../common_constructs/python_function.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/backend/compact-connect/common_constructs/python_function.py b/backend/compact-connect/common_constructs/python_function.py index 625eb60ef..043e3da47 100644 --- a/backend/compact-connect/common_constructs/python_function.py +++ b/backend/compact-connect/common_constructs/python_function.py @@ -5,7 +5,7 @@ from aws_cdk import Duration from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Stats, TreatMissingData from aws_cdk.aws_cloudwatch_actions import SnsAction -from aws_cdk.aws_iam import IRole, Role, ServicePrincipal, ManagedPolicy +from aws_cdk.aws_iam import IRole, ManagedPolicy, Role, ServicePrincipal from aws_cdk.aws_lambda import ILayerVersion, Runtime from aws_cdk.aws_lambda_python_alpha import PythonFunction as CdkPythonFunction from aws_cdk.aws_logs import ILogGroup, LogGroup, RetentionDays @@ -81,23 +81,24 @@ def __init__( assumed_by=ServicePrincipal('lambda.amazonaws.com'), ) log_group.grant_write(role) - if 'vpc' in kwargs: - # if the function is being created in a VPC, add the AWSLambdaVPCAccessExecutionRole policy to the role - role.add_managed_policy( - ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaVPCAccessExecutionRole') - ) - NagSuppressions.add_resource_suppressions( - role, - suppressions=[ - { - 'id': 'AwsSolutions-IAM4', - 'appliesTo': [ - 'Policy::arn::iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole' - ], - 'reason': "Lambdas deployed within a VPC require this policy to access the VPC.", - }, - ], - ) + + if 'vpc' in kwargs: + # if the function is being created in a VPC, add the AWSLambdaVPCAccessExecutionRole policy to the role + role.add_managed_policy( + ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaVPCAccessExecutionRole') + ) + NagSuppressions.add_resource_suppressions( + role, + suppressions=[ + { + 'id': 'AwsSolutions-IAM4', + 'appliesTo': [ + 'Policy::arn::iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole' + ], + 'reason': 'Lambdas deployed within a VPC require this policy to access the VPC.', + }, + ], + ) # We can't directly grant a provided role permission to log to our log group, since that could create a # circular dependency with the stack the role came from. The role creator will have to be responsible for From f411f6d43e53206cc3401b82bd76949487b273a6 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 25 Nov 2025 17:08:09 -0600 Subject: [PATCH 018/137] PR feedback --- .../lambdas/python/common/cc_common/config.py | 2 +- .../python/search/handlers/manage_opensearch_indices.py | 2 +- .../stacks/search_persistent_stack/index_manager.py | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/config.py b/backend/compact-connect/lambdas/python/common/cc_common/config.py index 7ab9b803f..e7a170b71 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/config.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/config.py @@ -32,7 +32,7 @@ def environment_region(self): @property def opensearch_host_endpoint(self): """ - Returns the region name of the region the lambda is running in. + Returns the OpenSearch host endpoint for the domain. """ return os.environ['OPENSEARCH_HOST_ENDPOINT'] diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index b031a7ac2..7e3b298d9 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -8,7 +8,7 @@ class OpenSearchIndexManager(CustomResourceHandler): Custom resource handler to create OpenSearch indices for compacts. """ - def on_create(self) -> None: + def on_create(self, _properties: dict) -> None: """ Create the indices on creation. """ diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index 9785d27bd..0aacc52fc 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -1,6 +1,7 @@ import os from aws_cdk import CustomResource, Duration +from aws_cdk.aws_ec2 import SubnetSelection from aws_cdk.aws_iam import IRole from aws_cdk.aws_logs import LogGroup, RetentionDays from aws_cdk.aws_opensearchservice import Domain @@ -8,7 +9,6 @@ from cdk_nag import NagSuppressions from common_constructs.stack import Stack from constructs import Construct -from moto.ec2.models.subnets import Subnet from common_constructs.python_function import PythonFunction from stacks.vpc_stack import VpcStack @@ -28,7 +28,7 @@ def __init__( construct_id: str, opensearch_domain: Domain, vpc_stack: VpcStack, - vpc_subnets: list[Subnet], + vpc_subnets: SubnetSelection, lambda_role: IRole, ): """ @@ -44,7 +44,6 @@ def __init__( stack = Stack.of(scope) # Create Lambda function for managing OpenSearch indices - # This function is reused across all FeatureFlagResource instances self.manage_function = PythonFunction( self, 'IndexManagerFunction', From a30147c3e4e572b9aeeb3c187c6911f6079f34cc Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 10:40:44 -0600 Subject: [PATCH 019/137] Add needed nag suppressions --- .../search_persistent_stack/__init__.py | 69 +++++++++++++++++++ .../search_persistent_stack/index_manager.py | 4 ++ 2 files changed, 73 insertions(+) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 9b18cd007..9bab7e37e 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -248,6 +248,9 @@ def __init__( self.domain.add_access_policies( opensearch_ingest_access_policy, opensearch_index_manager_access_policy, opensearch_search_api_access_policy ) + # CDK creates a lambda function to manage the access policies, we need to add suppressions for it + self._add_access_policy_lambda_suppressions() + # grant lambda roles access to domain self.domain.grant_read(self.search_api_lambda_role) self.domain.grant_write(self.opensearch_ingest_lambda_role) @@ -268,6 +271,10 @@ def __init__( # Add capacity monitoring alarms for proactive scaling self._add_capacity_alarms(environment_name) + # Add CDK Nag suppressions for Index Manager Custom Resource + self._add_opensearch_lambda_role_suppressions(self.search_api_lambda_role) + self._add_opensearch_lambda_role_suppressions(self.opensearch_ingest_lambda_role) + def _get_capacity_config(self, environment_name: str) -> CapacityConfig: """ Determine OpenSearch cluster capacity configuration based on environment. @@ -503,3 +510,65 @@ def _add_opensearch_suppressions(self, environment_name: str): ], apply_to_children=True, ) + + def _add_access_policy_lambda_suppressions(self): + """ + Add CDK Nag suppressions for the auto-generated Lambda function created by add_access_policies. + + The CDK Domain.add_access_policies() method creates an AwsCustomResource Lambda to manage + the domain's access policy. CDK generates these with IDs starting with 'AWS' followed by a hash. + We find these dynamically to avoid relying on a specific hash value. + """ + # Find auto-generated Lambda constructs by looking for children with IDs starting with 'AWS' + # These are created by CDK's AwsCustomResource for managing domain access policies + for child in self.node.children: + if child.node.id.startswith('AWS'): + NagSuppressions.add_resource_suppressions( + child, + suppressions=[ + { + 'id': 'AwsSolutions-IAM4', + 'appliesTo': [ + 'Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' + ], + 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' + 'OpenSearch domain access policies. It uses the standard execution role.', + }, + { + 'id': 'AwsSolutions-IAM5', + 'appliesTo': ['Action::kms:Describe*', 'Action::kms:List*'], + 'reason': 'This is an AWS-managed custom resource Lambda that requires KMS permissions to ' + 'access the encryption key used by the OpenSearch domain.', + }, + { + 'id': 'HIPAA.Security-LambdaDLQ', + 'reason': 'This is an AWS-managed custom resource Lambda used only during deployment to ' + 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' + 'functions.', + }, + { + 'id': 'HIPAA.Security-LambdaInsideVPC', + 'reason': 'This is an AWS-managed custom resource Lambda that needs internet access to ' + 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' + 'required.', + }, + ], + apply_to_children=True, + ) + + def _add_opensearch_lambda_role_suppressions(self, lambda_role: Role): + """ + Add CDK Nag suppressions for OpenSearch Lambda role configuration. + + param environment_name: The deployment environment name + """ + NagSuppressions.add_resource_suppressions( + lambda_role, + suppressions=[ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The lambda role is used to grant access to the OpenSearch domain.', + }, + ], + apply_to_children=True, + ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index 0aacc52fc..e06c4c0b5 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -122,6 +122,10 @@ def __init__( 'id': 'HIPAA.Security-LambdaDLQ', 'reason': 'This is a synchronous function that runs at deploy time. It does not need a DLQ', }, + { + 'id': 'HIPAA.Security-LambdaInsideVPC', + 'reason': 'Provider framework lambda is managed by AWS and does not function inside a VPC', + }, ], ) From 7bd93447b65931823aaf3f7b276343fed51cecc3 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 10:52:56 -0600 Subject: [PATCH 020/137] Use custom analyzer to support ascii folding in name searches --- .../handlers/manage_opensearch_indices.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index 7e3b298d9..6a4ded9ff 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -100,14 +100,17 @@ def _get_provider_index_mapping(self) -> dict: 'licenseNumber': {'type': 'keyword'}, 'givenName': { 'type': 'text', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'middleName': { 'type': 'text', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'familyName': { 'type': 'text', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'suffix': {'type': 'keyword'}, @@ -118,6 +121,7 @@ def _get_provider_index_mapping(self) -> dict: 'homeAddressStreet2': {'type': 'text'}, 'homeAddressCity': { 'type': 'text', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'homeAddressState': {'type': 'keyword'}, @@ -168,17 +172,22 @@ def _get_provider_index_mapping(self) -> dict: 'settings': { 'index': { 'number_of_shards': 1, - 'number_of_replicas': 0, - 'refresh_interval': '1s', + # no replicas for non-prod envs (since there is only one data node) + # one replica for prod + 'number_of_replicas': 0 if config.environment_name != 'prod' else 1, }, 'analysis': { + # this custom analyzer is recommended by Opensearch when you have international character + # sets, and you want to support searching by their closest ASCII equivalents. + # See https://docs.opensearch.org/latest/analyzers/token-filters/asciifolding/ + 'filter': {'custom_ascii_folding': {'type': 'asciifolding', 'preserve_original': True}}, 'analyzer': { - 'name_analyzer': { + 'custom_ascii_analyzer': { 'type': 'custom', 'tokenizer': 'standard', - 'filter': ['lowercase', 'asciifolding'], + 'filter': ['lowercase', 'custom_ascii_folding'], } - } + }, }, }, 'mappings': { @@ -195,17 +204,17 @@ def _get_provider_index_mapping(self) -> dict: 'npi': {'type': 'keyword'}, 'givenName': { 'type': 'text', - 'analyzer': 'name_analyzer', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'middleName': { 'type': 'text', - 'analyzer': 'name_analyzer', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'familyName': { 'type': 'text', - 'analyzer': 'name_analyzer', + 'analyzer': 'custom_ascii_analyzer', 'fields': {'keyword': {'type': 'keyword', 'ignore_above': 256}}, }, 'suffix': {'type': 'keyword'}, @@ -214,7 +223,7 @@ def _get_provider_index_mapping(self) -> dict: 'jurisdictionUploadedLicenseStatus': {'type': 'keyword'}, 'jurisdictionUploadedCompactEligibility': {'type': 'keyword'}, 'privilegeJurisdictions': {'type': 'keyword'}, - 'providerFamGivMid': {'type': 'text', 'analyzer': 'name_analyzer'}, + 'providerFamGivMid': {'type': 'keyword'}, 'providerDateOfUpdate': {'type': 'date'}, 'birthMonthDay': {'type': 'keyword'}, # Nested arrays From 4078df0a980bcfac15b8946f1175777b7203d6aa Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 11:20:09 -0600 Subject: [PATCH 021/137] Add needed HEAD permission for checking if an index exists --- .../stacks/search_persistent_stack/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 9bab7e37e..a3745bfc5 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -231,6 +231,7 @@ def __init__( principals=[self.opensearch_index_manager_lambda_role], actions=[ 'es:ESHttpGet', + 'es:ESHttpHead', # Required for index_exists() checks 'es:ESHttpPost', 'es:ESHttpPut', ], @@ -532,25 +533,25 @@ def _add_access_policy_lambda_suppressions(self): 'Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' ], 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' - 'OpenSearch domain access policies. It uses the standard execution role.', + 'OpenSearch domain access policies. It uses the standard execution role.', }, { 'id': 'AwsSolutions-IAM5', 'appliesTo': ['Action::kms:Describe*', 'Action::kms:List*'], 'reason': 'This is an AWS-managed custom resource Lambda that requires KMS permissions to ' - 'access the encryption key used by the OpenSearch domain.', + 'access the encryption key used by the OpenSearch domain.', }, { 'id': 'HIPAA.Security-LambdaDLQ', 'reason': 'This is an AWS-managed custom resource Lambda used only during deployment to ' - 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' - 'functions.', + 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' + 'functions.', }, { 'id': 'HIPAA.Security-LambdaInsideVPC', 'reason': 'This is an AWS-managed custom resource Lambda that needs internet access to ' - 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' - 'required.', + 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' + 'required.', }, ], apply_to_children=True, @@ -567,7 +568,8 @@ def _add_opensearch_lambda_role_suppressions(self, lambda_role: Role): suppressions=[ { 'id': 'AwsSolutions-IAM5', - 'reason': 'The lambda role is used to grant access to the OpenSearch domain.', + 'reason': 'This lambda role access is restricted to the specific' + 'OpenSearch domain and its indices within the VPC.', }, ], apply_to_children=True, From 4dc665cc5cf0a3788a24d5b886e247cd8ccdce59 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 11:45:14 -0600 Subject: [PATCH 022/137] Add HEAD permission to access policy for search lambda role --- .../compact-connect/stacks/search_persistent_stack/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index a3745bfc5..293203083 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -242,6 +242,7 @@ def __init__( principals=[self.search_api_lambda_role], actions=[ 'es:ESHttpGet', + 'es:ESHttpHead', ], resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], ) From 6a965ded3f8b87dc607a4ded56c5c14e764b6638 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 11:58:19 -0600 Subject: [PATCH 023/137] PR feedback --- .../python/search/handlers/manage_opensearch_indices.py | 4 ++-- .../search/tests/function/test_manage_opensearch_indices.py | 4 +++- .../stacks/search_persistent_stack/__init__.py | 2 +- .../compact-connect/tests/app/test_search_persistent_stack.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index 6a4ded9ff..bcf1f01f8 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -8,7 +8,7 @@ class OpenSearchIndexManager(CustomResourceHandler): Custom resource handler to create OpenSearch indices for compacts. """ - def on_create(self, _properties: dict) -> None: + def on_create(self, _properties: dict) -> CustomResourceResponse | None: """ Create the indices on creation. """ @@ -20,7 +20,7 @@ def on_create(self, _properties: dict) -> None: index_name = f'compact_{compact}_providers' self._create_provider_index(client, index_name) - def on_update(self, properties: dict) -> None: + def on_update(self, properties: dict) -> CustomResourceResponse | None: """ No-op on update. """ diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index c28011510..d55e4ee30 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -5,7 +5,9 @@ @mock_aws class TestOpenSearchIndexManager(TstFunction): - """Test suite for ManageFeatureFlagHandler custom resource.""" + """Test suite for OpenSearchIndexManager custom resource.""" def setUp(self): super().setUp() + + # TODO - add test cases for checking api calls diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 293203083..1f33e85e8 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -383,7 +383,7 @@ def _add_capacity_alarms(self, environment_name: str): These proactive thresholds give the DevOps team time to plan scaling activities: - Free Storage Space < 50% of allocated capacity - - JVM Memory Pressure > 60% + - JVM Memory Pressure > 70% - CPU Utilization > 60% param environment_name: The deployment environment name diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 62b1d5702..01cf82967 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -172,7 +172,7 @@ def test_capacity_alarms_configured(self): Verifies three critical alarms: 1. Free Storage Space < 50% threshold - 2. JVM Memory Pressure > 60% threshold + 2. JVM Memory Pressure > 70% threshold 3. CPU Utilization > 60% threshold These alarms give DevOps team time to plan scaling activities before hitting limits. From 8f79d4d23c4c7e1d8b1abd2b1fb3b63364ed0438 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 12:00:10 -0600 Subject: [PATCH 024/137] fix commment --- .../compact-connect/stacks/search_persistent_stack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 1f33e85e8..bc06f2278 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -419,7 +419,7 @@ def _add_capacity_alarms(self, environment_name: str): ), ).add_alarm_action(SnsAction(self.alarm_topic)) - # Alarm: JVM Memory Pressure > 60% + # Alarm: JVM Memory Pressure > 70% # Sustained high memory pressure indicates need for instance scaling Alarm( self, From ad4cca82b17b0872dc439c8ec0239aec0318d566 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 15:52:59 -0600 Subject: [PATCH 025/137] Add Search API Stack with resources --- backend/common-cdk/common_constructs/stack.py | 6 + .../data_model/schema/provider/api.py | 32 ++ .../search/handlers/search_providers.py | 110 +++++ .../python/search/opensearch_client.py | 10 + .../compact-connect/pipeline/backend_stage.py | 12 + .../stacks/search_api_stack/__init__.py | 38 ++ .../stacks/search_api_stack/api.py | 41 ++ .../search_api_stack/v1_api/__init__.py | 4 + .../stacks/search_api_stack/v1_api/api.py | 54 +++ .../search_api_stack/v1_api/api_model.py | 457 ++++++++++++++++++ .../v1_api/provider_search.py | 64 +++ .../search_persistent_stack/__init__.py | 12 + .../search_providers_handler.py | 87 ++++ 13 files changed, 927 insertions(+) create mode 100644 backend/compact-connect/lambdas/python/search/handlers/search_providers.py create mode 100644 backend/compact-connect/stacks/search_api_stack/__init__.py create mode 100644 backend/compact-connect/stacks/search_api_stack/api.py create mode 100644 backend/compact-connect/stacks/search_api_stack/v1_api/__init__.py create mode 100644 backend/compact-connect/stacks/search_api_stack/v1_api/api.py create mode 100644 backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py create mode 100644 backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py create mode 100644 backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py diff --git a/backend/common-cdk/common_constructs/stack.py b/backend/common-cdk/common_constructs/stack.py index 652b0383f..74224427c 100644 --- a/backend/common-cdk/common_constructs/stack.py +++ b/backend/common-cdk/common_constructs/stack.py @@ -122,6 +122,12 @@ def state_api_domain_name(self) -> str | None: return f'state-api.{self.hosted_zone.zone_name}' return None + @property + def search_api_domain_name(self) -> str | None: + if self.hosted_zone is not None: + return f'search.{self.hosted_zone.zone_name}' + return None + @property def ui_domain_name(self) -> str | None: if self.hosted_zone is not None: diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index 40deefa06..47016c3b3 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -449,3 +449,35 @@ class StateProviderDetailGeneralResponseSchema(ForgivingSchema): privileges = List(Nested(StatePrivilegeGeneralResponseSchema, required=True, allow_none=False)) providerUIUrl = String(required=True, allow_none=False) + + +class SearchProvidersRequestSchema(CCRequestSchema): + """ + Schema for advanced search providers requests. + + This schema is used to validate incoming requests to the advanced search providers API endpoint. + It accepts an OpenSearch DSL query body for flexible querying of the provider index. + + The request body closely mirrors OpenSearch DSL for pagination using `search_after`. + See: https://docs.opensearch.org/latest/search-plugins/searching-data/paginate/#the-search_after-parameter + + Serialization direction: + API -> load() -> Python + """ + + # The OpenSearch query body - we use Raw to allow the full flexibility of OpenSearch queries + query = Raw(required=True, allow_none=False) + + # Pagination parameters following OpenSearch DSL + # 'from' is a reserved word in Python, so we use 'from_' with data_key='from' + from_ = Integer(required=False, allow_none=False, data_key='from') + size = Integer(required=False, allow_none=False) + + # Sort order - required when using search_after pagination + # Example: [{"providerId": "asc"}, {"dateOfUpdate": "desc"}] + sort = Raw(required=False, allow_none=False) + + # The search_after parameter for cursor-based pagination + # This should be the 'sort' values from the last hit of the previous page + # Example: ["provider-uuid-123", "2024-01-15T10:30:00Z"] + search_after = Raw(required=False, allow_none=False) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search_providers.py b/backend/compact-connect/lambdas/python/search/handlers/search_providers.py new file mode 100644 index 000000000..46cd3b365 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/handlers/search_providers.py @@ -0,0 +1,110 @@ +from aws_lambda_powertools.utilities.typing import LambdaContext +from cc_common.config import logger +from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema, SearchProvidersRequestSchema +from cc_common.exceptions import CCInvalidRequestException +from cc_common.utils import api_handler +from marshmallow import ValidationError + +from opensearch_client import OpenSearchClient + +# Default and maximum page sizes for search results +DEFAULT_SIZE = 10 +MAX_SIZE = 100 + + +@api_handler +def search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument + """ + Search providers using OpenSearch. + + This endpoint accepts an OpenSearch DSL query body and returns sanitized provider records. + Pagination follows OpenSearch DSL using `from`/`size` or `search_after` with `sort`. + + See: https://docs.opensearch.org/latest/search-plugins/searching-data/paginate/ + + :param event: Standard API Gateway event, API schema documented in the CDK ApiStack + :param LambdaContext context: + :return: Dictionary with providers array and pagination metadata + """ + compact = event['pathParameters']['compact'] + + # Parse and validate the request body using the schema + try: + schema = SearchProvidersRequestSchema() + body = schema.loads(event['body']) + except ValidationError as e: + logger.warning('Invalid request body', errors=e.messages) + raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e + + # Build the OpenSearch search body - pass through parameters directly + search_body = { + 'query': body.get('query', {'match_all': {}}), + } + + # Add pagination parameters following OpenSearch DSL + # 'from_' in Python maps to 'from' in the JSON (due to data_key in schema) + from_param = body.get('from_') + if from_param is not None: + search_body['from'] = from_param + + size = body.get('size', DEFAULT_SIZE) + search_body['size'] = min(size, MAX_SIZE) + + # Add sort if provided - required for search_after pagination + sort = body.get('sort') + if sort is not None: + search_body['sort'] = sort + + # Add search_after for cursor-based pagination + search_after = body.get('search_after') + if search_after is not None: + search_body['search_after'] = search_after + # search_after requires sort to be specified + if 'sort' not in search_body: + raise CCInvalidRequestException('sort is required when using search_after pagination') + + # Build the index name for this compact + index_name = f'compact_{compact}_providers' + + logger.info('Executing OpenSearch query', compact=compact, index_name=index_name) + + # Execute the search + client = OpenSearchClient() + response = client.search(index_name=index_name, body=search_body) + + # Extract hits from the response + hits_data = response.get('hits', {}) + hits = hits_data.get('hits', []) + total = hits_data.get('total', {}) + + # Sanitize the provider records using ProviderGeneralResponseSchema + general_schema = ProviderGeneralResponseSchema() + sanitized_providers = [] + last_sort = None + + for hit in hits: + source = hit.get('_source', {}) + try: + sanitized_provider = general_schema.load(source) + sanitized_providers.append(sanitized_provider) + # Track the sort values from the last hit for search_after pagination + last_sort = hit.get('sort') + except ValidationError as e: + # Log the error but continue processing other records + logger.warning( + 'Failed to sanitize provider record', + provider_id=source.get('providerId'), + errors=e.messages, + ) + + # Build response following OpenSearch DSL structure + response_body = { + 'providers': sanitized_providers, + 'total': total, + } + + # Include sort values from last hit to enable search_after pagination + if last_sort is not None: + response_body['lastSort'] = last_sort + + return response_body diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index fdcd30fd9..653d76004 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -21,3 +21,13 @@ def create_index(self, index_name: str, index_mapping: dict) -> None: def index_exists(self, index_name: str) -> bool: return self._client.indices.exists(index=index_name) + + def search(self, index_name: str, body: dict) -> dict: + """ + Execute a search query against the specified index. + + :param index_name: The name of the index to search + :param body: The OpenSearch query body + :return: The search response from OpenSearch + """ + return self._client.search(index=index_name, body=body) diff --git a/backend/compact-connect/pipeline/backend_stage.py b/backend/compact-connect/pipeline/backend_stage.py index d7a828739..f74b8b8a0 100644 --- a/backend/compact-connect/pipeline/backend_stage.py +++ b/backend/compact-connect/pipeline/backend_stage.py @@ -15,6 +15,7 @@ from stacks.persistent_stack import PersistentStack from stacks.provider_users import ProviderUsersStack from stacks.reporting_stack import ReportingStack +from stacks.search_api_stack import SearchApiStack from stacks.search_persistent_stack import SearchPersistentStack from stacks.state_api_stack import StateApiStack from stacks.state_auth import StateAuthStack @@ -246,3 +247,14 @@ def __init__( environment_name=environment_name, vpc_stack=self.vpc_stack, ) + + self.search_api_stack = SearchApiStack( + self, + 'SearchAPIStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + persistent_stack=self.persistent_stack, + search_persistent_stack=self.search_persistent_stack, + ) diff --git a/backend/compact-connect/stacks/search_api_stack/__init__.py b/backend/compact-connect/stacks/search_api_stack/__init__.py new file mode 100644 index 000000000..b3f84e9b5 --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/__init__.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from common_constructs.security_profile import SecurityProfile +from common_constructs.stack import AppStack +from constructs import Construct + +from stacks import persistent_stack, search_persistent_stack + +from .api import SearchApi + + +class SearchApiStack(AppStack): + def __init__( + self, + scope: Construct, + construct_id: str, + *, + environment_name: str, + environment_context: dict, + persistent_stack: persistent_stack.PersistentStack, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + **kwargs, + ): + super().__init__( + scope, construct_id, environment_context=environment_context, environment_name=environment_name, **kwargs + ) + + security_profile = SecurityProfile[environment_context.get('security_profile', 'RECOMMENDED')] + + self.api = SearchApi( + self, + 'SearchApi', + environment_name=environment_name, + security_profile=security_profile, + persistent_stack=persistent_stack, + search_persistent_stack=search_persistent_stack, + domain_name=self.search_api_domain_name, + ) diff --git a/backend/compact-connect/stacks/search_api_stack/api.py b/backend/compact-connect/stacks/search_api_stack/api.py new file mode 100644 index 000000000..ec34ade1a --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/api.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from functools import cached_property + +from constructs import Construct + +from common_constructs.cc_api import CCApi +from stacks import persistent_stack, search_persistent_stack + + +class SearchApi(CCApi): + def __init__( + self, + scope: Construct, + construct_id: str, + *, + persistent_stack: persistent_stack.PersistentStack, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + **kwargs, + ): + super().__init__( + scope, + construct_id, + persistent_stack=persistent_stack, + **kwargs, + ) + from stacks.search_api_stack.v1_api import V1Api + + self.v1_api = V1Api( + self.root, + persistent_stack=persistent_stack, + search_persistent_stack=search_persistent_stack + ) + + @cached_property + def staff_users_authorizer(self): + from aws_cdk.aws_apigateway import CognitoUserPoolsAuthorizer + + return CognitoUserPoolsAuthorizer( + self, 'StaffUsersPoolAuthorizer', cognito_user_pools=[self._persistent_stack.staff_users] + ) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/__init__.py b/backend/compact-connect/stacks/search_api_stack/v1_api/__init__.py new file mode 100644 index 000000000..e14e23d9e --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/__init__.py @@ -0,0 +1,4 @@ +# ruff: noqa: F401 +# We place this import here so it can be referenced by other +# CDK resources +from .api import V1Api diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py new file mode 100644 index 000000000..18e638ee0 --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from aws_cdk.aws_apigateway import AuthorizationType, IResource, MethodOptions + +from stacks import persistent_stack, search_persistent_stack +from stacks.search_api_stack.v1_api.provider_search import ProviderSearch + +from .api_model import ApiModel + + +class V1Api: + """v1 of the State API""" + + def __init__(self, + root: IResource, + persistent_stack: persistent_stack.PersistentStack, + search_persistent_stack: search_persistent_stack.SearchPersistentStack + ): + super().__init__() + from stacks.search_api_stack.api import SearchApi + + self.root = root + self.resource = root.add_resource('v1') + self.api: SearchApi = root.api + self.api_model = ApiModel(api=self.api) + _active_compacts = persistent_stack.get_list_of_compact_abbreviations() + + read_scopes = [] + # set the compact level scopes + for compact in _active_compacts: + # We only set the readGeneral permission scope at the compact level, since users with any permissions + # within a compact are implicitly granted this scope + read_scopes.append(f'{compact}/readGeneral') + + read_auth_method_options = MethodOptions( + authorization_type=AuthorizationType.COGNITO, + authorizer=self.api.staff_users_authorizer, + authorization_scopes=read_scopes, + ) + + # /v1/compacts + self.compacts_resource = self.resource.add_resource('compacts') + # /v1/compacts/{compact} + self.compact_resource = self.compacts_resource.add_resource('{compact}') + + # POST /v1/compacts/{compact}/providers + providers_resource = self.compact_resource.add_resource('providers') + self.provider_management = ProviderSearch( + resource=providers_resource, + method_options=read_auth_method_options, + search_persistent_stack=search_persistent_stack, + api_model=self.api_model, + ) + diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py new file mode 100644 index 000000000..0e2d1b9ab --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py @@ -0,0 +1,457 @@ +# ruff: noqa: SLF001 +# This class initializes the api models for the root api, which we then want to set as protected +# so other classes won't modify it. This is a valid use case for protected access to work with cdk. +from __future__ import annotations + +from aws_cdk.aws_apigateway import JsonSchema, JsonSchemaType, Model +from common_constructs.stack import AppStack + +# Importing module level to allow lazy loading for typing +from common_constructs import cc_api + + +class ApiModel: + """This class is responsible for defining the model definitions used in the Search API endpoints.""" + + def __init__(self, api: cc_api.CCApi): + self.stack: AppStack = AppStack.of(api) + self.api = api + + @property + def search_providers_request_model(self) -> Model: + """ + Return the search providers request model, which should only be created once per API. + + This model closely mirrors OpenSearch DSL for pagination using search_after. + See: https://docs.opensearch.org/latest/search-plugins/searching-data/paginate/ + """ + if hasattr(self.api, '_v1_search_providers_request_model'): + return self.api._v1_search_providers_request_model + self.api._v1_search_providers_request_model = self.api.add_model( + 'V1SearchProvidersRequestModel', + description='Search providers request model following OpenSearch DSL', + schema=JsonSchema( + type=JsonSchemaType.OBJECT, + additional_properties=False, + required=['query'], + properties={ + 'query': JsonSchema( + type=JsonSchemaType.OBJECT, + description='The OpenSearch query body', + ), + 'from': JsonSchema( + type=JsonSchemaType.INTEGER, + minimum=0, + description='Starting document offset for pagination', + ), + 'size': JsonSchema( + type=JsonSchemaType.INTEGER, + minimum=1, + maximum=1000, + description='Number of results to return', + ), + 'sort': JsonSchema( + type=JsonSchemaType.ARRAY, + description='Sort order for results (required for search_after pagination)', + items=JsonSchema(type=JsonSchemaType.OBJECT), + ), + 'search_after': JsonSchema( + type=JsonSchemaType.ARRAY, + description='Sort values from the last hit of the previous page for cursor-based pagination', + ), + }, + ), + ) + return self.api._v1_search_providers_request_model + + @property + def search_providers_response_model(self) -> Model: + """Return the search providers response model, which should only be created once per API""" + if hasattr(self.api, '_v1_search_providers_response_model'): + return self.api._v1_search_providers_response_model + self.api._v1_search_providers_response_model = self.api.add_model( + 'V1SearchProvidersResponseModel', + description='Search providers response model', + schema=JsonSchema( + type=JsonSchemaType.OBJECT, + required=['providers', 'total'], + properties={ + 'providers': JsonSchema( + type=JsonSchemaType.ARRAY, + items=self._providers_response_schema, + ), + 'total': JsonSchema( + type=JsonSchemaType.OBJECT, + description='Total hits information from OpenSearch', + properties={ + 'value': JsonSchema(type=JsonSchemaType.INTEGER), + 'relation': JsonSchema(type=JsonSchemaType.STRING, enum=['eq', 'gte']), + }, + ), + 'lastSort': JsonSchema( + type=JsonSchemaType.ARRAY, + description='Sort values from the last hit to use with search_after for the next page', + ), + }, + ), + ) + return self.api._v1_search_providers_response_model + + @property + def _providers_response_schema(self): + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'type', + 'providerId', + 'givenName', + 'familyName', + 'licenseStatus', + 'compactEligibility', + 'jurisdictionUploadedLicenseStatus', + 'jurisdictionUploadedCompactEligibility', + 'compact', + 'licenseJurisdiction', + 'privilegeJurisdictions', + 'dateOfUpdate', + 'dateOfExpiration', + 'birthMonthDay', + ], + properties={ + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['provider']), + 'providerId': JsonSchema( + type=JsonSchemaType.STRING, + pattern=cc_api.UUID4_FORMAT, + ), + 'givenName': JsonSchema( + type=JsonSchemaType.STRING, + max_length=100, + ), + 'middleName': JsonSchema( + type=JsonSchemaType.STRING, + max_length=100, + ), + 'familyName': JsonSchema( + type=JsonSchemaType.STRING, + max_length=100, + ), + 'suffix': JsonSchema( + type=JsonSchemaType.STRING, + max_length=100, + ), + 'npi': JsonSchema( + type=JsonSchemaType.STRING, + pattern='^[0-9]{10}$', + ), + 'licenseStatus': JsonSchema( + type=JsonSchemaType.STRING, + enum=['active', 'inactive'], + ), + 'compactEligibility': JsonSchema( + type=JsonSchemaType.STRING, + enum=['eligible', 'ineligible'], + ), + 'jurisdictionUploadedLicenseStatus': JsonSchema( + type=JsonSchemaType.STRING, + enum=['active', 'inactive'], + ), + 'jurisdictionUploadedCompactEligibility': JsonSchema( + type=JsonSchemaType.STRING, + enum=['eligible', 'ineligible'], + ), + 'compact': JsonSchema( + type=JsonSchemaType.STRING, + enum=stack.node.get_context('compacts'), + ), + 'licenseJurisdiction': JsonSchema( + type=JsonSchemaType.STRING, + enum=stack.node.get_context('jurisdictions'), + ), + 'currentHomeJurisdiction': JsonSchema( + type=JsonSchemaType.STRING, + enum=stack.node.get_context('jurisdictions'), + ), + 'privilegeJurisdictions': JsonSchema( + type=JsonSchemaType.ARRAY, + items=JsonSchema( + type=JsonSchemaType.STRING, + enum=stack.node.get_context('jurisdictions'), + ), + ), + 'dateOfUpdate': JsonSchema( + type=JsonSchemaType.STRING, + format='date-time', + ), + 'dateOfExpiration': JsonSchema( + type=JsonSchemaType.STRING, + format='date', + ), + 'birthMonthDay': JsonSchema( + type=JsonSchemaType.STRING, + pattern='^[0-1]{1}[0-9]{1}-[0-3]{1}[0-9]{1}', + ), + 'compactConnectRegisteredEmailAddress': JsonSchema( + type=JsonSchemaType.STRING, + format='email', + ), + 'licenses': JsonSchema( + type=JsonSchemaType.ARRAY, + items=self._license_general_response_schema, + ), + 'privileges': JsonSchema( + type=JsonSchemaType.ARRAY, + items=self._privilege_general_response_schema, + ), + 'militaryAffiliations': JsonSchema( + type=JsonSchemaType.ARRAY, + items=self._military_affiliation_general_response_schema, + ), + }, + ) + + @property + def _license_general_response_schema(self): + """ + Schema for LicenseGeneralResponseSchema - license fields visible to staff users + with 'readGeneral' permission. + """ + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'providerId', + 'type', + 'dateOfUpdate', + 'compact', + 'jurisdiction', + 'licenseType', + 'licenseStatus', + 'jurisdictionUploadedLicenseStatus', + 'compactEligibility', + 'jurisdictionUploadedCompactEligibility', + 'givenName', + 'familyName', + 'dateOfIssuance', + 'dateOfExpiration', + 'homeAddressStreet1', + 'homeAddressCity', + 'homeAddressState', + 'homeAddressPostalCode', + ], + properties={ + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['license-home']), + 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'jurisdiction': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions')), + 'licenseType': JsonSchema(type=JsonSchemaType.STRING), + 'licenseStatusName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'licenseStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + 'jurisdictionUploadedLicenseStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + 'compactEligibility': JsonSchema(type=JsonSchemaType.STRING, enum=['eligible', 'ineligible']), + 'jurisdictionUploadedCompactEligibility': JsonSchema( + type=JsonSchemaType.STRING, enum=['eligible', 'ineligible'] + ), + 'npi': JsonSchema(type=JsonSchemaType.STRING, pattern='^[0-9]{10}$'), + 'licenseNumber': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'givenName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'middleName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'familyName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'suffix': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'dateOfIssuance': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfRenewal': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfExpiration': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'homeAddressStreet1': JsonSchema(type=JsonSchemaType.STRING, min_length=2, max_length=100), + 'homeAddressStreet2': JsonSchema(type=JsonSchemaType.STRING, min_length=1, max_length=100), + 'homeAddressCity': JsonSchema(type=JsonSchemaType.STRING, min_length=2, max_length=100), + 'homeAddressState': JsonSchema(type=JsonSchemaType.STRING, min_length=2, max_length=100), + 'homeAddressPostalCode': JsonSchema(type=JsonSchemaType.STRING, min_length=5, max_length=7), + 'emailAddress': JsonSchema(type=JsonSchemaType.STRING, format='email'), + 'phoneNumber': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.PHONE_NUMBER_FORMAT), + 'adverseActions': JsonSchema(type=JsonSchemaType.ARRAY, items=self._adverse_action_general_schema), + 'investigations': JsonSchema(type=JsonSchemaType.ARRAY, items=self._investigation_general_schema), + 'investigationStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['underInvestigation']), + }, + ) + + @property + def _privilege_general_response_schema(self): + """ + Schema for PrivilegeGeneralResponseSchema - privilege fields visible to staff users + with 'readGeneral' permission. + """ + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'type', + 'providerId', + 'compact', + 'jurisdiction', + 'licenseJurisdiction', + 'licenseType', + 'dateOfIssuance', + 'dateOfRenewal', + 'dateOfExpiration', + 'dateOfUpdate', + 'administratorSetStatus', + 'privilegeId', + 'status', + ], + properties={ + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['privilege']), + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'jurisdiction': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions')), + 'licenseJurisdiction': JsonSchema( + type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions') + ), + 'licenseType': JsonSchema(type=JsonSchemaType.STRING), + 'dateOfIssuance': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfRenewal': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfExpiration': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'adverseActions': JsonSchema(type=JsonSchemaType.ARRAY, items=self._adverse_action_general_schema), + 'investigations': JsonSchema(type=JsonSchemaType.ARRAY, items=self._investigation_general_schema), + 'administratorSetStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + 'compactTransactionId': JsonSchema(type=JsonSchemaType.STRING), + 'attestations': JsonSchema( + type=JsonSchemaType.ARRAY, + items=JsonSchema( + type=JsonSchemaType.OBJECT, + required=['attestationId', 'version'], + properties={ + 'attestationId': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'version': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + }, + ), + ), + 'privilegeId': JsonSchema(type=JsonSchemaType.STRING), + 'status': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + 'activeSince': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'investigationStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['underInvestigation']), + }, + ) + + @property + def _military_affiliation_general_response_schema(self): + """ + Schema for MilitaryAffiliationGeneralResponseSchema - military affiliation fields visible + to staff users with 'readGeneral' permission. + """ + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'type', + 'dateOfUpdate', + 'providerId', + 'compact', + 'fileNames', + 'affiliationType', + 'dateOfUpload', + 'status', + ], + properties={ + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['militaryAffiliation']), + 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'fileNames': JsonSchema( + type=JsonSchemaType.ARRAY, + items=JsonSchema(type=JsonSchemaType.STRING), + ), + 'affiliationType': JsonSchema( + type=JsonSchemaType.STRING, enum=['militaryMember', 'militaryMemberSpouse'] + ), + 'dateOfUpload': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'status': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + }, + ) + + @property + def _adverse_action_general_schema(self): + """ + Schema for AdverseActionGeneralResponseSchema - adverse action fields visible + to staff users with 'readGeneral' permission. + """ + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'type', + 'compact', + 'providerId', + 'jurisdiction', + 'licenseTypeAbbreviation', + 'licenseType', + 'actionAgainst', + 'effectiveStartDate', + 'creationDate', + 'adverseActionId', + 'dateOfUpdate', + 'encumbranceType', + 'submittingUser', + ], + properties={ + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['adverseAction']), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'jurisdiction': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions')), + 'licenseTypeAbbreviation': JsonSchema(type=JsonSchemaType.STRING), + 'licenseType': JsonSchema(type=JsonSchemaType.STRING), + 'actionAgainst': JsonSchema(type=JsonSchemaType.STRING, enum=['license', 'privilege']), + 'effectiveStartDate': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'creationDate': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'adverseActionId': JsonSchema(type=JsonSchemaType.STRING), + 'effectiveLiftDate': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'encumbranceType': JsonSchema(type=JsonSchemaType.STRING), + 'clinicalPrivilegeActionCategories': JsonSchema( + type=JsonSchemaType.ARRAY, + items=JsonSchema(type=JsonSchemaType.STRING), + ), + 'liftingUser': JsonSchema(type=JsonSchemaType.STRING), + 'submittingUser': JsonSchema(type=JsonSchemaType.STRING), + }, + ) + + @property + def _investigation_general_schema(self): + """ + Schema for InvestigationGeneralResponseSchema - investigation fields visible + to staff users with 'readGeneral' permission. + """ + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'type', + 'compact', + 'providerId', + 'investigationId', + 'jurisdiction', + 'licenseType', + 'dateOfUpdate', + 'creationDate', + 'submittingUser', + ], + properties={ + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['investigation']), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'investigationId': JsonSchema(type=JsonSchemaType.STRING), + 'jurisdiction': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions')), + 'licenseType': JsonSchema(type=JsonSchemaType.STRING), + 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'creationDate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'submittingUser': JsonSchema(type=JsonSchemaType.STRING), + }, + ) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py new file mode 100644 index 000000000..29f516c1c --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from aws_cdk import Duration +from aws_cdk.aws_apigateway import LambdaIntegration, MethodOptions, MethodResponse, Resource + +from common_constructs.cc_api import CCApi +from stacks import search_persistent_stack + +from .api_model import ApiModel + + +class ProviderSearch: + """ + These endpoints are used by state IT systems to view provider records + """ + + def __init__( + self, + *, + resource: Resource, + method_options: MethodOptions, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + api_model: ApiModel, + ): + super().__init__() + + self.resource = resource + self.api: CCApi = resource.api + self.api_model = api_model + + # Create the nested resources used by endpoints + self.provider_resource = self.resource.add_resource('{providerId}') + + self._add_search_providers( + method_options=method_options, + search_persistent_stack=search_persistent_stack, + ) + + def _add_search_providers( + self, + method_options: MethodOptions, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + ): + search_resource = self.resource.add_resource('search') + + # Get the search providers handler from the search persistent stack + handler = search_persistent_stack.search_providers_handler.handler + + search_resource.add_method( + 'POST', + request_validator=self.api.parameter_body_validator, + request_models={'application/json': self.api_model.search_providers_request_model}, + method_responses=[ + MethodResponse( + status_code='200', + response_models={'application/json': self.api_model.search_providers_response_model}, + ), + ], + integration=LambdaIntegration(handler, timeout=Duration.seconds(29)), + request_parameters={'method.request.header.Authorization': True}, + authorization_type=method_options.authorization_type, + authorizer=method_options.authorizer, + authorization_scopes=method_options.authorization_scopes, + ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index bc06f2278..35f3d60bc 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -22,6 +22,7 @@ from common_constructs.constants import PROD_ENV_NAME from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource +from stacks.search_persistent_stack.search_providers_handler import SearchProvidersHandler from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack PROD_EBS_VOLUME_SIZE = 25 @@ -267,6 +268,17 @@ def __init__( lambda_role=self.opensearch_index_manager_lambda_role, ) + # Create the search providers handler for API Gateway integration + self.search_providers_handler = SearchProvidersHandler( + self, + construct_id='searchProvidersHandler', + opensearch_domain=self.domain, + vpc_stack=vpc_stack, + vpc_subnets=vpc_subnets, + lambda_role=self.search_api_lambda_role, + alarm_topic=self.alarm_topic, + ) + # Add CDK Nag suppressions for OpenSearch Domain self._add_opensearch_suppressions(environment_name) diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py b/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py new file mode 100644 index 000000000..ee6a44a5d --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py @@ -0,0 +1,87 @@ +import os + +from aws_cdk import Duration +from aws_cdk.aws_ec2 import SubnetSelection +from aws_cdk.aws_iam import IRole +from aws_cdk.aws_logs import RetentionDays +from aws_cdk.aws_opensearchservice import Domain +from aws_cdk.aws_sns import ITopic +from cdk_nag import NagSuppressions +from common_constructs.stack import Stack +from constructs import Construct + +from common_constructs.python_function import PythonFunction +from stacks.vpc_stack import VpcStack + + +class SearchProvidersHandler(Construct): + """ + Construct for the Search Providers Lambda function. + + This construct creates the Lambda function that handles search requests + against the OpenSearch domain for provider records. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + opensearch_domain: Domain, + vpc_stack: VpcStack, + vpc_subnets: SubnetSelection, + lambda_role: IRole, + alarm_topic: ITopic, + ): + """ + Initialize the SearchProvidersHandler construct. + + :param scope: The scope of the construct + :param construct_id: The id of the construct + :param opensearch_domain: The reference to the OpenSearch domain resource + :param vpc_stack: The VPC stack + :param vpc_subnets: The VPC subnets for Lambda deployment + :param lambda_role: The IAM role for the Lambda function + :param alarm_topic: The SNS topic for alarms + """ + super().__init__(scope, construct_id) + stack = Stack.of(scope) + + # Create Lambda function for searching providers + self.handler = PythonFunction( + self, + 'SearchProvidersFunction', + description='Search providers handler for OpenSearch queries', + index=os.path.join('handlers', 'search_providers.py'), + lambda_dir='search', + handler='search_providers', + role=lambda_role, + log_retention=RetentionDays.ONE_MONTH, + environment={ + 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, + **stack.common_env_vars, + }, + timeout=Duration.seconds(29), + memory_size=256, + vpc=vpc_stack.vpc, + vpc_subnets=vpc_subnets, + security_groups=[vpc_stack.lambda_security_group], + alarm_topic=alarm_topic, + ) + + # Grant the handler read access to the OpenSearch domain + opensearch_domain.grant_read(self.handler) + + # Add CDK Nag suppressions for the Lambda function's IAM role + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{self.handler.role.node.path}/DefaultPolicy/Resource', + [ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The grant_read method requires wildcard permissions on the OpenSearch domain to ' + 'read from indices. This is appropriate for a search function that needs to query ' + 'provider indices in the domain.', + }, + ], + ) + From 6fff21a87625a36d7a639c98237cc46b19b28987 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 15:55:41 -0600 Subject: [PATCH 026/137] Formatting --- .../python/search/handlers/search_providers.py | 1 - .../compact-connect/stacks/search_api_stack/api.py | 4 +--- .../stacks/search_api_stack/v1_api/api.py | 12 ++++++------ .../stacks/search_api_stack/v1_api/api_model.py | 4 +++- .../search_providers_handler.py | 1 - 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search_providers.py b/backend/compact-connect/lambdas/python/search/handlers/search_providers.py index 46cd3b365..dda3b8deb 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search_providers.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search_providers.py @@ -4,7 +4,6 @@ from cc_common.exceptions import CCInvalidRequestException from cc_common.utils import api_handler from marshmallow import ValidationError - from opensearch_client import OpenSearchClient # Default and maximum page sizes for search results diff --git a/backend/compact-connect/stacks/search_api_stack/api.py b/backend/compact-connect/stacks/search_api_stack/api.py index ec34ade1a..bb67e6def 100644 --- a/backend/compact-connect/stacks/search_api_stack/api.py +++ b/backend/compact-connect/stacks/search_api_stack/api.py @@ -27,9 +27,7 @@ def __init__( from stacks.search_api_stack.v1_api import V1Api self.v1_api = V1Api( - self.root, - persistent_stack=persistent_stack, - search_persistent_stack=search_persistent_stack + self.root, persistent_stack=persistent_stack, search_persistent_stack=search_persistent_stack ) @cached_property diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py index 18e638ee0..9d063f351 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py @@ -11,11 +11,12 @@ class V1Api: """v1 of the State API""" - def __init__(self, - root: IResource, - persistent_stack: persistent_stack.PersistentStack, - search_persistent_stack: search_persistent_stack.SearchPersistentStack - ): + def __init__( + self, + root: IResource, + persistent_stack: persistent_stack.PersistentStack, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + ): super().__init__() from stacks.search_api_stack.api import SearchApi @@ -51,4 +52,3 @@ def __init__(self, search_persistent_stack=search_persistent_stack, api_model=self.api_model, ) - diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py index 0e2d1b9ab..c13b540fc 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py @@ -250,7 +250,9 @@ def _license_general_response_schema(self): 'licenseType': JsonSchema(type=JsonSchemaType.STRING), 'licenseStatusName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), 'licenseStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), - 'jurisdictionUploadedLicenseStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + 'jurisdictionUploadedLicenseStatus': JsonSchema( + type=JsonSchemaType.STRING, enum=['active', 'inactive'] + ), 'compactEligibility': JsonSchema(type=JsonSchemaType.STRING, enum=['eligible', 'ineligible']), 'jurisdictionUploadedCompactEligibility': JsonSchema( type=JsonSchemaType.STRING, enum=['eligible', 'ineligible'] diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py b/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py index ee6a44a5d..d38a1af9e 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py @@ -84,4 +84,3 @@ def __init__( }, ], ) - From 91236e58197232ab765a86d7cfd78da3643bbbe7 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 1 Dec 2025 17:25:54 -0600 Subject: [PATCH 027/137] WIP - add lambda to index provider documents --- .../handlers/index_provider_documents.py | 0 .../python/search/opensearch_client.py | 29 +++++ .../compact-connect/pipeline/backend_stage.py | 1 + .../search_persistent_stack/__init__.py | 17 +++ .../populate_provider_documents_handler.py | 100 ++++++++++++++++++ 5 files changed, 147 insertions(+) create mode 100644 backend/compact-connect/lambdas/python/search/handlers/index_provider_documents.py create mode 100644 backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py diff --git a/backend/compact-connect/lambdas/python/search/handlers/index_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/index_provider_documents.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 653d76004..51c831847 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -31,3 +31,32 @@ def search(self, index_name: str, body: dict) -> dict: :return: The search response from OpenSearch """ return self._client.search(index=index_name, body=body) + + def index_document(self, index_name: str, document_id: str, document: dict) -> dict: + """ + Index a single document into the specified index. + + :param index_name: The name of the index to write to + :param document_id: The unique identifier for the document + :param document: The document to index + :return: The response from OpenSearch + """ + return self._client.index(index=index_name, id=document_id, body=document) + + def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'providerId') -> dict: + """ + Bulk index multiple documents into the specified index. + + :param index_name: The name of the index to write to + :param documents: List of documents to index + :param id_field: The field name to use as the document ID (default: 'providerId') + :return: The bulk response from OpenSearch + """ + if not documents: + return {'items': [], 'errors': False} + + actions = [] + for doc in documents: + actions.append({'index': {'_index': index_name, '_id': doc[id_field]}}) + actions.append(doc) + return self._client.bulk(body=actions) diff --git a/backend/compact-connect/pipeline/backend_stage.py b/backend/compact-connect/pipeline/backend_stage.py index f74b8b8a0..a6aff227e 100644 --- a/backend/compact-connect/pipeline/backend_stage.py +++ b/backend/compact-connect/pipeline/backend_stage.py @@ -246,6 +246,7 @@ def __init__( standard_tags=standard_tags, environment_name=environment_name, vpc_stack=self.vpc_stack, + persistent_stack=self.persistent_stack, ) self.search_api_stack = SearchApiStack( diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 35f3d60bc..277a077fa 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -21,7 +21,9 @@ from constructs import Construct from common_constructs.constants import PROD_ENV_NAME +from stacks.persistent_stack import PersistentStack from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource +from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler from stacks.search_persistent_stack.search_providers_handler import SearchProvidersHandler from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack @@ -55,6 +57,7 @@ def __init__( environment_name: str, environment_context: dict, vpc_stack: VpcStack, + persistent_stack: PersistentStack, **kwargs, ): super().__init__( @@ -279,6 +282,20 @@ def __init__( alarm_topic=self.alarm_topic, ) + # Create the populate provider documents handler for manual invocation + # This handler is used to bulk index provider documents from DynamoDB into OpenSearch + self.populate_provider_documents_handler = PopulateProviderDocumentsHandler( + self, + construct_id='populateProviderDocumentsHandler', + opensearch_domain=self.domain, + vpc_stack=vpc_stack, + vpc_subnets=vpc_subnets, + lambda_role=self.opensearch_ingest_lambda_role, + provider_table=persistent_stack.provider_table, + provider_date_of_update_index_name=persistent_stack.provider_table.provider_date_of_update_index_name, + alarm_topic=self.alarm_topic, + ) + # Add CDK Nag suppressions for OpenSearch Domain self._add_opensearch_suppressions(environment_name) diff --git a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py new file mode 100644 index 000000000..1a6824e93 --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py @@ -0,0 +1,100 @@ +import os + +from aws_cdk import Duration +from aws_cdk.aws_dynamodb import ITable +from aws_cdk.aws_ec2 import SubnetSelection +from aws_cdk.aws_iam import IRole +from aws_cdk.aws_logs import RetentionDays +from aws_cdk.aws_opensearchservice import Domain +from aws_cdk.aws_sns import ITopic +from cdk_nag import NagSuppressions +from common_constructs.stack import Stack +from constructs import Construct + +from common_constructs.python_function import PythonFunction +from stacks.vpc_stack import VpcStack + + +class PopulateProviderDocumentsHandler(Construct): + """ + Construct for the Populate Provider Documents Lambda function. + + This construct creates the Lambda function that populates the OpenSearch + indices with provider documents by scanning the provider table and + bulk indexing the sanitized records. + + This Lambda is intended to be invoked manually through the AWS console + for initial data population or re-indexing operations. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + opensearch_domain: Domain, + vpc_stack: VpcStack, + vpc_subnets: SubnetSelection, + lambda_role: IRole, + provider_table: ITable, + alarm_topic: ITopic, + ): + """ + Initialize the PopulateProviderDocumentsHandler construct. + + :param scope: The scope of the construct + :param construct_id: The id of the construct + :param opensearch_domain: The reference to the OpenSearch domain resource + :param vpc_stack: The VPC stack + :param vpc_subnets: The VPC subnets for Lambda deployment + :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) + :param provider_table: The DynamoDB provider table + :param provider_date_of_update_index_name: The name of the providerDateOfUpdate GSI + :param alarm_topic: The SNS topic for alarms + """ + super().__init__(scope, construct_id) + stack = Stack.of(scope) + + # Create Lambda function for populating provider documents + self.handler = PythonFunction( + self, + 'PopulateProviderDocumentsFunction', + description='Populates OpenSearch indices with provider documents from DynamoDB', + index=os.path.join('handlers', 'populate_provider_documents.py'), + lambda_dir='search', + handler='populate_provider_documents', + role=lambda_role, + log_retention=RetentionDays.ONE_MONTH, + environment={ + 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, + 'PROVIDER_TABLE_NAME': provider_table.table_name, + 'PROV_DATE_OF_UPDATE_INDEX_NAME': provider_table.provider_date_of_update_index_name, + **stack.common_env_vars, + }, + # Longer timeout for processing large datasets + timeout=Duration.minutes(15), + memory_size=512, + vpc=vpc_stack.vpc, + vpc_subnets=vpc_subnets, + security_groups=[vpc_stack.lambda_security_group], + alarm_topic=alarm_topic, + ) + + # Grant the handler write access to the OpenSearch domain + opensearch_domain.grant_write(self.handler) + + # Grant the handler read access to the provider table + provider_table.grant_read_data(self.handler) + + # Add CDK Nag suppressions for the Lambda function's IAM role + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{self.handler.role.node.path}/DefaultPolicy/Resource', + [ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The grant_write method requires wildcard permissions on the OpenSearch domain to ' + 'write to indices. This is appropriate for a function that needs to bulk index ' + 'provider documents. The DynamoDB grant_read_data also requires index permissions.', + }, + ], + ) From fc2025742176b465a1a0c57b8744ee4d73acd633 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 10:15:38 -0600 Subject: [PATCH 028/137] add lambda runtime logic to index provider documents --- .../data_model/schema/provider/api.py | 1 - .../python/provider-data-v1/tests/__init__.py | 5 + .../handlers/populate_provider_documents.py | 241 ++++++++++++++++++ .../lambdas/python/search/tests/__init__.py | 17 ++ .../python/search/tests/function/__init__.py | 69 +++++ .../test_populate_provider_documents.py | 166 ++++++++++++ 6 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py create mode 100644 backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index 47016c3b3..9561cb078 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -131,7 +131,6 @@ class ProviderGeneralResponseSchema(ForgivingSchema): familyName = String(required=True, allow_none=False, validate=Length(1, 100)) suffix = String(required=False, allow_none=False, validate=Length(1, 100)) # This date is determined by the license records uploaded by a state - # they do not include a timestamp, so we use the Date field type dateOfExpiration = Raw(required=True, allow_none=False) compactConnectRegisteredEmailAddress = Email(required=False, allow_none=False) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/__init__.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/__init__.py index a6e1f1cc9..08a9c0b79 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/tests/__init__.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/tests/__init__.py @@ -97,6 +97,11 @@ def setUpClass(cls): {'name': 'audiologist', 'abbreviation': 'aud'}, {'name': 'speech-language pathologist', 'abbreviation': 'slp'}, ], + 'octp': [ + {'name': 'occupational therapist', 'abbreviation': 'ot'}, + {'name': 'occupational therapy assistant', 'abbreviation': 'ota'}, + ], + 'coun': [{'name': 'licensed professional counselor', 'abbreviation': 'lpc'}], }, ), }, diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py new file mode 100644 index 000000000..cac2dfb78 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -0,0 +1,241 @@ +""" +Lambda handler to populate OpenSearch with provider documents. + +This Lambda scans the provider table using the providerDateOfUpdate GSI, +retrieves complete provider records, sanitizes them, and bulk indexes them +into OpenSearch. + +This Lambda is intended to be invoked manually through the AWS console for +initial data population or re-indexing operations. +""" + +from aws_lambda_powertools.utilities.typing import LambdaContext +from cc_common.config import config, logger +from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema +from cc_common.exceptions import CCNotFoundException +from marshmallow import ValidationError +from opensearch_client import OpenSearchClient + +# Batch size for DynamoDB pagination +DYNAMODB_PAGE_SIZE = 1000 +# Batch size for OpenSearch bulk indexing +OPENSEARCH_BULK_SIZE = 100 + + +def populate_provider_documents(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument + """ + Populate OpenSearch indices with provider documents. + + This function scans all providers in the provider table using the providerDateOfUpdate GSI, + retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, + and bulk indexes them into the appropriate compact-specific OpenSearch index. + + :param event: Lambda event (not used, but required for Lambda signature) + :param context: Lambda context + :return: Summary of indexing operation + """ + data_client = config.data_client + opensearch_client = OpenSearchClient() + + # Track statistics + stats = { + 'total_providers_processed': 0, + 'total_providers_indexed': 0, + 'total_providers_failed': 0, + 'compacts_processed': [], + 'errors': [], + } + + for compact in config.compacts: + logger.info('Processing compact', compact=compact) + index_name = f'compact_{compact}_providers' + + documents_to_index = [] + compact_stats = { + 'providers_processed': 0, + 'providers_indexed': 0, + 'providers_failed': 0, + } + + # Track pagination state + last_key = None + has_more = True + + while has_more: + # Build pagination parameters + dynamo_pagination = {'pageSize': DYNAMODB_PAGE_SIZE} + if last_key: + dynamo_pagination['lastKey'] = last_key + + # Query providers from the GSI + try: + result = data_client.get_providers_sorted_by_updated( + compact=compact, + scan_forward=True, + pagination=dynamo_pagination, + ) + except Exception as e: + logger.exception('Failed to query providers from GSI', compact=compact, error=str(e)) + stats['errors'].append({'compact': compact, 'error': f'GSI query failed: {str(e)}'}) + break + + providers = result.get('items', []) + last_key = result.get('pagination', {}).get('lastKey') + has_more = last_key is not None + + logger.info( + 'Retrieved providers batch', + compact=compact, + batch_size=len(providers), + has_more=has_more, + ) + + # Process each provider in the batch + for provider_record in providers: + compact_stats['providers_processed'] += 1 + provider_id = provider_record.get('providerId') + + if not provider_id: + logger.warning('Provider record missing providerId', record=provider_record) + compact_stats['providers_failed'] += 1 + continue + + try: + # Get complete provider records + provider_user_records = data_client.get_provider_user_records( + compact=compact, + provider_id=provider_id, + consistent_read=False, # Eventual consistency is fine for indexing + ) + + # Generate API response object with all nested records + api_response = provider_user_records.generate_api_response_object() + + # Sanitize using ProviderGeneralResponseSchema + schema = ProviderGeneralResponseSchema() + sanitized_document = schema.load(api_response) + documents_to_index.append(sanitized_document) + + except CCNotFoundException: + logger.warning('Provider not found when fetching records', provider_id=provider_id, compact=compact) + compact_stats['providers_failed'] += 1 + continue + except ValidationError as e: + logger.warning( + 'Failed to sanitize provider record', + provider_id=provider_id, + compact=compact, + errors=e.messages, + ) + compact_stats['providers_failed'] += 1 + continue + except Exception as e: + logger.exception( + 'Unexpected error processing provider', + provider_id=provider_id, + compact=compact, + error=str(e), + ) + compact_stats['providers_failed'] += 1 + continue + + # Bulk index when batch is full + if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index, stats) + compact_stats['providers_indexed'] += indexed_count + documents_to_index = [] + + # Index any remaining documents for this compact + if documents_to_index: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index, stats) + compact_stats['providers_indexed'] += indexed_count + + # Update overall stats + stats['total_providers_processed'] += compact_stats['providers_processed'] + stats['total_providers_indexed'] += compact_stats['providers_indexed'] + stats['total_providers_failed'] += compact_stats['providers_failed'] + stats['compacts_processed'].append( + { + 'compact': compact, + **compact_stats, + } + ) + + logger.info( + 'Completed processing compact', + compact=compact, + providers_processed=compact_stats['providers_processed'], + providers_indexed=compact_stats['providers_indexed'], + providers_failed=compact_stats['providers_failed'], + ) + + logger.info( + 'Completed populating provider documents', + total_providers_processed=stats['total_providers_processed'], + total_providers_indexed=stats['total_providers_indexed'], + total_providers_failed=stats['total_providers_failed'], + ) + + return stats + + +def _bulk_index_documents( + opensearch_client: OpenSearchClient, index_name: str, documents: list[dict], stats: dict +) -> int: + """ + Bulk index documents into OpenSearch. + + :param opensearch_client: The OpenSearch client + :param index_name: The index to write to + :param documents: List of documents to index + :param stats: Statistics dictionary to update with errors + :return: Number of successfully indexed documents + """ + if not documents: + return 0 + + try: + response = opensearch_client.bulk_index(index_name=index_name, documents=documents) + + # Check for errors in the bulk response + if response.get('errors'): + error_count = 0 + for item in response.get('items', []): + index_result = item.get('index', {}) + if index_result.get('error'): + error_count += 1 + logger.warning( + 'Bulk index item error', + document_id=index_result.get('_id'), + error=index_result.get('error'), + ) + logger.warning( + 'Bulk index completed with errors', + index_name=index_name, + total_documents=len(documents), + error_count=error_count, + ) + return len(documents) - error_count + + logger.info( + 'Indexed documents', + index_name=index_name, + document_count=len(documents), + ) + return len(documents) + + except Exception as e: + logger.exception( + 'Failed to bulk index documents', + index_name=index_name, + document_count=len(documents), + error=str(e), + ) + stats['errors'].append( + { + 'index': index_name, + 'error': f'Bulk index failed: {str(e)}', + 'document_count': len(documents), + } + ) + return 0 diff --git a/backend/compact-connect/lambdas/python/search/tests/__init__.py b/backend/compact-connect/lambdas/python/search/tests/__init__.py index 214a66839..6b11fbe2f 100644 --- a/backend/compact-connect/lambdas/python/search/tests/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/__init__.py @@ -18,6 +18,10 @@ def setUpClass(cls): 'AWS_REGION': 'us-east-1', 'ENVIRONMENT_NAME': 'test', 'COMPACTS': '["aslp", "octp", "coun"]', + 'PROVIDER_TABLE_NAME': 'provider-table', + 'PROV_DATE_OF_UPDATE_INDEX_NAME': 'providerDateOfUpdate', + 'PROV_FAM_GIV_MID_INDEX_NAME': 'providerFamGivMid', + 'LICENSE_GSI_NAME': 'licenseGSI', 'OPENSEARCH_HOST_ENDPOINT': 'vpc-providersearchd-5bzuqxhpxffk-w6dkpddu.us-east-1.es.amazonaws.com', 'JURISDICTIONS': json.dumps( [ @@ -76,6 +80,19 @@ def setUpClass(cls): 'wy', ] ), + 'LICENSE_TYPES': json.dumps( + { + 'aslp': [ + {'name': 'audiologist', 'abbreviation': 'aud'}, + {'name': 'speech-language pathologist', 'abbreviation': 'slp'}, + ], + 'octp': [ + {'name': 'occupational therapist', 'abbreviation': 'ot'}, + {'name': 'occupational therapy assistant', 'abbreviation': 'ota'}, + ], + 'coun': [{'name': 'licensed professional counselor', 'abbreviation': 'lpc'}], + }, + ), }, ) # Monkey-patch config object to be sure we have it based diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py index 1d127059a..0703f14a6 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -1,3 +1,6 @@ +import os + +import boto3 from moto import mock_aws from tests import TstLambdas @@ -9,8 +12,74 @@ class TstFunction(TstLambdas): def setUp(self): # noqa: N801 invalid-name super().setUp() + # we want to see any diffs in failed tests, regardless of how large the object is + self.maxDiff = None + + self.build_resources() # This must be imported within the tests, since they import modules which require # environment variables that are not set until the TstLambdas class is initialized from common_test.test_data_generator import TestDataGenerator self.test_data_generator = TestDataGenerator + + self.addCleanup(self.delete_resources) + + def build_resources(self): + self.create_provider_table() + + def delete_resources(self): + self._provider_table.delete() + + def create_provider_table(self): + self._provider_table = boto3.resource('dynamodb').create_table( + AttributeDefinitions=[ + {'AttributeName': 'pk', 'AttributeType': 'S'}, + {'AttributeName': 'sk', 'AttributeType': 'S'}, + {'AttributeName': 'providerFamGivMid', 'AttributeType': 'S'}, + {'AttributeName': 'providerDateOfUpdate', 'AttributeType': 'S'}, + {'AttributeName': 'licenseGSIPK', 'AttributeType': 'S'}, + {'AttributeName': 'licenseGSISK', 'AttributeType': 'S'}, + {'AttributeName': 'licenseUploadDateGSIPK', 'AttributeType': 'S'}, + {'AttributeName': 'licenseUploadDateGSISK', 'AttributeType': 'S'}, + ], + TableName=os.environ['PROVIDER_TABLE_NAME'], + KeySchema=[{'AttributeName': 'pk', 'KeyType': 'HASH'}, {'AttributeName': 'sk', 'KeyType': 'RANGE'}], + BillingMode='PAY_PER_REQUEST', + GlobalSecondaryIndexes=[ + { + 'IndexName': os.environ['PROV_FAM_GIV_MID_INDEX_NAME'], + 'KeySchema': [ + {'AttributeName': 'sk', 'KeyType': 'HASH'}, + {'AttributeName': 'providerFamGivMid', 'KeyType': 'RANGE'}, + ], + 'Projection': {'ProjectionType': 'ALL'}, + }, + { + 'IndexName': os.environ['PROV_DATE_OF_UPDATE_INDEX_NAME'], + 'KeySchema': [ + {'AttributeName': 'sk', 'KeyType': 'HASH'}, + {'AttributeName': 'providerDateOfUpdate', 'KeyType': 'RANGE'}, + ], + 'Projection': {'ProjectionType': 'ALL'}, + }, + { + 'IndexName': os.environ['LICENSE_GSI_NAME'], + 'KeySchema': [ + {'AttributeName': 'licenseGSIPK', 'KeyType': 'HASH'}, + {'AttributeName': 'licenseGSISK', 'KeyType': 'RANGE'}, + ], + 'Projection': {'ProjectionType': 'ALL'}, + }, + { + 'IndexName': 'licenseUploadDateGSI', + 'KeySchema': [ + {'AttributeName': 'licenseUploadDateGSIPK', 'KeyType': 'HASH'}, + {'AttributeName': 'licenseUploadDateGSISK', 'KeyType': 'RANGE'}, + ], + 'Projection': { + 'ProjectionType': 'INCLUDE', + 'NonKeyAttributes': ['providerId'], + }, + }, + ], + ) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py new file mode 100644 index 000000000..1cd01b9f9 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -0,0 +1,166 @@ +from datetime import date, datetime, timedelta, timezone +from unittest.mock import Mock, call, patch +from uuid import UUID + +from common_test.test_constants import ( + DEFAULT_LICENSE_EXPIRATION_DATE, + DEFAULT_LICENSE_ISSUANCE_DATE, + DEFAULT_LICENSE_RENEWAL_DATE, + DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, + DEFAULT_PROVIDER_UPDATE_DATETIME, + DEFAULT_REGISTERED_EMAIL_ADDRESS, +) +from moto import mock_aws + +from . import TstFunction + +MOCK_ASLP_PROVIDER_ID = '00000000-0000-0000-0000-000000000001' +MOCK_OCTP_PROVIDER_ID = '00000000-0000-0000-0000-000000000002' +MOCK_COUN_PROVIDER_ID = '00000000-0000-0000-0000-000000000003' + +test_license_type_mapping = { + 'aslp': 'audiologist', + 'octp': 'occupational therapist', + 'coun': 'licensed professional counselor', +} +test_provider_id_mapping = { + 'aslp': MOCK_ASLP_PROVIDER_ID, + 'octp': MOCK_OCTP_PROVIDER_ID, + 'coun': MOCK_COUN_PROVIDER_ID, +} + + +@mock_aws +class TestPopulateProviderDocuments(TstFunction): + """Test suite for OpenSearchIndexManager custom resource.""" + + def setUp(self): + super().setUp() + + def _put_test_provider_and_license_record_in_dynamodb_table(self, compact): + self.test_data_generator.put_default_provider_record_in_provider_table( + value_overrides={ + 'compact': compact, + 'providerId': test_provider_id_mapping[compact], + 'givenName': f'test{compact}GivenName', + 'familyName': f'test{compact}FamilyName', + }, + date_of_update_override=DEFAULT_PROVIDER_UPDATE_DATETIME, + ) + self.test_data_generator.put_default_license_record_in_provider_table( + value_overrides={ + 'compact': compact, + 'providerId': test_provider_id_mapping[compact], + 'givenName': f'test{compact}GivenName', + 'familyName': f'test{compact}FamilyName', + 'licenseType': test_license_type_mapping[compact], + }, + date_of_update_override=DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, + ) + + def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_index_response: dict = None): + if not bulk_index_response: + bulk_index_response = {'items': [], 'errors': False} + + # Create a mock instance that will be returned by the OpenSearchClient constructor + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_index.return_value = bulk_index_response + return mock_client_instance + + def _generate_expected_call_for_document(self, compact): + # Use timezone(timedelta(0), '+0000') to match how the code creates UTC timezone + utc_tz = timezone(timedelta(0), '+0000') + return call( + index_name=f'compact_{compact}_providers', + documents=[ + { + 'providerId': UUID(test_provider_id_mapping[compact]), + 'type': 'provider', + 'dateOfUpdate': datetime.fromisoformat(DEFAULT_PROVIDER_UPDATE_DATETIME).replace(tzinfo=utc_tz), + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'currentHomeJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'compactEligibility': 'ineligible', + 'npi': '0608337260', + 'givenName': f'test{compact}GivenName', + 'middleName': 'Gunnar', + 'familyName': f'test{compact}FamilyName', + 'dateOfExpiration': date.fromisoformat(DEFAULT_LICENSE_EXPIRATION_DATE), + 'compactConnectRegisteredEmailAddress': DEFAULT_REGISTERED_EMAIL_ADDRESS, + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'privilegeJurisdictions': {'ne'}, + 'birthMonthDay': '06-06', + 'licenses': [ + { + 'providerId': UUID(test_provider_id_mapping[compact]), + 'type': 'license', + 'dateOfUpdate': datetime.fromisoformat(DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE).replace( + tzinfo=utc_tz + ), + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': test_license_type_mapping[compact], + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseStatus': 'inactive', + 'jurisdictionUploadedLicenseStatus': 'active', + 'compactEligibility': 'ineligible', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'npi': '0608337260', + 'licenseNumber': 'A0608337260', + 'givenName': f'test{compact}GivenName', + 'middleName': 'Gunnar', + 'familyName': f'test{compact}FamilyName', + 'dateOfIssuance': date.fromisoformat(DEFAULT_LICENSE_ISSUANCE_DATE), + 'dateOfRenewal': date.fromisoformat(DEFAULT_LICENSE_RENEWAL_DATE), + 'dateOfExpiration': date.fromisoformat(DEFAULT_LICENSE_EXPIRATION_DATE), + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'homeAddressCity': 'Columbus', + 'homeAddressState': 'oh', + 'homeAddressPostalCode': '43004', + 'emailAddress': 'björk@example.com', + 'phoneNumber': '+13213214321', + 'adverseActions': [], + 'investigations': [], + } + ], + 'privileges': [], + 'militaryAffiliations': [], + } + ], + ) + + @patch('handlers.populate_provider_documents.OpenSearchClient') + def test_provider_records_from_all_three_compacts_are_indexed_in_expected_index(self, mock_opensearch_client): + from handlers.populate_provider_documents import populate_provider_documents + + # Set up the mock opensearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + compacts = ['aslp', 'octp', 'coun'] + # add a provider and license record for each of the three compacts + for compact in compacts: + self._put_test_provider_and_license_record_in_dynamodb_table(compact) + + # now run the handler + result = populate_provider_documents({}, self.mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk indexing was called once for each compact (3 times total) + self.assertEqual(3, mock_client_instance.bulk_index.call_count) + + # Get all calls to bulk_index and verify each compact was indexed + bulk_index_calls = mock_client_instance.bulk_index.call_args_list + self.assertEqual(self._generate_expected_call_for_document('aslp'), bulk_index_calls[0]) + self.assertEqual(self._generate_expected_call_for_document('octp'), bulk_index_calls[1]) + self.assertEqual(self._generate_expected_call_for_document('coun'), bulk_index_calls[2]) + + # Verify the result statistics + self.assertEqual(3, result['total_providers_processed']) + self.assertEqual(3, result['total_providers_indexed']) + self.assertEqual(0, result['total_providers_failed']) From 7bfafe7dfe16deaae84a8008660d01b718ea9f07 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 10:36:02 -0600 Subject: [PATCH 029/137] PR feedback --- .../handlers/populate_provider_documents.py | 15 +++++---------- .../function/test_populate_provider_documents.py | 2 +- .../stacks/search_api_stack/v1_api/api_model.py | 4 +++- .../stacks/search_persistent_stack/__init__.py | 15 +++++++-------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index cac2dfb78..6c449fa2d 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -68,16 +68,11 @@ def populate_provider_documents(event: dict, context: LambdaContext): # noqa: A dynamo_pagination['lastKey'] = last_key # Query providers from the GSI - try: - result = data_client.get_providers_sorted_by_updated( - compact=compact, - scan_forward=True, - pagination=dynamo_pagination, - ) - except Exception as e: - logger.exception('Failed to query providers from GSI', compact=compact, error=str(e)) - stats['errors'].append({'compact': compact, 'error': f'GSI query failed: {str(e)}'}) - break + result = data_client.get_providers_sorted_by_updated( + compact=compact, + scan_forward=True, + pagination=dynamo_pagination, + ) providers = result.get('items', []) last_key = result.get('pagination', {}).get('lastKey') diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 1cd01b9f9..7f6d3d3de 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -32,7 +32,7 @@ @mock_aws class TestPopulateProviderDocuments(TstFunction): - """Test suite for OpenSearchIndexManager custom resource.""" + """Test suite for populate provider documents handler.""" def setUp(self): super().setUp() diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py index c13b540fc..37f65ba21 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py @@ -47,7 +47,9 @@ def search_providers_request_model(self) -> Model: 'size': JsonSchema( type=JsonSchemaType.INTEGER, minimum=1, - maximum=1000, + # setting low limit for now, as this search endpoint is only used by the UI client, + # and we don't anticipate needing to support more than 100 records per request + maximum=100, description='Number of results to return', ), 'sort': JsonSchema( diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 277a077fa..94c1e2aef 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -292,7 +292,6 @@ def __init__( vpc_subnets=vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, - provider_date_of_update_index_name=persistent_stack.provider_table.provider_date_of_update_index_name, alarm_topic=self.alarm_topic, ) @@ -508,16 +507,16 @@ def _add_opensearch_suppressions(self, environment_name: str): suppressions=[ { 'id': 'AwsSolutions-OS3', - 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups.' - 'The data in the domain is only accessible by the ingest lambda which indexes the' - 'documents and the search API lambda which can only be accessed by authenticated staff' + 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups. ' + 'The data in the domain is only accessible by the ingest lambda which indexes the ' + 'documents and the search API lambda which can only be accessed by authenticated staff ' 'users in CompactConnect.', }, { 'id': 'AwsSolutions-OS5', - 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups.' - 'The data in the domain is only accessible by the ingest lambda which indexes the' - 'documents and the search API lambda which can only be accessed by authenticated staff' + 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups. ' + 'The data in the domain is only accessible by the ingest lambda which indexes the ' + 'documents and the search API lambda which can only be accessed by authenticated staff ' 'users in CompactConnect.', }, ], @@ -598,7 +597,7 @@ def _add_opensearch_lambda_role_suppressions(self, lambda_role: Role): suppressions=[ { 'id': 'AwsSolutions-IAM5', - 'reason': 'This lambda role access is restricted to the specific' + 'reason': 'This lambda role access is restricted to the specific ' 'OpenSearch domain and its indices within the VPC.', }, ], From 391d57bd352a29fe13cd8aeb15b347dc93db87a0 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 12:31:17 -0600 Subject: [PATCH 030/137] serialize provider documents before indexing --- .../handlers/populate_provider_documents.py | 9 +++++++- .../test_populate_provider_documents.py | 23 ++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 6c449fa2d..a9b28467c 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -9,10 +9,13 @@ initial data population or re-indexing operations. """ +import json + from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema from cc_common.exceptions import CCNotFoundException +from cc_common.utils import ResponseEncoder from marshmallow import ValidationError from opensearch_client import OpenSearchClient @@ -109,7 +112,11 @@ def populate_provider_documents(event: dict, context: LambdaContext): # noqa: A # Sanitize using ProviderGeneralResponseSchema schema = ProviderGeneralResponseSchema() sanitized_document = schema.load(api_response) - documents_to_index.append(sanitized_document) + + # run the full provider document through our ResponseEncoder to convert sets + # to lists (e.g., privilegeJurisdictions) and datetime objects to strings for JSON serialization + serializable_document = json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) + documents_to_index.append(serializable_document) except CCNotFoundException: logger.warning('Provider not found when fetching records', provider_id=provider_id, compact=compact) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 7f6d3d3de..b8d9c0854 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -1,6 +1,4 @@ -from datetime import date, datetime, timedelta, timezone from unittest.mock import Mock, call, patch -from uuid import UUID from common_test.test_constants import ( DEFAULT_LICENSE_EXPIRATION_DATE, @@ -70,14 +68,13 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_inde def _generate_expected_call_for_document(self, compact): # Use timezone(timedelta(0), '+0000') to match how the code creates UTC timezone - utc_tz = timezone(timedelta(0), '+0000') return call( index_name=f'compact_{compact}_providers', documents=[ { - 'providerId': UUID(test_provider_id_mapping[compact]), + 'providerId': test_provider_id_mapping[compact], 'type': 'provider', - 'dateOfUpdate': datetime.fromisoformat(DEFAULT_PROVIDER_UPDATE_DATETIME).replace(tzinfo=utc_tz), + 'dateOfUpdate': DEFAULT_PROVIDER_UPDATE_DATETIME, 'compact': compact, 'licenseJurisdiction': 'oh', 'currentHomeJurisdiction': 'oh', @@ -87,19 +84,17 @@ def _generate_expected_call_for_document(self, compact): 'givenName': f'test{compact}GivenName', 'middleName': 'Gunnar', 'familyName': f'test{compact}FamilyName', - 'dateOfExpiration': date.fromisoformat(DEFAULT_LICENSE_EXPIRATION_DATE), + 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, 'compactConnectRegisteredEmailAddress': DEFAULT_REGISTERED_EMAIL_ADDRESS, 'jurisdictionUploadedLicenseStatus': 'active', 'jurisdictionUploadedCompactEligibility': 'eligible', - 'privilegeJurisdictions': {'ne'}, + 'privilegeJurisdictions': ['ne'], 'birthMonthDay': '06-06', 'licenses': [ { - 'providerId': UUID(test_provider_id_mapping[compact]), + 'providerId': test_provider_id_mapping[compact], 'type': 'license', - 'dateOfUpdate': datetime.fromisoformat(DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE).replace( - tzinfo=utc_tz - ), + 'dateOfUpdate': DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, 'compact': compact, 'jurisdiction': 'oh', 'licenseType': test_license_type_mapping[compact], @@ -113,9 +108,9 @@ def _generate_expected_call_for_document(self, compact): 'givenName': f'test{compact}GivenName', 'middleName': 'Gunnar', 'familyName': f'test{compact}FamilyName', - 'dateOfIssuance': date.fromisoformat(DEFAULT_LICENSE_ISSUANCE_DATE), - 'dateOfRenewal': date.fromisoformat(DEFAULT_LICENSE_RENEWAL_DATE), - 'dateOfExpiration': date.fromisoformat(DEFAULT_LICENSE_EXPIRATION_DATE), + 'dateOfIssuance': DEFAULT_LICENSE_ISSUANCE_DATE, + 'dateOfRenewal': DEFAULT_LICENSE_RENEWAL_DATE, + 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, 'homeAddressStreet1': '123 A St.', 'homeAddressStreet2': 'Apt 321', 'homeAddressCity': 'Columbus', From 6ebb48d4c293da1f75c57ca912613d024d448460 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 13:56:04 -0600 Subject: [PATCH 031/137] Do not specify index in bulk index body --- .../python/search/opensearch_client.py | 12 +- .../python/search/tests/unit/__init__.py | 0 .../tests/unit/test_opensearch_client.py | 158 ++++++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 backend/compact-connect/lambdas/python/search/tests/unit/__init__.py create mode 100644 backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 51c831847..235d0e5a0 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -36,8 +36,8 @@ def index_document(self, index_name: str, document_id: str, document: dict) -> d """ Index a single document into the specified index. - :param index_name: The name of the index to write to - :param document_id: The unique identifier for the document + :param index_name: The name of the index to write to. + :param document_id: The unique identifier for the document. :param document: The document to index :return: The response from OpenSearch """ @@ -57,6 +57,10 @@ def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'pr actions = [] for doc in documents: - actions.append({'index': {'_index': index_name, '_id': doc[id_field]}}) + # Note: We specify the index via the `index` parameter in the bulk() call below, + # not in the action metadata. This is required because the OpenSearch domain has + # `rest.action.multi.allow_explicit_index: false` which prevents specifying + # indices in the request body for security purposes. + actions.append({'index': {'_id': doc[id_field]}}) actions.append(doc) - return self._client.bulk(body=actions) + return self._client.bulk(body=actions, index=index_name) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/__init__.py b/backend/compact-connect/lambdas/python/search/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py new file mode 100644 index 000000000..b1d7d6c02 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -0,0 +1,158 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + + +class TestOpenSearchClient(TestCase): + """Test suite for OpenSearchClient to verify internal client calls.""" + + def _create_client_with_mock(self): + """Create an OpenSearchClient with a mocked internal client.""" + with patch('opensearch_client.boto3'), patch('opensearch_client.config'), patch( + 'opensearch_client.OpenSearch' + ) as mock_opensearch_class: + mock_internal_client = MagicMock() + mock_opensearch_class.return_value = mock_internal_client + + from opensearch_client import OpenSearchClient + + client = OpenSearchClient() + return client, mock_internal_client + + def test_create_index_calls_internal_client_with_expected_arguments(self): + """Test that create_index calls the internal client's indices.create method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + index_mapping = { + 'settings': {'number_of_shards': 1}, + 'mappings': {'properties': {'field1': {'type': 'text'}}}, + } + + client.create_index(index_name=index_name, index_mapping=index_mapping) + + mock_internal_client.indices.create.assert_called_once_with( + index=index_name, + body=index_mapping, + ) + + def test_index_exists_calls_internal_client_with_expected_arguments(self): + """Test that index_exists calls the internal client's indices.exists method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + mock_internal_client.indices.exists.return_value = True + + result = client.index_exists(index_name=index_name) + + mock_internal_client.indices.exists.assert_called_once_with(index=index_name) + self.assertTrue(result) + + def test_search_calls_internal_client_with_expected_arguments(self): + """Test that search calls the internal client's search method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + query_body = { + 'query': { + 'match': {'givenName': 'John'}, + }, + } + expected_response = { + 'hits': { + 'total': {'value': 1}, + 'hits': [{'_source': {'givenName': 'John', 'familyName': 'Doe'}}], + }, + } + mock_internal_client.search.return_value = expected_response + + result = client.search(index_name=index_name, body=query_body) + + mock_internal_client.search.assert_called_once_with( + index=index_name, + body=query_body, + ) + self.assertEqual(expected_response, result) + + def test_index_document_calls_internal_client_with_expected_arguments(self): + """Test that index_document calls the internal client's index method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + document_id = 'doc-123' + document = {'providerId': 'doc-123', 'givenName': 'John', 'familyName': 'Doe'} + expected_response = {'_index': index_name, '_id': document_id, 'result': 'created'} + mock_internal_client.index.return_value = expected_response + + result = client.index_document(index_name=index_name, document_id=document_id, document=document) + + mock_internal_client.index.assert_called_once_with( + index=index_name, + id=document_id, + body=document, + ) + self.assertEqual(expected_response, result) + + def test_bulk_index_calls_internal_client_with_expected_arguments(self): + """Test that bulk_index calls the internal client's bulk method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + documents = [ + {'providerId': 'provider-1', 'givenName': 'John', 'familyName': 'Doe'}, + {'providerId': 'provider-2', 'givenName': 'Jane', 'familyName': 'Smith'}, + ] + expected_response = {'errors': False, 'items': [{'index': {'_id': 'provider-1'}}, {'index': {'_id': 'provider-2'}}]} + mock_internal_client.bulk.return_value = expected_response + + result = client.bulk_index(index_name=index_name, documents=documents) + + # Verify that the bulk method is called with the index in the URL parameter + # and NOT in the action metadata (for security compliance) + expected_actions = [ + {'index': {'_id': 'provider-1'}}, + {'providerId': 'provider-1', 'givenName': 'John', 'familyName': 'Doe'}, + {'index': {'_id': 'provider-2'}}, + {'providerId': 'provider-2', 'givenName': 'Jane', 'familyName': 'Smith'}, + ] + mock_internal_client.bulk.assert_called_once_with( + body=expected_actions, + index=index_name, + ) + self.assertEqual(expected_response, result) + + def test_bulk_index_uses_custom_id_field(self): + """Test that bulk_index uses a custom id_field when specified.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + documents = [ + {'customId': 'custom-1', 'name': 'Document 1'}, + {'customId': 'custom-2', 'name': 'Document 2'}, + ] + mock_internal_client.bulk.return_value = {'errors': False, 'items': []} + + client.bulk_index(index_name=index_name, documents=documents, id_field='customId') + + expected_actions = [ + {'index': {'_id': 'custom-1'}}, + {'customId': 'custom-1', 'name': 'Document 1'}, + {'index': {'_id': 'custom-2'}}, + {'customId': 'custom-2', 'name': 'Document 2'}, + ] + mock_internal_client.bulk.assert_called_once_with( + body=expected_actions, + index=index_name, + ) + + def test_bulk_index_returns_early_for_empty_documents(self): + """Test that bulk_index returns early without calling the internal client for empty documents.""" + client, mock_internal_client = self._create_client_with_mock() + + result = client.bulk_index(index_name='test_index', documents=[]) + + mock_internal_client.bulk.assert_not_called() + self.assertEqual({'items': [], 'errors': False}, result) + + + + From 6eb717b25162a2d40935f559e6fc48f0bdeb62ee Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 14:19:47 -0600 Subject: [PATCH 032/137] Restrict api access to search operation --- .../search_persistent_stack/__init__.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 94c1e2aef..18a8579cc 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -241,18 +241,28 @@ def __init__( ], resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], ) - opensearch_search_api_access_policy = PolicyStatement( + # Search API policy - restricted to _search endpoint only + # POST is required for _search queries even though they are read-only operations + # because OpenSearch's search API uses POST to send the query DSL body. + # By restricting the resource to /_search, we prevent POST from being used + # for document indexing or other write operations. + # See: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html + opensearch_search_api_policy = PolicyStatement( effect=Effect.ALLOW, principals=[self.search_api_lambda_role], actions=[ - 'es:ESHttpGet', - 'es:ESHttpHead', + 'es:ESHttpPost', ], - resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + # define all compact indices to restrict the policy to the search operation + resources=[Fn.join(delimiter='', + list_of_values=[self.domain.domain_arn, f'/compact_{compact}_providers/_search']) + for compact in persistent_stack.get_list_of_compact_abbreviations()], ) # add access policy to restrict access to set of roles self.domain.add_access_policies( - opensearch_ingest_access_policy, opensearch_index_manager_access_policy, opensearch_search_api_access_policy + opensearch_ingest_access_policy, + opensearch_index_manager_access_policy, + opensearch_search_api_policy, ) # CDK creates a lambda function to manage the access policies, we need to add suppressions for it self._add_access_policy_lambda_suppressions() From e13a87538f4e798521ff7594ee910083abda7b98 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 15:18:58 -0600 Subject: [PATCH 033/137] Add search api spec --- backend/compact-connect/bin/trim_oas30.py | 16 +- .../compact-connect/bin/update_api_docs.sh | 22 +- .../bin/update_postman_collection.py | 18 +- .../api-specification/latest-oas30.json | 1440 +++++++++++++++++ .../api-specification/swagger.html | 22 + 5 files changed, 1507 insertions(+), 11 deletions(-) create mode 100644 backend/compact-connect/docs/search-internal/api-specification/latest-oas30.json create mode 100644 backend/compact-connect/docs/search-internal/api-specification/swagger.html diff --git a/backend/compact-connect/bin/trim_oas30.py b/backend/compact-connect/bin/trim_oas30.py index e28158d16..6fbcac725 100755 --- a/backend/compact-connect/bin/trim_oas30.py +++ b/backend/compact-connect/bin/trim_oas30.py @@ -35,6 +35,9 @@ def strip_options_endpoints(oas30: dict) -> dict: parser.add_argument( '-i', '--internal', action='store_true', help='Use internal API specification files instead of regular ones' ) + parser.add_argument( + '-s', '--search', action='store_true', help='Use search API specification files' + ) args = parser.parse_args() @@ -42,12 +45,13 @@ def strip_options_endpoints(oas30: dict) -> dict: script_dir = os.path.dirname(os.path.abspath(__file__)) workspace_dir = os.path.dirname(script_dir) - # Determine the base directory based on the internal flag - base_dir = ( - os.path.join('docs', 'internal', 'api-specification') - if args.internal - else os.path.join('docs', 'api-specification') - ) + # Determine the base directory based on the flags + if args.search: + base_dir = os.path.join('docs', 'search-internal', 'api-specification') + elif args.internal: + base_dir = os.path.join('docs', 'internal', 'api-specification') + else: + base_dir = os.path.join('docs', 'api-specification') file_path = os.path.join(workspace_dir, base_dir, 'latest-oas30.json') with open(file_path) as f: diff --git a/backend/compact-connect/bin/update_api_docs.sh b/backend/compact-connect/bin/update_api_docs.sh index 7a38f63fa..5f8537b9e 100755 --- a/backend/compact-connect/bin/update_api_docs.sh +++ b/backend/compact-connect/bin/update_api_docs.sh @@ -1,7 +1,7 @@ #!/bin/bash # Update API documentation workflow -# Downloads, trims, and updates Postman collections for both StateApi and LicenseApi +# Downloads, trims, and updates Postman collections for StateApi, LicenseApi, and SearchApi set -e # Exit immediately if any command fails @@ -99,6 +99,14 @@ trim_specs() { exit 1 fi print_success "LicenseApi specification trimmed" + + # Trim search API spec + print_status "Trimming SearchApi specification..." + if ! python3 bin/trim_oas30.py --search; then + print_error "Failed to trim SearchApi specification" + exit 1 + fi + print_success "SearchApi specification trimmed" } # Function to update Postman collections @@ -120,6 +128,14 @@ update_postman() { exit 1 fi print_success "LicenseApi Postman collection updated" + + # Update search Postman collection + print_status "Updating SearchApi Postman collection..." + if ! python3 bin/update_postman_collection.py --search; then + print_error "Failed to update SearchApi Postman collection" + exit 1 + fi + print_success "SearchApi Postman collection updated" } # Function to verify files exist @@ -129,8 +145,10 @@ verify_files() { local files=( "docs/api-specification/latest-oas30.json" "docs/internal/api-specification/latest-oas30.json" + "docs/search-internal/api-specification/latest-oas30.json" "docs/postman/postman-collection.json" "docs/internal/postman/postman-collection.json" + "docs/search-internal/postman/postman-collection.json" ) for file in "${files[@]}"; do @@ -174,8 +192,10 @@ main() { print_status "Updated files:" echo " - docs/api-specification/latest-oas30.json" echo " - docs/internal/api-specification/latest-oas30.json" + echo " - docs/search-internal/api-specification/latest-oas30.json" echo " - docs/postman/postman-collection.json" echo " - docs/internal/postman/postman-collection.json" + echo " - docs/search-internal/postman/postman-collection.json" } # Handle script interruption diff --git a/backend/compact-connect/bin/update_postman_collection.py b/backend/compact-connect/bin/update_postman_collection.py index 492b8b724..9a684c67f 100755 --- a/backend/compact-connect/bin/update_postman_collection.py +++ b/backend/compact-connect/bin/update_postman_collection.py @@ -196,6 +196,9 @@ def main(): parser.add_argument( '-i', '--internal', action='store_true', help='Use internal API specification files instead of regular ones' ) + parser.add_argument( + '-s', '--search', action='store_true', help='Use search API specification files' + ) args = parser.parse_args() @@ -203,9 +206,16 @@ def main(): script_dir = os.path.dirname(os.path.abspath(__file__)) workspace_dir = os.path.dirname(script_dir) - # Determine the base directory based on the internal flag - base_dir = os.path.join('internal', 'api-specification') if args.internal else os.path.join('api-specification') - postman_dir = os.path.join('internal', 'postman') if args.internal else os.path.join('postman') + # Determine the base directory based on the flags + if args.search: + base_dir = os.path.join('search-internal', 'api-specification') + postman_dir = os.path.join('search-internal', 'postman') + elif args.internal: + base_dir = os.path.join('internal', 'api-specification') + postman_dir = os.path.join('internal', 'postman') + else: + base_dir = os.path.join('api-specification') + postman_dir = os.path.join('postman') openapi_path = os.path.join(workspace_dir, 'docs', base_dir, 'latest-oas30.json') tmp_path = os.path.join(workspace_dir, 'tmp.json') @@ -215,7 +225,7 @@ def main(): generate_postman_collection(openapi_path, tmp_path) try: - # Load the generated and existing collections + # Load the generated collection with open(tmp_path) as f: new_collection = json.load(f) with open(postman_path) as f: diff --git a/backend/compact-connect/docs/search-internal/api-specification/latest-oas30.json b/backend/compact-connect/docs/search-internal/api-specification/latest-oas30.json new file mode 100644 index 000000000..c1c006228 --- /dev/null +++ b/backend/compact-connect/docs/search-internal/api-specification/latest-oas30.json @@ -0,0 +1,1440 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "SearchApi", + "version": "2025-12-02T19:49:45Z" + }, + "servers": [ + { + "url": "https://search.beta.compactconnect.org" + } + ], + "paths": { + "/v1/compacts/{compact}/providers/search": { + "post": { + "parameters": [ + { + "name": "compact", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SandboSearctZ4sfzliddmr" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "200 response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SandboSearcRcmFGOzNZ5TZ" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "SandboSearcRcmFGOzNZ5TZ": { + "required": [ + "providers", + "total" + ], + "type": "object", + "properties": { + "total": { + "type": "object", + "properties": { + "value": { + "type": "integer" + }, + "relation": { + "type": "string", + "enum": [ + "eq", + "gte" + ] + } + }, + "description": "Total hits information from OpenSearch" + }, + "lastSort": { + "type": "array", + "description": "Sort values from the last hit to use with search_after for the next page" + }, + "providers": { + "type": "array", + "items": { + "required": [ + "birthMonthDay", + "compact", + "compactEligibility", + "dateOfExpiration", + "dateOfUpdate", + "familyName", + "givenName", + "jurisdictionUploadedCompactEligibility", + "jurisdictionUploadedLicenseStatus", + "licenseJurisdiction", + "licenseStatus", + "privilegeJurisdictions", + "providerId", + "type" + ], + "type": "object", + "properties": { + "privileges": { + "type": "array", + "items": { + "required": [ + "administratorSetStatus", + "compact", + "dateOfExpiration", + "dateOfIssuance", + "dateOfRenewal", + "dateOfUpdate", + "jurisdiction", + "licenseJurisdiction", + "licenseType", + "privilegeId", + "providerId", + "status", + "type" + ], + "type": "object", + "properties": { + "investigationStatus": { + "type": "string", + "enum": [ + "underInvestigation" + ] + }, + "licenseJurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "jurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "attestations": { + "type": "array", + "items": { + "required": [ + "attestationId", + "version" + ], + "type": "object", + "properties": { + "attestationId": { + "maxLength": 100, + "type": "string" + }, + "version": { + "maxLength": 100, + "type": "string" + } + } + } + }, + "investigations": { + "type": "array", + "items": { + "required": [ + "compact", + "creationDate", + "dateOfUpdate", + "investigationId", + "jurisdiction", + "licenseType", + "providerId", + "submittingUser", + "type" + ], + "type": "object", + "properties": { + "licenseType": { + "type": "string" + }, + "investigationId": { + "type": "string" + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "jurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "submittingUser": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "investigation" + ] + }, + "creationDate": { + "type": "string", + "format": "date-time" + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + } + } + } + }, + "type": { + "type": "string", + "enum": [ + "privilege" + ] + }, + "compactTransactionId": { + "type": "string" + }, + "dateOfIssuance": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "licenseType": { + "type": "string" + }, + "administratorSetStatus": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "dateOfExpiration": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "activeSince": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "privilegeId": { + "type": "string" + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "dateOfRenewal": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "adverseActions": { + "type": "array", + "items": { + "required": [ + "actionAgainst", + "adverseActionId", + "compact", + "creationDate", + "dateOfUpdate", + "effectiveStartDate", + "encumbranceType", + "jurisdiction", + "licenseType", + "licenseTypeAbbreviation", + "providerId", + "submittingUser", + "type" + ], + "type": "object", + "properties": { + "clinicalPrivilegeActionCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "jurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "licenseTypeAbbreviation": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "adverseAction" + ] + }, + "creationDate": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "actionAgainst": { + "type": "string", + "enum": [ + "license", + "privilege" + ] + }, + "licenseType": { + "type": "string" + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "submittingUser": { + "type": "string" + }, + "effectiveStartDate": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "adverseActionId": { + "type": "string" + }, + "effectiveLiftDate": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "encumbranceType": { + "type": "string" + }, + "liftingUser": { + "type": "string" + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + } + } + } + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + } + } + } + }, + "licenseJurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "npi": { + "pattern": "^[0-9]{10}$", + "type": "string" + }, + "givenName": { + "maxLength": 100, + "type": "string" + }, + "compactEligibility": { + "type": "string", + "enum": [ + "eligible", + "ineligible" + ] + }, + "jurisdictionUploadedCompactEligibility": { + "type": "string", + "enum": [ + "eligible", + "ineligible" + ] + }, + "jurisdictionUploadedLicenseStatus": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "privilegeJurisdictions": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + } + }, + "type": { + "type": "string", + "enum": [ + "provider" + ] + }, + "suffix": { + "maxLength": 100, + "type": "string" + }, + "currentHomeJurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "licenses": { + "type": "array", + "items": { + "required": [ + "compact", + "compactEligibility", + "dateOfExpiration", + "dateOfIssuance", + "dateOfUpdate", + "familyName", + "givenName", + "homeAddressCity", + "homeAddressPostalCode", + "homeAddressState", + "homeAddressStreet1", + "jurisdiction", + "jurisdictionUploadedCompactEligibility", + "jurisdictionUploadedLicenseStatus", + "licenseStatus", + "licenseType", + "providerId", + "type" + ], + "type": "object", + "properties": { + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "homeAddressStreet2": { + "maxLength": 100, + "minLength": 1, + "type": "string" + }, + "jurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "homeAddressStreet1": { + "maxLength": 100, + "minLength": 2, + "type": "string" + }, + "investigations": { + "type": "array", + "items": { + "required": [ + "compact", + "creationDate", + "dateOfUpdate", + "investigationId", + "jurisdiction", + "licenseType", + "providerId", + "submittingUser", + "type" + ], + "type": "object", + "properties": { + "licenseType": { + "type": "string" + }, + "investigationId": { + "type": "string" + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "jurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "submittingUser": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "investigation" + ] + }, + "creationDate": { + "type": "string", + "format": "date-time" + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + } + } + } + }, + "type": { + "type": "string", + "enum": [ + "license-home" + ] + }, + "suffix": { + "maxLength": 100, + "type": "string" + }, + "dateOfIssuance": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "licenseType": { + "type": "string" + }, + "emailAddress": { + "type": "string", + "format": "email" + }, + "dateOfExpiration": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "homeAddressState": { + "maxLength": 100, + "minLength": 2, + "type": "string" + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "dateOfRenewal": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "familyName": { + "maxLength": 100, + "type": "string" + }, + "homeAddressCity": { + "maxLength": 100, + "minLength": 2, + "type": "string" + }, + "licenseNumber": { + "maxLength": 100, + "type": "string" + }, + "investigationStatus": { + "type": "string", + "enum": [ + "underInvestigation" + ] + }, + "npi": { + "pattern": "^[0-9]{10}$", + "type": "string" + }, + "homeAddressPostalCode": { + "maxLength": 7, + "minLength": 5, + "type": "string" + }, + "compactEligibility": { + "type": "string", + "enum": [ + "eligible", + "ineligible" + ] + }, + "givenName": { + "maxLength": 100, + "type": "string" + }, + "jurisdictionUploadedCompactEligibility": { + "type": "string", + "enum": [ + "eligible", + "ineligible" + ] + }, + "jurisdictionUploadedLicenseStatus": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "phoneNumber": { + "pattern": "^\\+[0-9]{8,15}$", + "type": "string" + }, + "licenseStatus": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "licenseStatusName": { + "maxLength": 100, + "type": "string" + }, + "middleName": { + "maxLength": 100, + "type": "string" + }, + "adverseActions": { + "type": "array", + "items": { + "required": [ + "actionAgainst", + "adverseActionId", + "compact", + "creationDate", + "dateOfUpdate", + "effectiveStartDate", + "encumbranceType", + "jurisdiction", + "licenseType", + "licenseTypeAbbreviation", + "providerId", + "submittingUser", + "type" + ], + "type": "object", + "properties": { + "clinicalPrivilegeActionCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "jurisdiction": { + "type": "string", + "enum": [ + "al", + "ak", + "az", + "ar", + "ca", + "co", + "ct", + "de", + "dc", + "fl", + "ga", + "hi", + "id", + "il", + "in", + "ia", + "ks", + "ky", + "la", + "me", + "md", + "ma", + "mi", + "mn", + "ms", + "mo", + "mt", + "ne", + "nv", + "nh", + "nj", + "nm", + "ny", + "nc", + "nd", + "oh", + "ok", + "or", + "pa", + "pr", + "ri", + "sc", + "sd", + "tn", + "tx", + "ut", + "vt", + "va", + "vi", + "wa", + "wv", + "wi", + "wy" + ] + }, + "licenseTypeAbbreviation": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "adverseAction" + ] + }, + "creationDate": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "actionAgainst": { + "type": "string", + "enum": [ + "license", + "privilege" + ] + }, + "licenseType": { + "type": "string" + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "submittingUser": { + "type": "string" + }, + "effectiveStartDate": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "adverseActionId": { + "type": "string" + }, + "effectiveLiftDate": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "encumbranceType": { + "type": "string" + }, + "liftingUser": { + "type": "string" + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + } + } + } + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + } + } + } + }, + "dateOfExpiration": { + "type": "string", + "format": "date" + }, + "militaryAffiliations": { + "type": "array", + "items": { + "required": [ + "affiliationType", + "compact", + "dateOfUpdate", + "dateOfUpload", + "fileNames", + "providerId", + "status", + "type" + ], + "type": "object", + "properties": { + "dateOfUpload": { + "pattern": "^[12]{1}[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$", + "type": "string", + "format": "date" + }, + "compact": { + "type": "string", + "enum": [ + "aslp", + "octp", + "coun" + ] + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "affiliationType": { + "type": "string", + "enum": [ + "militaryMember", + "militaryMemberSpouse" + ] + }, + "type": { + "type": "string", + "enum": [ + "militaryAffiliation" + ] + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + }, + "fileNames": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + } + } + } + }, + "providerId": { + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", + "type": "string" + }, + "licenseStatus": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "familyName": { + "maxLength": 100, + "type": "string" + }, + "middleName": { + "maxLength": 100, + "type": "string" + }, + "birthMonthDay": { + "pattern": "^[0-1]{1}[0-9]{1}-[0-3]{1}[0-9]{1}", + "type": "string" + }, + "compactConnectRegisteredEmailAddress": { + "type": "string", + "format": "email" + }, + "dateOfUpdate": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "SandboSearctZ4sfzliddmr": { + "required": [ + "query" + ], + "type": "object", + "properties": { + "search_after": { + "type": "array", + "description": "Sort values from the last hit of the previous page for cursor-based pagination" + }, + "size": { + "maximum": 100, + "minimum": 1, + "type": "integer", + "description": "Number of results to return" + }, + "query": { + "type": "object", + "description": "The OpenSearch query body" + }, + "from": { + "minimum": 0, + "type": "integer", + "description": "Starting document offset for pagination" + }, + "sort": { + "type": "array", + "description": "Sort order for results (required for search_after pagination)", + "items": { + "type": "object" + } + } + }, + "additionalProperties": false + } + } + }, + "x-amazon-apigateway-security-policy": "TLS_1_0" +} diff --git a/backend/compact-connect/docs/search-internal/api-specification/swagger.html b/backend/compact-connect/docs/search-internal/api-specification/swagger.html new file mode 100644 index 000000000..44396776c --- /dev/null +++ b/backend/compact-connect/docs/search-internal/api-specification/swagger.html @@ -0,0 +1,22 @@ + + + + + + + SwaggerUI + + + +
+ + + + From c552bec751a4bf3590bbb2efa0d3f24f23f74eef Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 15:19:29 -0600 Subject: [PATCH 034/137] update download spec script --- backend/compact-connect/bin/download_oas30.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/bin/download_oas30.py b/backend/compact-connect/bin/download_oas30.py index 7c09f31f6..8869da8f0 100755 --- a/backend/compact-connect/bin/download_oas30.py +++ b/backend/compact-connect/bin/download_oas30.py @@ -106,6 +106,8 @@ def update_server_urls(spec: dict, api_name: str) -> None: base_url = 'https://state-api.beta.compactconnect.org' elif api_name == 'LicenseApi': base_url = 'https://api.beta.compactconnect.org' + elif api_name == 'SearchApi': + base_url = 'https://search.beta.compactconnect.org' else: # Keep original URL if API name is not recognized return @@ -155,6 +157,7 @@ def main(): parser = argparse.ArgumentParser(description='Download OpenAPI v3 specifications from AWS API Gateway') parser.add_argument('--state-api-only', action='store_true', help='Download only the StateApi specification') parser.add_argument('--license-api-only', action='store_true', help='Download only the LicenseApi specification') + parser.add_argument('--search-api-only', action='store_true', help='Download only the SearchApi specification') args = parser.parse_args() @@ -165,17 +168,23 @@ def main(): # Define output paths state_api_path = os.path.join(workspace_dir, 'docs', 'api-specification', 'latest-oas30.json') license_api_path = os.path.join(workspace_dir, 'docs', 'internal', 'api-specification', 'latest-oas30.json') + search_api_path = os.path.join(workspace_dir, 'docs', 'search-internal', 'api-specification', 'latest-oas30.json') # Download StateApi (external API) - if not args.license_api_only: + if not args.license_api_only and not args.search_api_only: sys.stdout.write('\n=== Downloading StateApi specification ===\n') download_api_spec('StateApi', state_api_path) # Download LicenseApi (internal API) - if not args.state_api_only: + if not args.state_api_only and not args.search_api_only: sys.stdout.write('\n=== Downloading LicenseApi specification ===\n') download_api_spec('LicenseApi', license_api_path) + # Download SearchApi (search internal API) + if not args.state_api_only and not args.license_api_only: + sys.stdout.write('\n=== Downloading SearchApi specification ===\n') + download_api_spec('SearchApi', search_api_path) + sys.stdout.write('\nAll specifications downloaded successfully!\n') sys.exit(0) From bf8e6da5a7e1f9cc98fa85696a831ce9012725bc Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 15:35:01 -0600 Subject: [PATCH 035/137] remove lines added from merge conflict --- backend/compact-connect/bin/compile_requirements.sh | 2 -- backend/compact-connect/bin/sync_deps.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/backend/compact-connect/bin/compile_requirements.sh b/backend/compact-connect/bin/compile_requirements.sh index 7ee49131a..1a843d80e 100755 --- a/backend/compact-connect/bin/compile_requirements.sh +++ b/backend/compact-connect/bin/compile_requirements.sh @@ -21,8 +21,6 @@ pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/provi # avoid installation failures # pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements-dev.in # pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements.in -pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements-dev.in -pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/purchases/requirements.in pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/search/requirements-dev.in pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/search/requirements.in pip-compile --no-emit-index-url --upgrade --no-strip-extras lambdas/python/staff-user-pre-token/requirements-dev.in diff --git a/backend/compact-connect/bin/sync_deps.sh b/backend/compact-connect/bin/sync_deps.sh index 8801bdcf6..d3bfea152 100755 --- a/backend/compact-connect/bin/sync_deps.sh +++ b/backend/compact-connect/bin/sync_deps.sh @@ -20,8 +20,6 @@ pip-sync \ lambdas/python/disaster-recovery/requirements.txt \ lambdas/python/provider-data-v1/requirements-dev.txt \ lambdas/python/provider-data-v1/requirements.txt \ - lambdas/python/purchases/requirements-dev.txt \ - lambdas/python/purchases/requirements.txt \ lambdas/python/search/requirements-dev.txt \ lambdas/python/search/requirements.txt \ lambdas/python/staff-user-pre-token/requirements-dev.txt \ From 4117c1cd7e3f06060ded70ffa6f1da8745028f72 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 15:42:50 -0600 Subject: [PATCH 036/137] update requirements to latest --- .../cognito-backup/requirements-dev.txt | 14 ++++---- .../python/cognito-backup/requirements.txt | 8 ++--- .../python/common/requirements-dev.txt | 35 ++++++++++--------- .../lambdas/python/common/requirements.txt | 8 ++--- .../requirements-dev.txt | 12 +++---- .../compact-configuration/requirements.txt | 2 +- .../custom-resources/requirements-dev.txt | 12 +++---- .../python/custom-resources/requirements.txt | 2 +- .../python/data-events/requirements-dev.txt | 12 +++---- .../python/data-events/requirements.txt | 2 +- .../disaster-recovery/requirements-dev.txt | 12 +++---- .../python/disaster-recovery/requirements.txt | 2 +- .../provider-data-v1/requirements-dev.txt | 12 +++---- .../python/provider-data-v1/requirements.txt | 2 +- .../python/search/requirements-dev.txt | 10 +++--- .../staff-user-pre-token/requirements-dev.txt | 12 +++---- .../staff-user-pre-token/requirements.txt | 2 +- .../python/staff-users/requirements-dev.txt | 14 ++++---- .../python/staff-users/requirements.txt | 2 +- backend/compact-connect/requirements-dev.txt | 16 +++++---- backend/compact-connect/requirements.txt | 10 +++--- 21 files changed, 103 insertions(+), 98 deletions(-) diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt index f94e5cec9..b289dda64 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt @@ -1,16 +1,16 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/cognito-backup/requirements-dev.in # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements-dev.in -boto3==1.41.0 +boto3==1.42.1 # via # -r lambdas/python/cognito-backup/requirements-dev.in # moto -botocore==1.41.0 +botocore==1.42.1 # via # -r lambdas/python/cognito-backup/requirements-dev.in # boto3 @@ -37,13 +37,13 @@ jmespath==1.0.1 # aws-lambda-powertools # boto3 # botocore -joserfc==1.4.3 +joserfc==1.5.0 # via moto markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[cognitoidp,s3]==5.1.17 +moto[cognitoidp,s3]==5.1.18 # via -r lambdas/python/cognito-backup/requirements-dev.in packaging==25.0 # via pytest @@ -71,7 +71,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -82,7 +82,7 @@ urllib3==2.5.0 # botocore # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt index a58094bd4..077b34697 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/cognito-backup/requirements.in # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements.in -boto3==1.41.0 +boto3==1.42.1 # via -r lambdas/python/cognito-backup/requirements.in -botocore==1.41.0 +botocore==1.42.1 # via # -r lambdas/python/cognito-backup/requirements.in # boto3 @@ -20,7 +20,7 @@ jmespath==1.0.1 # botocore python-dateutil==2.9.0.post0 # via botocore -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil diff --git a/backend/compact-connect/lambdas/python/common/requirements-dev.txt b/backend/compact-connect/lambdas/python/common/requirements-dev.txt index 4dbfbec7e..b6c13ba34 100644 --- a/backend/compact-connect/lambdas/python/common/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/common/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/common/requirements-dev.in @@ -12,31 +12,31 @@ attrs==25.4.0 # via # jsonschema # referencing -aws-sam-translator==1.102.0 +aws-sam-translator==1.105.0 # via cfn-lint aws-xray-sdk==2.15.0 # via moto -boto3==1.41.0 +boto3==1.42.1 # via # aws-sam-translator # moto -boto3-stubs[full]==1.40.76 +boto3-stubs[full]==1.41.5 # via -r lambdas/python/common/requirements-dev.in -boto3-stubs-full==1.40.76 +boto3-stubs-full==1.41.5 # via boto3-stubs -botocore==1.41.0 +botocore==1.42.1 # via # aws-xray-sdk # boto3 # moto # s3transfer -botocore-stubs==1.40.76 +botocore-stubs==1.42.1 # via boto3-stubs certifi==2025.11.12 # via requests cffi==2.0.0 # via cryptography -cfn-lint==1.41.0 +cfn-lint==1.42.0 # via moto charset-normalizer==3.4.4 # via requests @@ -59,7 +59,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joserfc==1.4.3 +joserfc==1.5.0 # via moto jsonpatch==1.33 # via cfn-lint @@ -85,13 +85,13 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[all]==5.1.17 +moto[all]==5.1.18 # via -r lambdas/python/common/requirements-dev.in mpmath==1.3.0 # via sympy multipart==1.3.0 # via moto -networkx==3.5 +networkx==3.6 # via cfn-lint openapi-schema-validator==0.6.3 # via openapi-spec-validator @@ -105,7 +105,7 @@ py-partiql-parser==0.6.3 # via moto pycparser==2.23 # via cffi -pydantic==2.12.4 +pydantic==2.12.5 # via aws-sam-translator pydantic-core==2.41.5 # via pydantic @@ -138,11 +138,11 @@ responses==0.25.8 # via moto rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.29.0 +rpds-py==0.30.0 # via # jsonschema # referencing -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via @@ -150,9 +150,9 @@ six==1.17.0 # rfc3339-validator sympy==1.14.0 # via cfn-lint -types-awscrt==0.28.4 +types-awscrt==0.29.1 # via botocore-stubs -types-s3transfer==0.14.0 +types-s3transfer==0.15.0 # via boto3-stubs typing-extensions==4.15.0 # via @@ -160,6 +160,7 @@ typing-extensions==4.15.0 # cfn-lint # pydantic # pydantic-core + # referencing # typing-inspection typing-inspection==0.4.2 # via pydantic @@ -171,7 +172,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto wrapt==2.0.1 # via aws-xray-sdk diff --git a/backend/compact-connect/lambdas/python/common/requirements.txt b/backend/compact-connect/lambdas/python/common/requirements.txt index 6f49c3833..278d8c7be 100644 --- a/backend/compact-connect/lambdas/python/common/requirements.txt +++ b/backend/compact-connect/lambdas/python/common/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/common/requirements.in @@ -10,9 +10,9 @@ argon2-cffi-bindings==25.1.0 # via argon2-cffi aws-lambda-powertools==3.23.0 # via -r lambdas/python/common/requirements.in -boto3==1.41.0 +boto3==1.42.1 # via -r lambdas/python/common/requirements.in -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # s3transfer @@ -43,7 +43,7 @@ python-dateutil==2.9.0.post0 # via botocore requests==2.32.5 # via -r lambdas/python/common/requirements.in -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil diff --git a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt index 043ab6de2..82f46dbab 100644 --- a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/compact-configuration/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -33,7 +33,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb,s3]==5.1.17 +moto[dynamodb,s3]==5.1.18 # via -r lambdas/python/compact-configuration/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -54,7 +54,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -64,7 +64,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt b/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt index 79d227dbe..9eafb5559 100644 --- a/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt +++ b/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/compact-configuration/requirements.in diff --git a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt index db56d974e..a6886ea70 100644 --- a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/custom-resources/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -33,7 +33,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb,s3]==5.1.17 +moto[dynamodb,s3]==5.1.18 # via -r lambdas/python/custom-resources/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -54,7 +54,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -64,7 +64,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/custom-resources/requirements.txt b/backend/compact-connect/lambdas/python/custom-resources/requirements.txt index 2a8810a14..e1110d66e 100644 --- a/backend/compact-connect/lambdas/python/custom-resources/requirements.txt +++ b/backend/compact-connect/lambdas/python/custom-resources/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/custom-resources/requirements.in diff --git a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt index 22a836bd7..29a9d9928 100644 --- a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/data-events/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -33,7 +33,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb,s3]==5.1.17 +moto[dynamodb,s3]==5.1.18 # via -r lambdas/python/data-events/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -54,7 +54,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -64,7 +64,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/data-events/requirements.txt b/backend/compact-connect/lambdas/python/data-events/requirements.txt index 7a1fc37aa..70343d550 100644 --- a/backend/compact-connect/lambdas/python/data-events/requirements.txt +++ b/backend/compact-connect/lambdas/python/data-events/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/data-events/requirements.in diff --git a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt index 44d9a0022..950742a88 100644 --- a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/disaster-recovery/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -33,7 +33,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb,s3]==5.1.17 +moto[dynamodb,s3]==5.1.18 # via -r lambdas/python/disaster-recovery/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -54,7 +54,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -64,7 +64,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt b/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt index 9ad49d395..1366cbf3b 100644 --- a/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt +++ b/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/disaster-recovery/requirements.in diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt index 35825ad72..287cb6c44 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/provider-data-v1/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -35,7 +35,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb,s3]==5.1.17 +moto[dynamodb,s3]==5.1.18 # via -r lambdas/python/provider-data-v1/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -56,7 +56,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -68,7 +68,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt b/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt index f9665c63b..8ca619fb1 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt +++ b/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/provider-data-v1/requirements.in diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.txt b/backend/compact-connect/lambdas/python/search/requirements-dev.txt index aba8ea2b7..5dd4157cd 100644 --- a/backend/compact-connect/lambdas/python/search/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/search/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements-dev.in # -boto3==1.41.4 +boto3==1.42.1 # via moto -botocore==1.41.4 +botocore==1.42.1 # via # boto3 # moto @@ -33,7 +33,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb]==5.1.17 +moto[dynamodb]==5.1.18 # via -r lambdas/python/search/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -52,7 +52,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.15.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -62,7 +62,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt index fa4f50392..0d413d5ef 100644 --- a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-user-pre-token/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -33,7 +33,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[dynamodb,s3]==5.1.17 +moto[dynamodb,s3]==5.1.18 # via -r lambdas/python/staff-user-pre-token/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -54,7 +54,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -64,7 +64,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt index e7cabe5cb..e4844a1f4 100644 --- a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt +++ b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-user-pre-token/requirements.in diff --git a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt index 54e10b15f..a5c613557 100644 --- a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-users/requirements-dev.in # -boto3==1.41.0 +boto3==1.42.1 # via moto -botocore==1.41.0 +botocore==1.42.1 # via # boto3 # moto @@ -33,13 +33,13 @@ jmespath==1.0.1 # via # boto3 # botocore -joserfc==1.4.3 +joserfc==1.5.0 # via moto markupsafe==3.0.3 # via # jinja2 # werkzeug -moto[cognitoidp,dynamodb,s3]==5.1.17 +moto[cognitoidp,dynamodb,s3]==5.1.18 # via -r lambdas/python/staff-users/requirements-dev.in py-partiql-parser==0.6.3 # via moto @@ -60,7 +60,7 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil @@ -72,7 +72,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/compact-connect/lambdas/python/staff-users/requirements.txt b/backend/compact-connect/lambdas/python/staff-users/requirements.txt index d32dda42a..84457061f 100644 --- a/backend/compact-connect/lambdas/python/staff-users/requirements.txt +++ b/backend/compact-connect/lambdas/python/staff-users/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-users/requirements.in diff --git a/backend/compact-connect/requirements-dev.txt b/backend/compact-connect/requirements-dev.txt index 6ea674de6..007dddc8f 100644 --- a/backend/compact-connect/requirements-dev.txt +++ b/backend/compact-connect/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras requirements-dev.in @@ -22,7 +22,7 @@ coverage[toml]==7.12.0 # via # -r requirements-dev.in # pytest-cov -cyclonedx-python-lib==9.1.0 +cyclonedx-python-lib==11.6.0 # via pip-audit defusedxml==0.7.1 # via py-serializable @@ -42,7 +42,7 @@ mdurl==0.1.2 # via markdown-it-py msgpack==1.1.2 # via cachecontrol -packageurl-python==0.17.5 +packageurl-python==0.17.6 # via cyclonedx-python-lib packaging==25.0 # via @@ -52,7 +52,7 @@ packaging==25.0 # pytest pip-api==0.0.34 # via pip-audit -pip-audit==2.9.0 +pip-audit==2.10.0 # via -r requirements-dev.in pip-requirements-parser==32.0.1 # via pip-audit @@ -88,12 +88,16 @@ requests==2.32.5 # pip-audit rich==14.2.0 # via pip-audit -ruff==0.14.5 +ruff==0.14.7 # via -r requirements-dev.in sortedcontainers==2.4.0 # via cyclonedx-python-lib -toml==0.10.2 +tomli==2.3.0 # via pip-audit +tomli-w==1.2.0 + # via pip-audit +typing-extensions==4.15.0 + # via cyclonedx-python-lib tzdata==2025.2 # via faker urllib3==2.5.0 diff --git a/backend/compact-connect/requirements.txt b/backend/compact-connect/requirements.txt index 7ead839a2..39829962e 100644 --- a/backend/compact-connect/requirements.txt +++ b/backend/compact-connect/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.14 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras requirements.in @@ -12,11 +12,11 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-aws-lambda-python-alpha==2.225.0a0 +aws-cdk-aws-lambda-python-alpha==2.231.0a0 # via -r requirements.in aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.225.0 +aws-cdk-lib==2.231.0 # via # -r requirements.in # aws-cdk-aws-lambda-python-alpha @@ -33,7 +33,7 @@ constructs==10.4.3 # cdk-nag importlib-resources==6.5.2 # via jsii -jsii==1.119.0 +jsii==1.120.0 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 @@ -58,7 +58,7 @@ pyyaml==6.0.3 # via -r requirements.in six==1.17.0 # via python-dateutil -typeguard==4.2.1 +typeguard==2.13.3 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 From 98cb664a36889a32b110ca320ffc0276c7e8bcf1 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 16:59:04 -0600 Subject: [PATCH 037/137] Add test coverage for search related logic --- .../test_manage_opensearch_indices.py | 381 +++++++++++++++++- .../tests/function/test_search_providers.py | 325 +++++++++++++++ .../tests/unit/test_opensearch_client.py | 17 +- 3 files changed, 714 insertions(+), 9 deletions(-) create mode 100644 backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index d55e4ee30..03e5a757d 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock, call, patch + from moto import mock_aws from . import TstFunction @@ -10,4 +12,381 @@ class TestOpenSearchIndexManager(TstFunction): def setUp(self): super().setUp() - # TODO - add test cases for checking api calls + def _create_event(self, request_type: str, properties: dict = None) -> dict: + """Create a CloudFormation custom resource event.""" + return { + 'RequestType': request_type, + 'ResourceProperties': properties or {}, + } + + def _when_testing_mock_opensearch_client( + self, mock_opensearch_client, index_exists_return_value: bool | dict = False + ): + """ + Configure the mock OpenSearchClient for testing. + + :param mock_opensearch_client: The patched OpenSearchClient class + :param index_exists_return_value: Either a boolean (applied to all indices) + or a dict mapping index names to booleans + :return: The mock client instance + """ + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + + # If a dict is provided, use side_effect to return different values per index + if isinstance(index_exists_return_value, dict): + mock_client_instance.index_exists.side_effect = lambda index_name: index_exists_return_value.get( + index_name, False + ) + else: + mock_client_instance.index_exists.return_value = index_exists_return_value + + return mock_client_instance + + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_create_creates_indices_for_all_compacts_when_none_exist(self, mock_opensearch_client): + """Test that on_create creates indices for all compacts when they don't exist.""" + from handlers.manage_opensearch_indices import on_event + + # Set up the mock opensearch client - no indices exist + mock_client_instance = self._when_testing_mock_opensearch_client( + mock_opensearch_client, index_exists_return_value=False + ) + + # Create the event for a 'Create' request + event = self._create_event('Create') + + # Call the handler + on_event(event, self.mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that index_exists was called for each compact + expected_index_exists_calls = [ + call('compact_aslp_providers'), + call('compact_octp_providers'), + call('compact_coun_providers'), + ] + mock_client_instance.index_exists.assert_has_calls(expected_index_exists_calls, any_order=False) + self.assertEqual(3, mock_client_instance.index_exists.call_count) + + # Assert that create_index was called for each compact + self.assertEqual(3, mock_client_instance.create_index.call_count) + + # Verify the index names in create_index calls + create_index_calls = mock_client_instance.create_index.call_args_list + index_names_created = [call_args[0][0] for call_args in create_index_calls] + self.assertEqual( + ['compact_aslp_providers', 'compact_octp_providers', 'compact_coun_providers'], + index_names_created, + ) + + # Verify the mapping was passed to create_index + for call_args in create_index_calls: + index_mapping = call_args[0][1] + # Verify the mapping has the expected structure + self.assertEqual( + { + 'mappings': { + 'properties': { + 'birthMonthDay': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'compactConnectRegisteredEmailAddress': {'type': 'keyword'}, + 'compactEligibility': {'type': 'keyword'}, + 'currentHomeJurisdiction': {'type': 'keyword'}, + 'dateOfExpiration': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'familyName': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'givenName': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'jurisdictionUploadedCompactEligibility': {'type': 'keyword'}, + 'jurisdictionUploadedLicenseStatus': {'type': 'keyword'}, + 'licenseJurisdiction': {'type': 'keyword'}, + 'licenseStatus': {'type': 'keyword'}, + 'licenses': { + 'properties': { + 'adverseActions': { + 'properties': { + 'actionAgainst': {'type': 'keyword'}, + 'adverseActionId': {'type': 'keyword'}, + 'clinicalPrivilegeActionCategories': {'type': 'keyword'}, + 'clinicalPrivilegeActionCategory': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'creationDate': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'effectiveLiftDate': {'type': 'date'}, + 'effectiveStartDate': {'type': 'date'}, + 'encumbranceType': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'licenseTypeAbbreviation': {'type': 'keyword'}, + 'liftingUser': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'submittingUser': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'compact': {'type': 'keyword'}, + 'compactEligibility': {'type': 'keyword'}, + 'dateOfExpiration': {'type': 'date'}, + 'dateOfIssuance': {'type': 'date'}, + 'dateOfRenewal': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'emailAddress': {'type': 'keyword'}, + 'familyName': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'givenName': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'homeAddressCity': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'homeAddressPostalCode': {'type': 'keyword'}, + 'homeAddressState': {'type': 'keyword'}, + 'homeAddressStreet1': {'type': 'text'}, + 'homeAddressStreet2': {'type': 'text'}, + 'investigationStatus': {'type': 'keyword'}, + 'investigations': { + 'properties': { + 'compact': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + 'investigationId': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'status': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'jurisdiction': {'type': 'keyword'}, + 'jurisdictionUploadedCompactEligibility': {'type': 'keyword'}, + 'jurisdictionUploadedLicenseStatus': {'type': 'keyword'}, + 'licenseNumber': {'type': 'keyword'}, + 'licenseStatus': {'type': 'keyword'}, + 'licenseStatusName': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'middleName': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'npi': {'type': 'keyword'}, + 'phoneNumber': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'suffix': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'middleName': { + 'analyzer': 'custom_ascii_analyzer', + 'fields': {'keyword': {'ignore_above': 256, 'type': 'keyword'}}, + 'type': 'text', + }, + 'militaryAffiliations': { + 'properties': { + 'affiliationType': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + 'dateOfUpload': {'type': 'date'}, + 'fileNames': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'status': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'npi': {'type': 'keyword'}, + 'privilegeJurisdictions': {'type': 'keyword'}, + 'privileges': { + 'properties': { + 'activeSince': {'type': 'date'}, + 'administratorSetStatus': {'type': 'keyword'}, + 'adverseActions': { + 'properties': { + 'actionAgainst': {'type': 'keyword'}, + 'adverseActionId': {'type': 'keyword'}, + 'clinicalPrivilegeActionCategories': {'type': 'keyword'}, + 'clinicalPrivilegeActionCategory': {'type': 'keyword'}, + 'compact': {'type': 'keyword'}, + 'creationDate': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'effectiveLiftDate': {'type': 'date'}, + 'effectiveStartDate': {'type': 'date'}, + 'encumbranceType': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'licenseTypeAbbreviation': {'type': 'keyword'}, + 'liftingUser': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'submittingUser': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'attestations': { + 'properties': { + 'attestationId': {'type': 'keyword'}, + 'version': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'compact': {'type': 'keyword'}, + 'compactTransactionId': {'type': 'keyword'}, + 'dateOfExpiration': {'type': 'date'}, + 'dateOfIssuance': {'type': 'date'}, + 'dateOfRenewal': {'type': 'date'}, + 'dateOfUpdate': {'type': 'date'}, + 'investigationStatus': {'type': 'keyword'}, + 'investigations': { + 'properties': { + 'compact': {'type': 'keyword'}, + 'dateOfUpdate': {'type': 'date'}, + 'investigationId': {'type': 'keyword'}, + 'jurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'status': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'jurisdiction': {'type': 'keyword'}, + 'licenseJurisdiction': {'type': 'keyword'}, + 'licenseType': {'type': 'keyword'}, + 'privilegeId': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'status': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + }, + 'type': 'nested', + }, + 'providerDateOfUpdate': {'type': 'date'}, + 'providerFamGivMid': {'type': 'keyword'}, + 'providerId': {'type': 'keyword'}, + 'suffix': {'type': 'keyword'}, + 'type': {'type': 'keyword'}, + } + }, + 'settings': { + 'analysis': { + 'analyzer': { + 'custom_ascii_analyzer': { + 'filter': ['lowercase', 'custom_ascii_folding'], + 'tokenizer': 'standard', + 'type': 'custom', + } + }, + 'filter': {'custom_ascii_folding': {'preserve_original': True, 'type': 'asciifolding'}}, + }, + 'index': {'number_of_replicas': 0, 'number_of_shards': 1}, + }, + }, + index_mapping, + ) + + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_create_skips_index_creation_when_all_indices_exist(self, mock_opensearch_client): + """Test that on_create skips index creation when indices already exist.""" + from handlers.manage_opensearch_indices import on_event + + # Set up the mock opensearch client - all indices exist + mock_client_instance = self._when_testing_mock_opensearch_client( + mock_opensearch_client, index_exists_return_value=True + ) + + # Create the event for a 'Create' request + event = self._create_event('Create') + + # Call the handler + on_event(event, self.mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that index_exists was called for each compact + self.assertEqual(3, mock_client_instance.index_exists.call_count) + + # Assert that create_index was NOT called since indices already exist + mock_client_instance.create_index.assert_not_called() + + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_create_only_creates_missing_indices(self, mock_opensearch_client): + """Test that on_create only creates indices that don't exist.""" + from handlers.manage_opensearch_indices import on_event + + # Set up the mock opensearch client - only aslp index exists + mock_client_instance = self._when_testing_mock_opensearch_client( + mock_opensearch_client, + index_exists_return_value={ + 'compact_aslp_providers': True, + 'compact_octp_providers': False, + 'compact_coun_providers': False, + }, + ) + + # Create the event for a 'Create' request + event = self._create_event('Create') + + # Call the handler + on_event(event, self.mock_context) + + # Assert that index_exists was called for each compact + self.assertEqual(3, mock_client_instance.index_exists.call_count) + + # Assert that create_index was called only for missing indices (octp and coun) + self.assertEqual(2, mock_client_instance.create_index.call_count) + + # Verify the correct indices were created + create_index_calls = mock_client_instance.create_index.call_args_list + index_names_created = [call_args[0][0] for call_args in create_index_calls] + self.assertEqual(['compact_octp_providers', 'compact_coun_providers'], index_names_created) + + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_update_is_noop(self, mock_opensearch_client): + """Test that on_update does not create or modify indices.""" + from handlers.manage_opensearch_indices import on_event + + # Create the event for an 'Update' request + event = self._create_event('Update') + + # Call the handler + result = on_event(event, self.mock_context) + + # Assert that the OpenSearchClient was NOT instantiated + mock_opensearch_client.assert_not_called() + + # Result should be None (no-op) + self.assertIsNone(result) + + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_delete_is_noop(self, mock_opensearch_client): + """Test that on_delete does not delete indices.""" + from handlers.manage_opensearch_indices import on_event + + # Create the event for a 'Delete' request + event = self._create_event('Delete') + + # Call the handler + result = on_event(event, self.mock_context) + + # Assert that the OpenSearchClient was NOT instantiated + mock_opensearch_client.assert_not_called() + + # Result should be None (no-op) + self.assertIsNone(result) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py new file mode 100644 index 000000000..6e67816e5 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -0,0 +1,325 @@ +import json +from unittest.mock import Mock, patch + +from moto import mock_aws + +from . import TstFunction + + +@mock_aws +class TestSearchProviders(TstFunction): + """Test suite for search_providers handler.""" + + def setUp(self): + super().setUp() + + def _create_api_event(self, compact: str, body: dict = None) -> dict: + """Create a standard API Gateway event for search_providers.""" + return { + 'resource': f'/v1/compacts/{compact}/providers/search', + 'path': f'/v1/compacts/{compact}/providers/search', + 'httpMethod': 'POST', + 'headers': { + 'accept': 'application/json', + 'content-type': 'application/json', + 'Content-Type': 'application/json', + 'origin': 'https://example.org', + 'Host': 'api.test.example.com', + }, + 'multiValueHeaders': {}, + 'queryStringParameters': None, + 'pathParameters': {'compact': compact}, + 'requestContext': { + 'resourcePath': f'/v1/compacts/{compact}/providers/search', + 'httpMethod': 'POST', + 'authorizer': { + 'claims': { + 'sub': 'test-user-id', + 'cognito:username': 'test-user', + } + }, + }, + 'body': json.dumps(body) if body else None, + 'isBase64Encoded': False, + } + + def _when_testing_mock_opensearch_client(self, mock_opensearch_client, search_response: dict = None): + """ + Configure the mock OpenSearchClient for testing. + + :param mock_opensearch_client: The patched OpenSearchClient class + :param search_response: The response to return from the search method + :return: The mock client instance + """ + if not search_response: + search_response = { + 'hits': { + 'total': {'value': 0, 'relation': 'eq'}, + 'hits': [], + } + } + + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.search.return_value = search_response + return mock_client_instance + + def _create_mock_provider_hit( + self, + provider_id: str = '00000000-0000-0000-0000-000000000001', + compact: str = 'aslp', + sort_values: list = None, + ) -> dict: + """Create a mock OpenSearch hit for a provider document.""" + hit = { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + }, + } + if sort_values: + hit['sort'] = sort_values + return hit + + @patch('handlers.search_providers.OpenSearchClient') + def test_basic_search_with_match_all_query(self, mock_opensearch_client): + """Test that a basic search with no query uses match_all.""" + from handlers.search_providers import search_providers + + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create event with minimal body - just the required query field + event = self._create_api_event(compact='aslp', body={'query': {'match_all': {}}}) + + response = search_providers(event, self.mock_context) + + # Verify OpenSearchClient was instantiated and search was called + mock_opensearch_client.assert_called_once() + mock_client_instance.search.assert_called_once() + + # Verify the search was called with correct parameters + mock_client_instance.search.assert_called_once_with( + index_name='compact_aslp_providers', body={'query': {'match_all': {}}, 'size': 10} + ) + + # Verify response structure + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual({'providers': [], 'total': {'relation': 'eq', 'value': 0}}, body) + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_with_custom_query(self, mock_opensearch_client): + """Test that a custom OpenSearch query is passed through correctly.""" + from handlers.search_providers import search_providers + + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create a custom bool query + custom_query = { + 'bool': { + 'must': [ + {'match': {'givenName': 'John'}}, + {'term': {'licenseStatus': 'active'}}, + ] + } + } + event = self._create_api_event('aslp', body={'query': custom_query, 'from': 20}) + + search_providers(event, self.mock_context) + + # Verify the custom query was passed through + mock_client_instance.search.assert_called_once_with( + index_name='compact_aslp_providers', + body={ + 'query': {'bool': {'must': [{'match': {'givenName': 'John'}}, {'term': {'licenseStatus': 'active'}}]}}, + 'size': 10, + 'from': 20, + }, + ) + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_size_capped_at_max(self, mock_opensearch_client): + """Test that size parameter is capped at MAX_SIZE (100).""" + from handlers.search_providers import search_providers + + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Request size larger than MAX_SIZE + event = self._create_api_event('aslp', body={'query': {'match_all': {}}, 'size': 500}) + + search_providers(event, self.mock_context) + + call_args = mock_client_instance.search.call_args + search_body = call_args.kwargs['body'] + self.assertEqual(100, search_body['size']) # Capped at MAX_SIZE + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_with_sort_parameter(self, mock_opensearch_client): + """Test that sort parameter is included in the search body.""" + from handlers.search_providers import search_providers + + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + sort_config = [{'providerId': 'asc'}, {'dateOfUpdate': 'desc'}] + search_after_values = ['provider-uuid-123'] + event = self._create_api_event( + 'aslp', + body={ + 'query': {'match_all': {}}, + 'sort': sort_config, + 'search_after': search_after_values, + }, + ) + + search_providers(event, self.mock_context) + + mock_client_instance.search.assert_called_once_with( + index_name='compact_aslp_providers', + body={ + 'query': {'match_all': {}}, + 'size': 10, + 'sort': sort_config, + 'search_after': search_after_values, + }, + ) + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_after_without_sort_returns_400(self, mock_opensearch_client): + """Test that search_after without sort raises an error.""" + from handlers.search_providers import search_providers + + self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # search_after without sort should fail + event = self._create_api_event( + 'aslp', + body={ + 'query': {'match_all': {}}, + 'search_after': ['provider-uuid-123'], + }, + ) + + response = search_providers(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('sort is required when using search_after pagination', body['message']) + + def test_invalid_request_body_returns_400(self): + """Test that an invalid request body returns a 400 error.""" + from handlers.search_providers import search_providers + + # Create event with missing required 'query' field + event = self._create_api_event('aslp', body={'size': 10}) + + response = search_providers(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Invalid request', body['message']) + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_returns_sanitized_providers(self, mock_opensearch_client): + """Test that provider records are sanitized through ProviderGeneralResponseSchema.""" + from handlers.search_providers import search_providers + + # Create a mock response with provider hits + mock_hit = self._create_mock_provider_hit() + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [mock_hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_providers(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual( + { + 'providers': [ + { + 'birthMonthDay': '06-15', + 'compact': 'aslp', + 'compactEligibility': 'eligible', + 'dateOfExpiration': '2025-12-31', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'familyName': 'Doe', + 'givenName': 'John', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'privilegeJurisdictions': [], + 'providerId': '00000000-0000-0000-0000-000000000001', + 'type': 'provider', + } + ], + 'total': {'relation': 'eq', 'value': 1}, + }, + body, + ) + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_response_includes_last_sort_for_pagination(self, mock_opensearch_client): + """Test that lastSort is included in response for search_after pagination.""" + from handlers.search_providers import search_providers + + # Create hits with sort values + mock_hit = self._create_mock_provider_hit(sort_values=['provider-uuid-123', '2024-01-15T10:30:00+00:00']) + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [mock_hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event( + 'aslp', + body={ + 'query': {'match_all': {}}, + 'sort': [{'providerId': 'asc'}, {'dateOfUpdate': 'asc'}], + }, + ) + + response = search_providers(event, self.mock_context) + + body = json.loads(response['body']) + self.assertIn('lastSort', body) + self.assertEqual(['provider-uuid-123', '2024-01-15T10:30:00+00:00'], body['lastSort']) + + @patch('handlers.search_providers.OpenSearchClient') + def test_search_uses_correct_index_for_compact(self, mock_opensearch_client): + """Test that the correct index name is used based on the compact parameter.""" + from handlers.search_providers import search_providers + + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Test with different compacts + for compact in ['aslp', 'octp', 'coun']: + mock_client_instance.reset_mock() + + event = self._create_api_event(compact, body={'query': {'match_all': {}}}) + search_providers(event, self.mock_context) + + call_args = mock_client_instance.search.call_args + self.assertEqual(f'compact_{compact}_providers', call_args.kwargs['index_name']) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index b1d7d6c02..b63b4bc71 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -7,9 +7,11 @@ class TestOpenSearchClient(TestCase): def _create_client_with_mock(self): """Create an OpenSearchClient with a mocked internal client.""" - with patch('opensearch_client.boto3'), patch('opensearch_client.config'), patch( - 'opensearch_client.OpenSearch' - ) as mock_opensearch_class: + with ( + patch('opensearch_client.boto3'), + patch('opensearch_client.config'), + patch('opensearch_client.OpenSearch') as mock_opensearch_class, + ): mock_internal_client = MagicMock() mock_opensearch_class.return_value = mock_internal_client @@ -101,7 +103,10 @@ def test_bulk_index_calls_internal_client_with_expected_arguments(self): {'providerId': 'provider-1', 'givenName': 'John', 'familyName': 'Doe'}, {'providerId': 'provider-2', 'givenName': 'Jane', 'familyName': 'Smith'}, ] - expected_response = {'errors': False, 'items': [{'index': {'_id': 'provider-1'}}, {'index': {'_id': 'provider-2'}}]} + expected_response = { + 'errors': False, + 'items': [{'index': {'_id': 'provider-1'}}, {'index': {'_id': 'provider-2'}}], + } mock_internal_client.bulk.return_value = expected_response result = client.bulk_index(index_name=index_name, documents=documents) @@ -152,7 +157,3 @@ def test_bulk_index_returns_early_for_empty_documents(self): mock_internal_client.bulk.assert_not_called() self.assertEqual({'items': [], 'errors': False}, result) - - - - From 693962583cbba8a76c258372628139eb4c3e88cd Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 17:24:50 -0600 Subject: [PATCH 038/137] update domain engine version to latest --- .../stacks/search_persistent_stack/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 18a8579cc..0e026c22a 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -183,8 +183,7 @@ def __init__( self.domain = Domain( self, 'ProviderSearchDomain', - # TODO - set this to OPENSEARCH_3_1 after runtime migration PR is merged - version=EngineVersion.OPENSEARCH_2_19, + version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, # VPC configuration for network isolation vpc=vpc_stack.vpc, @@ -254,9 +253,10 @@ def __init__( 'es:ESHttpPost', ], # define all compact indices to restrict the policy to the search operation - resources=[Fn.join(delimiter='', - list_of_values=[self.domain.domain_arn, f'/compact_{compact}_providers/_search']) - for compact in persistent_stack.get_list_of_compact_abbreviations()], + resources=[ + Fn.join(delimiter='', list_of_values=[self.domain.domain_arn, f'/compact_{compact}_providers/_search']) + for compact in persistent_stack.get_list_of_compact_abbreviations() + ], ) # add access policy to restrict access to set of roles self.domain.add_access_policies( @@ -566,6 +566,11 @@ def _add_access_policy_lambda_suppressions(self): NagSuppressions.add_resource_suppressions( child, suppressions=[ + { + 'id': 'AwsSolutions-L1', + 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' + 'OpenSearch domain access policies. We cannot specify the runtime version.', + }, { 'id': 'AwsSolutions-IAM4', 'appliesTo': [ From e6f21d5eb56eda44bf609ab91c862b7ae849414c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 21:15:47 -0600 Subject: [PATCH 039/137] Extract domain definition into separate construct --- .../search_persistent_stack/__init__.py | 519 +--------------- .../provider_search_domain.py | 570 ++++++++++++++++++ .../tests/app/test_search_persistent_stack.py | 2 +- 3 files changed, 593 insertions(+), 498 deletions(-) create mode 100644 backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 0e026c22a..9f39ad030 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,21 +1,6 @@ -from aws_cdk import Duration, Fn, RemovalPolicy -from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Metric, TreatMissingData -from aws_cdk.aws_cloudwatch_actions import SnsAction -from aws_cdk.aws_ec2 import SubnetSelection, SubnetType -from aws_cdk.aws_iam import Effect, PolicyStatement, Role, ServicePrincipal +from aws_cdk import RemovalPolicy +from aws_cdk.aws_iam import Role, ServicePrincipal from aws_cdk.aws_kms import Key -from aws_cdk.aws_logs import LogGroup, ResourcePolicy, RetentionDays -from aws_cdk.aws_opensearchservice import ( - CapacityConfig, - Domain, - EbsOptions, - EncryptionAtRestOptions, - EngineVersion, - LoggingOptions, - TLSSecurityPolicy, - ZoneAwarenessConfig, -) -from cdk_nag import NagSuppressions from common_constructs.alarm_topic import AlarmTopic from common_constructs.stack import AppStack from constructs import Construct @@ -24,11 +9,9 @@ from stacks.persistent_stack import PersistentStack from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler +from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain from stacks.search_persistent_stack.search_providers_handler import SearchProvidersHandler -from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack - -PROD_EBS_VOLUME_SIZE = 25 -NON_PROD_EBS_VOLUME_SIZE = 10 +from stacks.vpc_stack import VpcStack class SearchPersistentStack(AppStack): @@ -67,23 +50,6 @@ def __init__( # Determine removal policy based on environment removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY - # Create dedicated KMS key for OpenSearch domain encryption - self.opensearch_encryption_key = Key( - self, - 'OpenSearchEncryptionKey', - enable_key_rotation=True, - alias=f'{self.stack_name}-opensearch-encryption-key', - removal_policy=removal_policy, - ) - - # Grant OpenSearch service principal permission to use the key - opensearch_principal = ServicePrincipal('es.amazonaws.com') - self.opensearch_encryption_key.grant_encrypt_decrypt(opensearch_principal) - - # Grant cloudwatch service principal permission to use the key - log_principal = ServicePrincipal('logs.amazonaws.com') - self.opensearch_encryption_key.grant_encrypt_decrypt(log_principal) - # Create IAM roles for Lambda functions that need OpenSearch access self.opensearch_ingest_lambda_role = Role( self, @@ -127,157 +93,30 @@ def __init__( slack_subscriptions=notifications.get('slack', []), ) - # Determine instance type and capacity based on environment - capacity_config = self._get_capacity_config(environment_name) - # determine AZ awareness based on environment - zone_awareness_config = self._get_zone_awareness_config(environment_name) - # Determine subnet selection based on environment - vpc_subnets = self._get_vpc_subnets(environment_name, vpc_stack) - - opensearch_app_log_group = LogGroup( - self, - 'OpensearchAppLogGroup', - retention=RetentionDays.ONE_MONTH, - removal_policy=removal_policy, - encryption_key=self.opensearch_encryption_key, - ) - slow_search_log_group = LogGroup( - self, - 'SlowSearchLogGroup', - retention=RetentionDays.ONE_MONTH, - removal_policy=removal_policy, - encryption_key=self.opensearch_encryption_key, - ) - slow_index_log_group = LogGroup( - self, - 'SlowIndexLogGroup', - retention=RetentionDays.ONE_MONTH, - removal_policy=removal_policy, - encryption_key=self.opensearch_encryption_key, - ) - - # Create CloudWatch Logs resource policy to allow OpenSearch to write logs - # This is done manually to avoid CDK creating an auto-generated Lambda function - # The resource ARNs must include ':*' to grant permissions on log streams within the log groups - ResourcePolicy( - self, - 'OpenSearchLogsResourcePolicy', - policy_statements=[ - PolicyStatement( - effect=Effect.ALLOW, - principals=[ServicePrincipal('es.amazonaws.com')], - actions=[ - 'logs:PutLogEvents', - 'logs:CreateLogStream', - ], - resources=[ - f'{opensearch_app_log_group.log_group_arn}:*', - f'{slow_search_log_group.log_group_arn}:*', - f'{slow_index_log_group.log_group_arn}:*', - ], - ), - ], - ) - - # Create OpenSearch Domain - self.domain = Domain( + # Create the OpenSearch domain and associated resources + self.provider_search_domain = ProviderSearchDomain( self, 'ProviderSearchDomain', - version=EngineVersion.OPENSEARCH_3_3, - capacity=capacity_config, - # VPC configuration for network isolation - vpc=vpc_stack.vpc, - vpc_subnets=[vpc_subnets], - security_groups=[vpc_stack.opensearch_security_group], - # EBS volume configuration - ebs=EbsOptions( - enabled=True, - volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, - ), - # Encryption settings - encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.opensearch_encryption_key), - node_to_node_encryption=True, - enforce_https=True, - tls_security_policy=TLSSecurityPolicy.TLS_1_2, - # Advanced security options - advanced_options={ - # Prevent queries from accessing multiple indices in a single request - # This is a security control to ensure queries are scoped to a single index - 'rest.action.multi.allow_explicit_index': 'false', - }, - logging=LoggingOptions( - app_log_enabled=True, - app_log_group=opensearch_app_log_group, - slow_search_log_enabled=True, - slow_search_log_group=slow_search_log_group, - slow_index_log_enabled=True, - slow_index_log_group=slow_index_log_group, - ), - # Suppress auto-generated Lambda for log resource policy (we created it manually above) - suppress_logs_resource_policy=True, - # Domain removal policy - removal_policy=removal_policy, - zone_awareness=zone_awareness_config, - ) - - opensearch_ingest_access_policy = PolicyStatement( - effect=Effect.ALLOW, - principals=[self.opensearch_ingest_lambda_role], - actions=[ - 'es:ESHttpPost', - 'es:ESHttpPut', - ], - resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], - ) - opensearch_index_manager_access_policy = PolicyStatement( - effect=Effect.ALLOW, - principals=[self.opensearch_index_manager_lambda_role], - actions=[ - 'es:ESHttpGet', - 'es:ESHttpHead', # Required for index_exists() checks - 'es:ESHttpPost', - 'es:ESHttpPut', - ], - resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], - ) - # Search API policy - restricted to _search endpoint only - # POST is required for _search queries even though they are read-only operations - # because OpenSearch's search API uses POST to send the query DSL body. - # By restricting the resource to /_search, we prevent POST from being used - # for document indexing or other write operations. - # See: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html - opensearch_search_api_policy = PolicyStatement( - effect=Effect.ALLOW, - principals=[self.search_api_lambda_role], - actions=[ - 'es:ESHttpPost', - ], - # define all compact indices to restrict the policy to the search operation - resources=[ - Fn.join(delimiter='', list_of_values=[self.domain.domain_arn, f'/compact_{compact}_providers/_search']) - for compact in persistent_stack.get_list_of_compact_abbreviations() - ], - ) - # add access policy to restrict access to set of roles - self.domain.add_access_policies( - opensearch_ingest_access_policy, - opensearch_index_manager_access_policy, - opensearch_search_api_policy, + environment_name=environment_name, + vpc_stack=vpc_stack, + compact_abbreviations=persistent_stack.get_list_of_compact_abbreviations(), + alarm_topic=self.alarm_topic, + ingest_lambda_role=self.opensearch_ingest_lambda_role, + index_manager_lambda_role=self.opensearch_index_manager_lambda_role, + search_api_lambda_role=self.search_api_lambda_role, ) - # CDK creates a lambda function to manage the access policies, we need to add suppressions for it - self._add_access_policy_lambda_suppressions() - # grant lambda roles access to domain - self.domain.grant_read(self.search_api_lambda_role) - self.domain.grant_write(self.opensearch_ingest_lambda_role) - self.domain.grant_read_write(self.opensearch_index_manager_lambda_role) + # Expose domain and encryption key for use by other constructs + self.domain = self.provider_search_domain.domain + self.opensearch_encryption_key = self.provider_search_domain.encryption_key + # Create the index manager custom resource self.index_manager_custom_resource = IndexManagerCustomResource( self, construct_id='indexManager', - opensearch_domain=self.domain, + opensearch_domain=self.provider_search_domain.domain, vpc_stack=vpc_stack, - vpc_subnets=vpc_subnets, + vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_index_manager_lambda_role, ) @@ -285,9 +124,9 @@ def __init__( self.search_providers_handler = SearchProvidersHandler( self, construct_id='searchProvidersHandler', - opensearch_domain=self.domain, + opensearch_domain=self.provider_search_domain.domain, vpc_stack=vpc_stack, - vpc_subnets=vpc_subnets, + vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.search_api_lambda_role, alarm_topic=self.alarm_topic, ) @@ -299,322 +138,8 @@ def __init__( construct_id='populateProviderDocumentsHandler', opensearch_domain=self.domain, vpc_stack=vpc_stack, - vpc_subnets=vpc_subnets, + vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, alarm_topic=self.alarm_topic, ) - - # Add CDK Nag suppressions for OpenSearch Domain - self._add_opensearch_suppressions(environment_name) - - # Add capacity monitoring alarms for proactive scaling - self._add_capacity_alarms(environment_name) - - # Add CDK Nag suppressions for Index Manager Custom Resource - self._add_opensearch_lambda_role_suppressions(self.search_api_lambda_role) - self._add_opensearch_lambda_role_suppressions(self.opensearch_ingest_lambda_role) - - def _get_capacity_config(self, environment_name: str) -> CapacityConfig: - """ - Determine OpenSearch cluster capacity configuration based on environment. - - Non-prod (sandbox, test, beta, etc.): Single t3.small.search node - Prod: 3 dedicated master (m7g.medium.search) + 3 data nodes (m7g.medium.search) with standby - - param environment_name: The deployment environment name - - return: CapacityConfig with appropriate instance types and counts - """ - if environment_name == PROD_ENV_NAME: - # Production configuration with high availability - # 3 dedicated master nodes + 3 data nodes across 3 AZs with standby - # Multi-AZ with standby does not support t3 instance types - return CapacityConfig( - # Data nodes - m7g.medium provides 4 vCPUs and 8GB RAM - data_node_instance_type='m7g.medium.search', - data_nodes=3, - # Dedicated master nodes for cluster management - master_node_instance_type='m7g.medium.search', - master_nodes=3, - # Multi-AZ with standby for high availability - multi_az_with_standby_enabled=True, - ) - - # Single node configuration for all non-prod environments - # (test, beta, and developer sandboxes) - return CapacityConfig( - data_node_instance_type='t3.small.search', - data_nodes=1, - # No dedicated master nodes for single-node clusters - master_nodes=None, - # No multi-AZ for single node - multi_az_with_standby_enabled=False, - ) - - def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConfig: - """ - Determine OpenSearch cluster availability zone awareness based on environment. - - 3 for production, not enabled for all other non-prod environments - - param environment_name: The deployment environment name - - return: ZoneAwarenessConfig with appropriate settings - """ - if environment_name == PROD_ENV_NAME: - return ZoneAwarenessConfig(enabled=True, availability_zone_count=3) - - # non-prod environments only use one data node, hence we don't enable zone awareness - return ZoneAwarenessConfig(enabled=False) - - def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> SubnetSelection: - """ - Determine VPC subnet selection based on environment. - - Production: All private isolated subnets (3 AZs) for zone awareness and high availability - Non-prod: Single subnet (privateSubnet1 with CIDR 10.0.0.0/20) for single-node deployment - - param environment_name: The deployment environment name - param vpc_stack: The VPC stack containing the private subnets - - return: List of SubnetSelection with appropriate subnet configuration - """ - if environment_name == PROD_ENV_NAME: - # Production: Use all private isolated subnets from the VPC. - # VPC is configured with max_azs=3, so this will select exactly 3 subnets - return SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED) - - # Non-prod: Single-node deployment explicitly uses privateSubnet1 (CIDR 10.0.0.0/20) - # OpenSearch requires exactly one subnet for single-node deployments - # We explicitly find the subnet by its construct name to guarantee consistency - private_subnet1 = self._find_subnet_by_name(vpc_stack.vpc, PRIVATE_SUBNET_ONE_NAME) - return SubnetSelection(subnets=[private_subnet1]) - - def _find_subnet_by_name(self, vpc, subnet_name: str): - """ - Find a specific subnet by its logical construct name in the VPC. - - This provides a guaranteed, explicit reference to a specific subnet regardless of - CDK's internal list ordering, which is critical for stateful resources like OpenSearch. - - param vpc: The VPC construct containing the subnet - param subnet_name: The logical name of the subnet (e.g., 'privateSubnet1') - - return: The ISubnet instance - - raises ValueError: If the subnet cannot be found - """ - # Navigate the construct tree to find the subnet by name - subnet_construct = vpc.node.try_find_child(subnet_name) - if subnet_construct is None: - raise ValueError( - f'Subnet {subnet_name} not found in VPC construct tree. ' - f'Available children: {[c.node.id for c in vpc.node.children]}' - ) - - return subnet_construct - - def _add_capacity_alarms(self, environment_name: str): - """ - Add CloudWatch alarms to monitor OpenSearch capacity and alert before hitting limits. - - These proactive thresholds give the DevOps team time to plan scaling activities: - - Free Storage Space < 50% of allocated capacity - - JVM Memory Pressure > 70% - - CPU Utilization > 60% - - param environment_name: The deployment environment name - """ - # Get the volume size for calculating storage threshold - volume_size_gb = PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE - # 50% threshold in MB (FreeStorageSpace metric is reported in megabytes) - # Formula: GB * 1024 MB/GB * 0.5 for 50% threshold - storage_threshold_mb = volume_size_gb * 1024 * 0.5 - - # Alarm: Free Storage Space < 50% - # This gives ample time to plan capacity increases before hitting critical levels - # Note: FreeStorageSpace metric is reported in megabytes (MB) - Alarm( - self, - 'FreeStorageSpaceAlarm', - metric=Metric( - namespace='AWS/ES', - metric_name='FreeStorageSpace', - dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, - # check twice a day - period=Duration.hours(12), - statistic='Minimum', - ), - evaluation_periods=1, # Notify the moment the storage space is less than 50% - threshold=storage_threshold_mb, - comparison_operator=ComparisonOperator.LESS_THAN_THRESHOLD, - treat_missing_data=TreatMissingData.NOT_BREACHING, - alarm_description=( - f'OpenSearch Domain {self.domain.domain_name} free storage space has dropped below 50% ' - f'({storage_threshold_mb}MB of {volume_size_gb * 1024}MB allocated EBS volume). ' - 'Consider planning to increase EBS volume size or scaling the cluster.' - ), - ).add_alarm_action(SnsAction(self.alarm_topic)) - - # Alarm: JVM Memory Pressure > 70% - # Sustained high memory pressure indicates need for instance scaling - Alarm( - self, - 'JVMMemoryPressureAlarm', - metric=Metric( - namespace='AWS/ES', - metric_name='JVMMemoryPressure', - dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, - period=Duration.minutes(5), - statistic='Maximum', - ), - evaluation_periods=6, # 30 minutes sustained - threshold=70, - comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=TreatMissingData.NOT_BREACHING, - alarm_description=( - f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 70%. ' - 'This indicates the cluster is using a significant portion of its heap memory. ' - 'Consider scaling to larger instance types if pressure continues to increase.' - ), - ).add_alarm_action(SnsAction(self.alarm_topic)) - - # Alarm: CPU Utilization > 60% - # Sustained high CPU indicates need for more compute capacity - Alarm( - self, - 'CPUUtilizationAlarm', - metric=Metric( - namespace='AWS/ES', - metric_name='CPUUtilization', - dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': self.account}, - period=Duration.minutes(5), - statistic='Average', - ), - evaluation_periods=3, # 15 minutes sustained - threshold=60, - comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=TreatMissingData.NOT_BREACHING, - alarm_description=( - f'OpenSearch Domain {self.domain.domain_name} CPU utilization has been above 60% for 15 minutes. ' - 'This indicates sustained high load. Review metrics and consider scaling to larger instance types ' - 'or adding more data nodes to distribute the load.' - ), - ).add_alarm_action(SnsAction(self.alarm_topic)) - - def _add_opensearch_suppressions(self, environment_name: str): - """ - Add CDK Nag suppressions for OpenSearch Domain configuration. - - Some security best practices are not applicable or will be implemented later: - - Fine-grained access control: Will be added with full API implementation - - Access policies: Will be configured when Lambda functions are added - - Dedicated master nodes: Only needed for prod (>3 nodes) - """ - NagSuppressions.add_resource_suppressions( - self.domain, - suppressions=[ - { - 'id': 'AwsSolutions-OS3', - 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups. ' - 'The data in the domain is only accessible by the ingest lambda which indexes the ' - 'documents and the search API lambda which can only be accessed by authenticated staff ' - 'users in CompactConnect.', - }, - { - 'id': 'AwsSolutions-OS5', - 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups. ' - 'The data in the domain is only accessible by the ingest lambda which indexes the ' - 'documents and the search API lambda which can only be accessed by authenticated staff ' - 'users in CompactConnect.', - }, - ], - apply_to_children=True, - ) - if environment_name != PROD_ENV_NAME: - NagSuppressions.add_resource_suppressions( - self.domain, - suppressions=[ - { - 'id': 'AwsSolutions-OS4', - 'reason': 'Dedicated master nodes are only used in production environments with multiple data ' - 'nodes. Single-node non-prod environments do not require dedicated master nodes.', - }, - { - 'id': 'AwsSolutions-OS7', - 'reason': 'Zone awareness with standby is only enabled for production environments with ' - 'multiple nodes. Single-node test environments do not require multi-AZ ' - 'configuration.', - }, - ], - apply_to_children=True, - ) - - def _add_access_policy_lambda_suppressions(self): - """ - Add CDK Nag suppressions for the auto-generated Lambda function created by add_access_policies. - - The CDK Domain.add_access_policies() method creates an AwsCustomResource Lambda to manage - the domain's access policy. CDK generates these with IDs starting with 'AWS' followed by a hash. - We find these dynamically to avoid relying on a specific hash value. - """ - # Find auto-generated Lambda constructs by looking for children with IDs starting with 'AWS' - # These are created by CDK's AwsCustomResource for managing domain access policies - for child in self.node.children: - if child.node.id.startswith('AWS'): - NagSuppressions.add_resource_suppressions( - child, - suppressions=[ - { - 'id': 'AwsSolutions-L1', - 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' - 'OpenSearch domain access policies. We cannot specify the runtime version.', - }, - { - 'id': 'AwsSolutions-IAM4', - 'appliesTo': [ - 'Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' - ], - 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' - 'OpenSearch domain access policies. It uses the standard execution role.', - }, - { - 'id': 'AwsSolutions-IAM5', - 'appliesTo': ['Action::kms:Describe*', 'Action::kms:List*'], - 'reason': 'This is an AWS-managed custom resource Lambda that requires KMS permissions to ' - 'access the encryption key used by the OpenSearch domain.', - }, - { - 'id': 'HIPAA.Security-LambdaDLQ', - 'reason': 'This is an AWS-managed custom resource Lambda used only during deployment to ' - 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' - 'functions.', - }, - { - 'id': 'HIPAA.Security-LambdaInsideVPC', - 'reason': 'This is an AWS-managed custom resource Lambda that needs internet access to ' - 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' - 'required.', - }, - ], - apply_to_children=True, - ) - - def _add_opensearch_lambda_role_suppressions(self, lambda_role: Role): - """ - Add CDK Nag suppressions for OpenSearch Lambda role configuration. - - param environment_name: The deployment environment name - """ - NagSuppressions.add_resource_suppressions( - lambda_role, - suppressions=[ - { - 'id': 'AwsSolutions-IAM5', - 'reason': 'This lambda role access is restricted to the specific ' - 'OpenSearch domain and its indices within the VPC.', - }, - ], - apply_to_children=True, - ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py new file mode 100644 index 000000000..4b1e8e2e9 --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -0,0 +1,570 @@ +from aws_cdk import Duration, Fn, RemovalPolicy +from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Metric, TreatMissingData +from aws_cdk.aws_cloudwatch_actions import SnsAction +from aws_cdk.aws_ec2 import SubnetSelection, SubnetType +from aws_cdk.aws_iam import Effect, IRole, PolicyStatement, ServicePrincipal +from aws_cdk.aws_kms import Key +from aws_cdk.aws_logs import LogGroup, ResourcePolicy, RetentionDays +from aws_cdk.aws_opensearchservice import ( + CapacityConfig, + Domain, + EbsOptions, + EncryptionAtRestOptions, + EngineVersion, + LoggingOptions, + TLSSecurityPolicy, + ZoneAwarenessConfig, +) +from aws_cdk.aws_sns import ITopic +from cdk_nag import NagSuppressions +from common_constructs.stack import Stack +from constructs import Construct + +from common_constructs.constants import PROD_ENV_NAME +from stacks.vpc_stack import PRIVATE_SUBNET_ONE_NAME, VpcStack + +PROD_EBS_VOLUME_SIZE = 25 +NON_PROD_EBS_VOLUME_SIZE = 10 + + +class ProviderSearchDomain(Construct): + """ + Construct for the OpenSearch Domain and related resources. + + This construct encapsulates: + - OpenSearch Domain with VPC deployment and encryption + - KMS encryption key for the domain + - CloudWatch log groups for OpenSearch logging + - Access policies restricting domain access to specific Lambda roles + - CloudWatch alarms for capacity monitoring + + Instance sizing by environment: + - Non-prod (sandbox/test/beta): t3.small.search, 1 node + - Prod: m7g.medium.search, 3 master + 3 data nodes (with standby) + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + *, + environment_name: str, + vpc_stack: VpcStack, + compact_abbreviations: list[str], + alarm_topic: ITopic, + ingest_lambda_role: IRole, + index_manager_lambda_role: IRole, + search_api_lambda_role: IRole, + ): + """ + Initialize the ProviderSearchDomain construct. + + :param scope: The scope of the construct + :param construct_id: The id of the construct + :param environment_name: The deployment environment name (e.g., 'prod', 'test') + :param vpc_stack: The VPC stack containing network resources + :param compact_abbreviations: List of compact abbreviations for index access policies + :param alarm_topic: The SNS topic for capacity alarms + :param ingest_lambda_role: IAM role for the ingest Lambda function (write access) + :param index_manager_lambda_role: IAM role for the index manager Lambda function (read/write access) + :param search_api_lambda_role: IAM role for the search API Lambda function (read access) + """ + super().__init__(scope, construct_id) + stack = Stack.of(self) + + # Store references to the Lambda roles for access policy configuration + self._ingest_lambda_role = ingest_lambda_role + self._index_manager_lambda_role = index_manager_lambda_role + self._search_api_lambda_role = search_api_lambda_role + + # Determine removal policy based on environment + removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY + + # Create dedicated KMS key for OpenSearch domain encryption + self.encryption_key = Key( + self, + 'EncryptionKey', + enable_key_rotation=True, + alias=f'{stack.stack_name}-opensearch-encryption-key', + removal_policy=removal_policy, + ) + + # Grant OpenSearch service principal permission to use the key + opensearch_principal = ServicePrincipal('es.amazonaws.com') + self.encryption_key.grant_encrypt_decrypt(opensearch_principal) + + # Grant cloudwatch service principal permission to use the key + log_principal = ServicePrincipal('logs.amazonaws.com') + self.encryption_key.grant_encrypt_decrypt(log_principal) + + # Create CloudWatch log groups for OpenSearch logging + app_log_group = LogGroup( + self, + 'AppLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + encryption_key=self.encryption_key, + ) + slow_search_log_group = LogGroup( + self, + 'SlowSearchLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + encryption_key=self.encryption_key, + ) + slow_index_log_group = LogGroup( + self, + 'SlowIndexLogGroup', + retention=RetentionDays.ONE_MONTH, + removal_policy=removal_policy, + encryption_key=self.encryption_key, + ) + + # Create CloudWatch Logs resource policy to allow OpenSearch to write logs + # This is done manually to avoid CDK creating an auto-generated Lambda function + # The resource ARNs must include ':*' to grant permissions on log streams within the log groups + ResourcePolicy( + self, + 'LogsResourcePolicy', + policy_statements=[ + PolicyStatement( + effect=Effect.ALLOW, + principals=[ServicePrincipal('es.amazonaws.com')], + actions=[ + 'logs:PutLogEvents', + 'logs:CreateLogStream', + ], + resources=[ + f'{app_log_group.log_group_arn}:*', + f'{slow_search_log_group.log_group_arn}:*', + f'{slow_index_log_group.log_group_arn}:*', + ], + ), + ], + ) + + # Determine instance type and capacity based on environment + capacity_config = self._get_capacity_config(environment_name) + # Determine AZ awareness based on environment + zone_awareness_config = self._get_zone_awareness_config(environment_name) + # Determine subnet selection based on environment + self.vpc_subnets = self._get_vpc_subnets(environment_name, vpc_stack) + + # Create OpenSearch Domain + self.domain = Domain( + self, + 'Domain', + version=EngineVersion.OPENSEARCH_3_1, + capacity=capacity_config, + # VPC configuration for network isolation + vpc=vpc_stack.vpc, + vpc_subnets=[self.vpc_subnets], + security_groups=[vpc_stack.opensearch_security_group], + # EBS volume configuration + ebs=EbsOptions( + enabled=True, + volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, + ), + # Encryption settings + encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.encryption_key), + node_to_node_encryption=True, + enforce_https=True, + tls_security_policy=TLSSecurityPolicy.TLS_1_2, + # Advanced security options + advanced_options={ + # Prevent queries from accessing multiple indices in a single request + # This is a security control to ensure queries are scoped to a single index + 'rest.action.multi.allow_explicit_index': 'false', + }, + logging=LoggingOptions( + app_log_enabled=True, + app_log_group=app_log_group, + slow_search_log_enabled=True, + slow_search_log_group=slow_search_log_group, + slow_index_log_enabled=True, + slow_index_log_group=slow_index_log_group, + ), + # Suppress auto-generated Lambda for log resource policy (we created it manually above) + suppress_logs_resource_policy=True, + # Domain removal policy + removal_policy=removal_policy, + zone_awareness=zone_awareness_config, + ) + + # Configure access policies + self._configure_access_policies(compact_abbreviations) + + # Grant lambda roles access to domain + self.domain.grant_read(self._search_api_lambda_role) + self.domain.grant_write(self._ingest_lambda_role) + self.domain.grant_read_write(self._index_manager_lambda_role) + + # Add CDK Nag suppressions + self._add_domain_suppressions(environment_name) + self._add_access_policy_lambda_suppressions() + self._add_lambda_role_suppressions(self._search_api_lambda_role) + self._add_lambda_role_suppressions(self._ingest_lambda_role) + + # Add capacity monitoring alarms + self._add_capacity_alarms(environment_name, alarm_topic) + + def _configure_access_policies(self, compact_abbreviations: list[str]): + """ + Configure access policies for the OpenSearch domain. + + Creates IAM-based access policies that restrict access to specific Lambda roles: + - Ingest role: POST/PUT access to compact indices + - Index manager role: GET/HEAD/POST/PUT access for index management + - Search API role: POST access restricted to _search endpoint only + + :param compact_abbreviations: List of compact abbreviations for index access policies + """ + ingest_access_policy = PolicyStatement( + effect=Effect.ALLOW, + principals=[self._ingest_lambda_role], + actions=[ + 'es:ESHttpPost', + 'es:ESHttpPut', + ], + resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + ) + index_manager_access_policy = PolicyStatement( + effect=Effect.ALLOW, + principals=[self._index_manager_lambda_role], + actions=[ + 'es:ESHttpGet', + 'es:ESHttpHead', # Required for index_exists() checks + 'es:ESHttpPost', + 'es:ESHttpPut', + ], + resources=[Fn.join('', [self.domain.domain_arn, '/compact*'])], + ) + # Search API policy - restricted to _search endpoint only + # POST is required for _search queries even though they are read-only operations + # because OpenSearch's search API uses POST to send the query DSL body. + # By restricting the resource to /_search, we prevent POST from being used + # for document indexing or other write operations. + # See: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html + search_api_policy = PolicyStatement( + effect=Effect.ALLOW, + principals=[self._search_api_lambda_role], + actions=[ + 'es:ESHttpPost', + ], + # define all compact indices to restrict the policy to the search operation + resources=[ + Fn.join(delimiter='', list_of_values=[self.domain.domain_arn, f'/compact_{compact}_providers/_search']) + for compact in compact_abbreviations + ], + ) + # Add access policy to restrict access to set of roles + self.domain.add_access_policies( + ingest_access_policy, + index_manager_access_policy, + search_api_policy, + ) + + def _get_capacity_config(self, environment_name: str) -> CapacityConfig: + """ + Determine OpenSearch cluster capacity configuration based on environment. + + Non-prod (sandbox, test, beta, etc.): Single t3.small.search node + Prod: 3 dedicated master (m7g.medium.search) + 3 data nodes (m7g.medium.search) with standby + + :param environment_name: The deployment environment name + :return: CapacityConfig with appropriate instance types and counts + """ + if environment_name == PROD_ENV_NAME: + # Production configuration with high availability + # 3 dedicated master nodes + 3 data nodes across 3 AZs with standby + # Multi-AZ with standby does not support t3 instance types + return CapacityConfig( + # Data nodes - m7g.medium provides 4 vCPUs and 8GB RAM + data_node_instance_type='m7g.medium.search', + data_nodes=3, + # Dedicated master nodes for cluster management + master_node_instance_type='m7g.medium.search', + master_nodes=3, + # Multi-AZ with standby for high availability + multi_az_with_standby_enabled=True, + ) + + # Single node configuration for all non-prod environments + # (test, beta, and developer sandboxes) + return CapacityConfig( + data_node_instance_type='t3.small.search', + data_nodes=1, + # No dedicated master nodes for single-node clusters + master_nodes=None, + # No multi-AZ for single node + multi_az_with_standby_enabled=False, + ) + + def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConfig: + """ + Determine OpenSearch cluster availability zone awareness based on environment. + + 3 for production, not enabled for all other non-prod environments + + :param environment_name: The deployment environment name + :return: ZoneAwarenessConfig with appropriate settings + """ + if environment_name == PROD_ENV_NAME: + return ZoneAwarenessConfig(enabled=True, availability_zone_count=3) + + # Non-prod environments only use one data node, hence we don't enable zone awareness + return ZoneAwarenessConfig(enabled=False) + + def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> SubnetSelection: + """ + Determine VPC subnet selection based on environment. + + Production: All private isolated subnets (3 AZs) for zone awareness and high availability + Non-prod: Single subnet (privateSubnet1 with CIDR 10.0.0.0/20) for single-node deployment + + :param environment_name: The deployment environment name + :param vpc_stack: The VPC stack containing the private subnets + :return: SubnetSelection with appropriate subnet configuration + """ + if environment_name == PROD_ENV_NAME: + # Production: Use all private isolated subnets from the VPC. + # VPC is configured with max_azs=3, so this will select exactly 3 subnets + return SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED) + + # Non-prod: Single-node deployment explicitly uses privateSubnet1 (CIDR 10.0.0.0/20) + # OpenSearch requires exactly one subnet for single-node deployments + # We explicitly find the subnet by its construct name to guarantee consistency + private_subnet1 = self._find_subnet_by_name(vpc_stack.vpc, PRIVATE_SUBNET_ONE_NAME) + return SubnetSelection(subnets=[private_subnet1]) + + def _find_subnet_by_name(self, vpc, subnet_name: str): + """ + Find a specific subnet by its logical construct name in the VPC. + + This provides a guaranteed, explicit reference to a specific subnet regardless of + CDK's internal list ordering, which is critical for stateful resources like OpenSearch. + + :param vpc: The VPC construct containing the subnet + :param subnet_name: The logical name of the subnet (e.g., 'privateSubnet1') + :return: The ISubnet instance + :raises ValueError: If the subnet cannot be found + """ + # Navigate the construct tree to find the subnet by name + subnet_construct = vpc.node.try_find_child(subnet_name) + if subnet_construct is None: + raise ValueError( + f'Subnet {subnet_name} not found in VPC construct tree. ' + f'Available children: {[c.node.id for c in vpc.node.children]}' + ) + + return subnet_construct + + def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): + """ + Add CloudWatch alarms to monitor OpenSearch capacity and alert before hitting limits. + + These proactive thresholds give the DevOps team time to plan scaling activities: + - Free Storage Space < 50% of allocated capacity + - JVM Memory Pressure > 70% + - CPU Utilization > 60% + + :param environment_name: The deployment environment name + :param alarm_topic: The SNS topic to send alarm notifications to + """ + stack = Stack.of(self) + + # Get the volume size for calculating storage threshold + volume_size_gb = PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE + # 50% threshold in MB (FreeStorageSpace metric is reported in megabytes) + # Formula: GB * 1024 MB/GB * 0.5 for 50% threshold + storage_threshold_mb = volume_size_gb * 1024 * 0.5 + + # Alarm: Free Storage Space < 50% + # This gives ample time to plan capacity increases before hitting critical levels + # Note: FreeStorageSpace metric is reported in megabytes (MB) + Alarm( + self, + 'FreeStorageSpaceAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='FreeStorageSpace', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, + # check twice a day + period=Duration.hours(12), + statistic='Minimum', + ), + evaluation_periods=1, # Notify the moment the storage space is less than 50% + threshold=storage_threshold_mb, + comparison_operator=ComparisonOperator.LESS_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} free storage space has dropped below 50% ' + f'({storage_threshold_mb}MB of {volume_size_gb * 1024}MB allocated EBS volume). ' + 'Consider planning to increase EBS volume size or scaling the cluster.' + ), + ).add_alarm_action(SnsAction(alarm_topic)) + + # Alarm: JVM Memory Pressure > 70% + # Sustained high memory pressure indicates need for instance scaling + Alarm( + self, + 'JVMMemoryPressureAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='JVMMemoryPressure', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, + period=Duration.minutes(5), + statistic='Maximum', + ), + evaluation_periods=6, # 30 minutes sustained + threshold=70, + comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 70%. ' + 'This indicates the cluster is using a significant portion of its heap memory. ' + 'Consider scaling to larger instance types if pressure continues to increase.' + ), + ).add_alarm_action(SnsAction(alarm_topic)) + + # Alarm: CPU Utilization > 60% + # Sustained high CPU indicates need for more compute capacity + Alarm( + self, + 'CPUUtilizationAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='CPUUtilization', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, + period=Duration.minutes(5), + statistic='Average', + ), + evaluation_periods=3, # 15 minutes sustained + threshold=60, + comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} CPU utilization has been above 60% for 15 minutes. ' + 'This indicates sustained high load. Review metrics and consider scaling to larger instance types ' + 'or adding more data nodes to distribute the load.' + ), + ).add_alarm_action(SnsAction(alarm_topic)) + + def _add_domain_suppressions(self, environment_name: str): + """ + Add CDK Nag suppressions for OpenSearch Domain configuration. + + Some security best practices are not applicable or will be implemented later: + - Fine-grained access control: Will be added with full API implementation + - Access policies: Will be configured when Lambda functions are added + - Dedicated master nodes: Only needed for prod (>3 nodes) + """ + NagSuppressions.add_resource_suppressions( + self.domain, + suppressions=[ + { + 'id': 'AwsSolutions-OS3', + 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups. ' + 'The data in the domain is only accessible by the ingest lambda which indexes the ' + 'documents and the search API lambda which can only be accessed by authenticated staff ' + 'users in CompactConnect.', + }, + { + 'id': 'AwsSolutions-OS5', + 'reason': 'Access to this domain is restricted by Access Policies and VPC security groups. ' + 'The data in the domain is only accessible by the ingest lambda which indexes the ' + 'documents and the search API lambda which can only be accessed by authenticated staff ' + 'users in CompactConnect.', + }, + ], + apply_to_children=True, + ) + if environment_name != PROD_ENV_NAME: + NagSuppressions.add_resource_suppressions( + self.domain, + suppressions=[ + { + 'id': 'AwsSolutions-OS4', + 'reason': 'Dedicated master nodes are only used in production environments with multiple data ' + 'nodes. Single-node non-prod environments do not require dedicated master nodes.', + }, + { + 'id': 'AwsSolutions-OS7', + 'reason': 'Zone awareness with standby is only enabled for production environments with ' + 'multiple nodes. Single-node test environments do not require multi-AZ ' + 'configuration.', + }, + ], + apply_to_children=True, + ) + + def _add_access_policy_lambda_suppressions(self): + """ + Add CDK Nag suppressions for the auto-generated Lambda function created by add_access_policies. + + The CDK Domain.add_access_policies() method creates an AwsCustomResource Lambda to manage + the domain's access policy. CDK generates these with IDs starting with 'AWS' followed by a hash. + We find these dynamically to avoid relying on a specific hash value. + """ + stack = Stack.of(self) + + # Find auto-generated Lambda constructs by looking for children with IDs starting with 'AWS' + # These are created by CDK's AwsCustomResource for managing domain access policies + for child in stack.node.children: + if child.node.id.startswith('AWS'): + NagSuppressions.add_resource_suppressions( + child, + suppressions=[ + { + 'id': 'AwsSolutions-L1', + 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' + 'OpenSearch domain access policies. We cannot specify the runtime version.', + }, + { + 'id': 'AwsSolutions-IAM4', + 'appliesTo': [ + 'Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' + ], + 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' + 'OpenSearch domain access policies. It uses the standard execution role.', + }, + { + 'id': 'AwsSolutions-IAM5', + 'appliesTo': ['Action::kms:Describe*', 'Action::kms:List*'], + 'reason': 'This is an AWS-managed custom resource Lambda that requires KMS permissions to ' + 'access the encryption key used by the OpenSearch domain.', + }, + { + 'id': 'HIPAA.Security-LambdaDLQ', + 'reason': 'This is an AWS-managed custom resource Lambda used only during deployment to ' + 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' + 'functions.', + }, + { + 'id': 'HIPAA.Security-LambdaInsideVPC', + 'reason': 'This is an AWS-managed custom resource Lambda that needs internet access to ' + 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' + 'required.', + }, + ], + apply_to_children=True, + ) + + def _add_lambda_role_suppressions(self, lambda_role: IRole): + """ + Add CDK Nag suppressions for OpenSearch Lambda role configuration. + + :param lambda_role: The Lambda role to add suppressions for + """ + NagSuppressions.add_resource_suppressions( + lambda_role, + suppressions=[ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'This lambda role access is restricted to the specific ' + 'OpenSearch domain and its indices within the VPC.', + }, + ], + apply_to_children=True, + ) + diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 01cf82967..bba549b38 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -44,7 +44,7 @@ def test_opensearch_version(self): search_template.has_resource_properties( 'AWS::OpenSearchService::Domain', { - 'EngineVersion': 'OpenSearch_2.19', + 'EngineVersion': 'OpenSearch_3.1', }, ) From da4e5e204edfd6fe67fe6a7885b18efacf0261a4 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 2 Dec 2025 21:25:47 -0600 Subject: [PATCH 040/137] PR feedback --- .../populate_provider_documents_handler.py | 1 - .../stacks/search_persistent_stack/provider_search_domain.py | 5 +++-- .../tests/app/test_search_persistent_stack.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py index 1a6824e93..d3c013ba4 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py @@ -48,7 +48,6 @@ def __init__( :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) :param provider_table: The DynamoDB provider table - :param provider_date_of_update_index_name: The name of the providerDateOfUpdate GSI :param alarm_topic: The SNS topic for alarms """ super().__init__(scope, construct_id) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index 4b1e8e2e9..c94b825cb 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -204,6 +204,7 @@ def __init__( self._add_access_policy_lambda_suppressions() self._add_lambda_role_suppressions(self._search_api_lambda_role) self._add_lambda_role_suppressions(self._ingest_lambda_role) + self._add_lambda_role_suppressions(self._index_manager_lambda_role) # Add capacity monitoring alarms self._add_capacity_alarms(environment_name, alarm_topic) @@ -414,9 +415,9 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): metric_name='JVMMemoryPressure', dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, period=Duration.minutes(5), - statistic='Maximum', + statistic='Average', ), - evaluation_periods=6, # 30 minutes sustained + evaluation_periods=3, # 30 minutes sustained threshold=70, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index bba549b38..1f6948660 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -201,7 +201,7 @@ def test_capacity_alarms_configured(self): 'Namespace': 'AWS/ES', 'Threshold': 70, 'ComparisonOperator': 'GreaterThanThreshold', - 'EvaluationPeriods': 6, + 'EvaluationPeriods': 3, }, ) From a7d7b709020d478fec907b3f44beaa20eafa8e00 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 3 Dec 2025 09:38:07 -0600 Subject: [PATCH 041/137] support pagination for large data sets --- .../handlers/populate_provider_documents.py | 106 ++++++++++++++++-- .../test_populate_provider_documents.py | 104 ++++++++++++++++- 2 files changed, 197 insertions(+), 13 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index a9b28467c..17a57cafc 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -7,6 +7,17 @@ This Lambda is intended to be invoked manually through the AWS console for initial data population or re-indexing operations. + +The Lambda supports pagination across multiple invocations. If processing +cannot complete within 12 minutes, it will return the current compact and +last pagination key. The developer can then re-invoke the Lambda with this +output as input to continue processing. + +Example input for resumption: +{ + "startingCompact": "aslp", + "startingLastKey": {"pk": "...", "sk": "..."} +} """ import json @@ -21,11 +32,14 @@ # Batch size for DynamoDB pagination DYNAMODB_PAGE_SIZE = 1000 -# Batch size for OpenSearch bulk indexing -OPENSEARCH_BULK_SIZE = 100 +# Batch size for OpenSearch bulk indexing (1 provider averages ~2KB, 1000 * 2KB = 2MB) +OPENSEARCH_BULK_SIZE = 1000 +# Time threshold in milliseconds - stop when less than 3 minutes remain +# This leaves a 3-minute buffer before the 15-minute Lambda timeout +TIME_THRESHOLD_MS = 60 * 3000 -def populate_provider_documents(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument +def populate_provider_documents(event: dict, context: LambdaContext): """ Populate OpenSearch indices with provider documents. @@ -33,13 +47,22 @@ def populate_provider_documents(event: dict, context: LambdaContext): # noqa: A retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, and bulk indexes them into the appropriate compact-specific OpenSearch index. - :param event: Lambda event (not used, but required for Lambda signature) + If processing cannot complete within 13 minutes, the function returns pagination + information that can be passed as input to continue processing. + + :param event: Lambda event with optional pagination parameters: + - startingCompact: The compact to start/resume processing from + - startingLastKey: The DynamoDB pagination key to resume from :param context: Lambda context - :return: Summary of indexing operation + :return: Summary of indexing operation, including pagination info if incomplete """ data_client = config.data_client opensearch_client = OpenSearchClient() + # Get optional pagination parameters from event for resumption + starting_compact = event.get('startingCompact') + starting_last_key = event.get('startingLastKey') + # Track statistics stats = { 'total_providers_processed': 0, @@ -47,9 +70,30 @@ def populate_provider_documents(event: dict, context: LambdaContext): # noqa: A 'total_providers_failed': 0, 'compacts_processed': [], 'errors': [], + 'completed': True, # Will be set to False if we need to paginate } - for compact in config.compacts: + # Determine which compacts to process + compacts_to_process = config.compacts + + # If resuming, skip compacts before the starting compact + if starting_compact: + if starting_compact in compacts_to_process: + start_index = compacts_to_process.index(starting_compact) + compacts_to_process = compacts_to_process[start_index:] + logger.info( + 'Resuming from compact', + starting_compact=starting_compact, + starting_last_key=starting_last_key, + ) + else: + logger.warning( + 'Starting compact not found, processing all compacts', + starting_compact=starting_compact, + ) + starting_last_key = None # Reset last key if compact not found + + for compact_index, compact in enumerate(compacts_to_process): logger.info('Processing compact', compact=compact) index_name = f'compact_{compact}_providers' @@ -61,10 +105,58 @@ def populate_provider_documents(event: dict, context: LambdaContext): # noqa: A } # Track pagination state - last_key = None + # Use starting_last_key only for the first compact being processed (resumption case). + # The starting_last_key is specific to the compact that was being processed when we timed out, + # so it's only valid for that compact (which is now the first in compacts_to_process). + # For all subsequent compacts, we start from the beginning with last_key = None. + last_key = starting_last_key if compact_index == 0 else None has_more = True while has_more: + # Check if we're running out of time before starting a new batch + remaining_time_ms = context.get_remaining_time_in_millis() + if remaining_time_ms < TIME_THRESHOLD_MS: + # We need to stop and return pagination info for resumption + logger.info( + 'Approaching time limit, returning pagination info', + remaining_time_ms=remaining_time_ms, + current_compact=compact, + last_key=last_key, + ) + + # Index any remaining documents before returning + if documents_to_index: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index, stats) + compact_stats['providers_indexed'] += indexed_count + + # Update stats for current compact + stats['total_providers_processed'] += compact_stats['providers_processed'] + stats['total_providers_indexed'] += compact_stats['providers_indexed'] + stats['total_providers_failed'] += compact_stats['providers_failed'] + if compact_stats['providers_processed'] > 0: + stats['compacts_processed'].append( + { + 'compact': compact, + **compact_stats, + } + ) + + # Return pagination info for resumption + stats['completed'] = False + stats['resumeFrom'] = { + 'startingCompact': compact, + 'startingLastKey': last_key, + } + + logger.info( + 'Returning for pagination', + total_providers_processed=stats['total_providers_processed'], + total_providers_indexed=stats['total_providers_indexed'], + resume_from=stats['resumeFrom'], + ) + + return stats + # Build pagination parameters dynamo_pagination = {'pageSize': DYNAMODB_PAGE_SIZE} if last_key: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index b8d9c0854..88074e873 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, call, patch +from unittest.mock import MagicMock, Mock, call, patch from common_test.test_constants import ( DEFAULT_LICENSE_EXPIRATION_DATE, @@ -130,7 +130,7 @@ def _generate_expected_call_for_document(self, compact): @patch('handlers.populate_provider_documents.OpenSearchClient') def test_provider_records_from_all_three_compacts_are_indexed_in_expected_index(self, mock_opensearch_client): - from handlers.populate_provider_documents import populate_provider_documents + from handlers.populate_provider_documents import TIME_THRESHOLD_MS, populate_provider_documents # Set up the mock opensearch client mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) @@ -140,8 +140,12 @@ def test_provider_records_from_all_three_compacts_are_indexed_in_expected_index( for compact in compacts: self._put_test_provider_and_license_record_in_dynamodb_table(compact) + # mock the context to always return time above the cutoff threshold + mock_context = MagicMock() + mock_context.get_remaining_time_in_millis.return_value = TIME_THRESHOLD_MS + 60000 + # now run the handler - result = populate_provider_documents({}, self.mock_context) + result = populate_provider_documents({}, mock_context) # Assert that the OpenSearchClient was instantiated mock_opensearch_client.assert_called_once() @@ -156,6 +160,94 @@ def test_provider_records_from_all_three_compacts_are_indexed_in_expected_index( self.assertEqual(self._generate_expected_call_for_document('coun'), bulk_index_calls[2]) # Verify the result statistics - self.assertEqual(3, result['total_providers_processed']) - self.assertEqual(3, result['total_providers_indexed']) - self.assertEqual(0, result['total_providers_failed']) + self.assertEqual( + { + 'compacts_processed': [ + {'compact': 'aslp', 'providers_failed': 0, 'providers_indexed': 1, 'providers_processed': 1}, + {'compact': 'octp', 'providers_failed': 0, 'providers_indexed': 1, 'providers_processed': 1}, + {'compact': 'coun', 'providers_failed': 0, 'providers_indexed': 1, 'providers_processed': 1}, + ], + 'completed': True, + 'errors': [], + 'total_providers_failed': 0, + 'total_providers_indexed': 3, + 'total_providers_processed': 3, + }, + result, + ) + + @patch('handlers.populate_provider_documents.OpenSearchClient') + def test_pagination_across_invocations_when_time_limit_reached(self, mock_opensearch_client): + """Test that the handler properly paginates across multiple invocations when approaching time limit. + + This test verifies: + 1. When the time limit is reached, the handler returns pagination info + 2. The pagination info can be passed to the next invocation to resume processing + 3. All records are eventually indexed across multiple invocations + """ + from handlers.populate_provider_documents import TIME_THRESHOLD_MS, populate_provider_documents + + # Time values for mocking (in milliseconds) + PLENTY_OF_TIME_MS = TIME_THRESHOLD_MS + 60000 # Above threshold, continue processing + LOW_TIME_MS = TIME_THRESHOLD_MS - 1000 # Below threshold, trigger timeout + + # Set up the mock opensearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + compacts = ['aslp', 'octp', 'coun'] + # Add a provider and license record for each of the three compacts + for compact in compacts: + self._put_test_provider_and_license_record_in_dynamodb_table(compact) + + # First invocation: Mock time to trigger timeout after processing first compact (aslp) + # The time check happens at the START of each while loop iteration: + # - Call 1: Processing aslp, plenty of time -> continue + # - Call 2: About to process octp, low time -> timeout and return + mock_context = MagicMock() + mock_context.get_remaining_time_in_millis.side_effect = [PLENTY_OF_TIME_MS, LOW_TIME_MS] + + # Run the first invocation + first_result = populate_provider_documents({}, mock_context) + + # Verify first invocation returned incomplete with pagination info + self.assertFalse(first_result['completed']) + self.assertIn('resumeFrom', first_result) + self.assertEqual('octp', first_result['resumeFrom']['startingCompact']) + # startingLastKey should be None since we haven't started processing octp yet + self.assertIsNone(first_result['resumeFrom']['startingLastKey']) + + # Verify only aslp was indexed in first invocation + self.assertEqual(1, first_result['total_providers_indexed']) + self.assertEqual(1, mock_client_instance.bulk_index.call_count) + + # Second invocation: Use the resumeFrom values as input + # Reset the mock for the second invocation + mock_opensearch_client.reset_mock() + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Mock time to allow completion - needs enough calls for both octp and coun + # - Call 1: Processing octp, plenty of time -> continue + # - Call 2: Processing coun, plenty of time -> continue + mock_context.get_remaining_time_in_millis.side_effect = [PLENTY_OF_TIME_MS, PLENTY_OF_TIME_MS] + + # Build the resume event from the first result + resume_event = { + 'startingCompact': first_result['resumeFrom']['startingCompact'], + 'startingLastKey': first_result['resumeFrom']['startingLastKey'], + } + + # Run the second invocation with pagination info + second_result = populate_provider_documents(resume_event, mock_context) + + # Verify second invocation completed successfully + self.assertTrue(second_result['completed']) + self.assertNotIn('resumeFrom', second_result) + + # Verify octp and coun were indexed in second invocation + self.assertEqual(2, second_result['total_providers_indexed']) + self.assertEqual(2, mock_client_instance.bulk_index.call_count) + + # Verify the correct indices were called + bulk_index_calls = mock_client_instance.bulk_index.call_args_list + self.assertEqual(self._generate_expected_call_for_document('octp'), bulk_index_calls[0]) + self.assertEqual(self._generate_expected_call_for_document('coun'), bulk_index_calls[1]) From 21ef219ddc23573690241f9cd2bfeb80f1b9d9e2 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 3 Dec 2025 09:38:36 -0600 Subject: [PATCH 042/137] formatting --- backend/compact-connect/bin/trim_oas30.py | 4 +- .../bin/update_postman_collection.py | 4 +- .../common_constructs/user_pool.py | 2 +- .../tests/function/test_encumbrance_events.py | 13 +- .../provider_search_domain.py | 1 - backend/compact-connect/tests/app/base.py | 129 +++++++++--------- 6 files changed, 70 insertions(+), 83 deletions(-) diff --git a/backend/compact-connect/bin/trim_oas30.py b/backend/compact-connect/bin/trim_oas30.py index 6fbcac725..53d80cc1a 100755 --- a/backend/compact-connect/bin/trim_oas30.py +++ b/backend/compact-connect/bin/trim_oas30.py @@ -35,9 +35,7 @@ def strip_options_endpoints(oas30: dict) -> dict: parser.add_argument( '-i', '--internal', action='store_true', help='Use internal API specification files instead of regular ones' ) - parser.add_argument( - '-s', '--search', action='store_true', help='Use search API specification files' - ) + parser.add_argument('-s', '--search', action='store_true', help='Use search API specification files') args = parser.parse_args() diff --git a/backend/compact-connect/bin/update_postman_collection.py b/backend/compact-connect/bin/update_postman_collection.py index 9a684c67f..1ffe060a2 100755 --- a/backend/compact-connect/bin/update_postman_collection.py +++ b/backend/compact-connect/bin/update_postman_collection.py @@ -196,9 +196,7 @@ def main(): parser.add_argument( '-i', '--internal', action='store_true', help='Use internal API specification files instead of regular ones' ) - parser.add_argument( - '-s', '--search', action='store_true', help='Use search API specification files' - ) + parser.add_argument('-s', '--search', action='store_true', help='Use search API specification files') args = parser.parse_args() diff --git a/backend/compact-connect/common_constructs/user_pool.py b/backend/compact-connect/common_constructs/user_pool.py index 7b63501f9..b766143a4 100644 --- a/backend/compact-connect/common_constructs/user_pool.py +++ b/backend/compact-connect/common_constructs/user_pool.py @@ -216,7 +216,7 @@ def add_custom_app_client_domain( suppressions=[ { 'id': 'AwsSolutions-L1', - 'reason': 'We do not maintain this lambda runtime. It will be updated with future CDK versions' + 'reason': 'We do not maintain this lambda runtime. It will be updated with future CDK versions', }, { 'id': 'HIPAA.Security-LambdaDLQ', diff --git a/backend/compact-connect/lambdas/python/data-events/tests/function/test_encumbrance_events.py b/backend/compact-connect/lambdas/python/data-events/tests/function/test_encumbrance_events.py index 48250af94..7600f8011 100644 --- a/backend/compact-connect/lambdas/python/data-events/tests/function/test_encumbrance_events.py +++ b/backend/compact-connect/lambdas/python/data-events/tests/function/test_encumbrance_events.py @@ -3074,9 +3074,7 @@ def test_license_encumbrance_listener_does_not_create_duplicate_update_records_f # Verify STILL only one update record exists (no duplicate created) update_records_after_retry = ( - self.test_data_generator.query_privilege_update_records_for_given_record_from_database( - privilege - ) + self.test_data_generator.query_privilege_update_records_for_given_record_from_database(privilege) ) matching_updates_after_retry = [ update @@ -3147,9 +3145,7 @@ def test_license_encumbrance_listener_does_not_create_duplicate_update_records_f # Verify STILL only one update record exists (no duplicate created) update_records_after_retry = ( - self.test_data_generator.query_privilege_update_records_for_given_record_from_database( - privilege - ) + self.test_data_generator.query_privilege_update_records_for_given_record_from_database(privilege) ) matching_updates_after_retry = [ update @@ -3241,9 +3237,7 @@ def test_license_encumbrance_lifted_listener_does_not_create_duplicate_update_re # license_encumbrance_lifted_listener will skip creating privilege updates because it only # does so on LICENSE_ENCUMBERED privileges and none of those would remain update_records_after_retry = ( - self.test_data_generator.query_privilege_update_records_for_given_record_from_database( - privilege - ) + self.test_data_generator.query_privilege_update_records_for_given_record_from_database(privilege) ) matching_updates_after_retry = [ update for update in update_records_after_retry if update.updateType == UpdateCategory.LIFTING_ENCUMBRANCE @@ -3369,4 +3363,3 @@ def test_license_encumbrance_notification_listener_creates_notification_events_t self.assertEqual(expected_sks, list(notification_records.keys())) for sk in expected_sks: self.assertEqual(NotificationStatus.SUCCESS, notification_records.get(sk).get('status')) - diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index c94b825cb..bbb1b2512 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -568,4 +568,3 @@ def _add_lambda_role_suppressions(self, lambda_role: IRole): ], apply_to_children=True, ) - diff --git a/backend/compact-connect/tests/app/base.py b/backend/compact-connect/tests/app/base.py index 439537908..12ff282ef 100644 --- a/backend/compact-connect/tests/app/base.py +++ b/backend/compact-connect/tests/app/base.py @@ -270,26 +270,31 @@ def _inspect_ssn_table(self, persistent_stack: PersistentStack, persistent_stack { 'Properties': { 'KeyPolicy': { - 'Statement': Match.array_with([ - { - 'Action': 'kms:*', - 'Effect': 'Allow', - 'Principal': {'AWS': f'arn:aws:iam::{persistent_stack.account}:root'}, - 'Resource': '*', - }, - { - 'Action': ['kms:Decrypt', 'kms:Encrypt', 'kms:GenerateDataKey*', 'kms:ReEncrypt*'], - 'Condition': { - 'StringNotEquals': { - 'aws:PrincipalArn': principal_arn_array, - 'aws:PrincipalServiceName': ['dynamodb.amazonaws.com', 'events.amazonaws.com'], - } + 'Statement': Match.array_with( + [ + { + 'Action': 'kms:*', + 'Effect': 'Allow', + 'Principal': {'AWS': f'arn:aws:iam::{persistent_stack.account}:root'}, + 'Resource': '*', + }, + { + 'Action': ['kms:Decrypt', 'kms:Encrypt', 'kms:GenerateDataKey*', 'kms:ReEncrypt*'], + 'Condition': { + 'StringNotEquals': { + 'aws:PrincipalArn': principal_arn_array, + 'aws:PrincipalServiceName': [ + 'dynamodb.amazonaws.com', + 'events.amazonaws.com', + ], + } + }, + 'Effect': 'Deny', + 'Principal': '*', + 'Resource': '*', }, - 'Effect': 'Deny', - 'Principal': '*', - 'Resource': '*', - }, - ]), + ] + ), 'Version': '2012-10-17', } } @@ -305,59 +310,53 @@ def _inspect_ssn_table(self, persistent_stack: PersistentStack, persistent_stack 'TableName': 'ssn-table-DataEventsLog', 'ResourcePolicy': { 'PolicyDocument': { - 'Statement': Match.array_with([ - { - 'Effect': 'Deny', - 'Principal': '*', - 'Resource': '*', - 'Action': 'dynamodb:CreateBackup', - 'Condition': { - 'StringNotEquals': { - 'aws:PrincipalServiceName': 'dynamodb.amazonaws.com' - } - } - }, - { - 'Effect': 'Deny', - 'Principal': '*', - 'Resource': '*', - 'Action': [ - 'dynamodb:BatchGetItem', - 'dynamodb:BatchWriteItem', - 'dynamodb:PartiQL*', - 'dynamodb:Scan', - ], - 'Condition': { - 'StringNotEquals': { - 'aws:PrincipalServiceName': 'dynamodb.amazonaws.com', - 'aws:PrincipalArn': Match.any_value(), - } + 'Statement': Match.array_with( + [ + { + 'Effect': 'Deny', + 'Principal': '*', + 'Resource': '*', + 'Action': 'dynamodb:CreateBackup', + 'Condition': { + 'StringNotEquals': {'aws:PrincipalServiceName': 'dynamodb.amazonaws.com'} + }, }, - }, - { - "Action": [ - "dynamodb:ConditionCheckItem", - "dynamodb:GetItem", - "dynamodb:Query" - ], - "Effect": "Deny", - "Principal": "*", - "NotResource": Match.string_like_regexp( - f"arn:aws:dynamodb:{persistent_stack.region}:{persistent_stack.account}:table/ssn-table-DataEventsLog/index/ssnIndex" - ) - }, - ]) + { + 'Effect': 'Deny', + 'Principal': '*', + 'Resource': '*', + 'Action': [ + 'dynamodb:BatchGetItem', + 'dynamodb:BatchWriteItem', + 'dynamodb:PartiQL*', + 'dynamodb:Scan', + ], + 'Condition': { + 'StringNotEquals': { + 'aws:PrincipalServiceName': 'dynamodb.amazonaws.com', + 'aws:PrincipalArn': Match.any_value(), + } + }, + }, + { + 'Action': ['dynamodb:ConditionCheckItem', 'dynamodb:GetItem', 'dynamodb:Query'], + 'Effect': 'Deny', + 'Principal': '*', + 'NotResource': Match.string_like_regexp( + f'arn:aws:dynamodb:{persistent_stack.region}:{persistent_stack.account}:table/ssn-table-DataEventsLog/index/ssnIndex' + ), + }, + ] + ) } }, 'SSESpecification': { - 'KMSMasterKeyId': { - 'Fn::GetAtt': [ssn_key_logical_id, 'Arn'] - }, + 'KMSMasterKeyId': {'Fn::GetAtt': [ssn_key_logical_id, 'Arn']}, 'SSEEnabled': True, 'SSEType': 'KMS', - } + }, } - } + }, ) def _inspect_backup_resources(self, persistent_stack: PersistentStack, persistent_stack_template: Template): From 432424980d303887ecb6db2fb6ce18f128f41907 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 3 Dec 2025 10:44:57 -0600 Subject: [PATCH 043/137] Add retry logic to handle read timeout errors --- .../handlers/populate_provider_documents.py | 163 ++++++++++++------ .../python/search/opensearch_client.py | 66 ++++++- .../test_populate_provider_documents.py | 75 ++++++++ .../tests/unit/test_opensearch_client.py | 102 +++++++++++ 4 files changed, 349 insertions(+), 57 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 17a57cafc..70f61a3bf 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -25,7 +25,7 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema -from cc_common.exceptions import CCNotFoundException +from cc_common.exceptions import CCInternalException, CCNotFoundException from cc_common.utils import ResponseEncoder from marshmallow import ValidationError from opensearch_client import OpenSearchClient @@ -110,6 +110,8 @@ def populate_provider_documents(event: dict, context: LambdaContext): # so it's only valid for that compact (which is now the first in compacts_to_process). # For all subsequent compacts, we start from the beginning with last_key = None. last_key = starting_last_key if compact_index == 0 else None + # Track the key used to fetch the current batch (needed for retry on indexing failure) + batch_start_key = last_key has_more = True while has_more: @@ -126,8 +128,14 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Index any remaining documents before returning if documents_to_index: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index, stats) - compact_stats['providers_indexed'] += indexed_count + try: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) + compact_stats['providers_indexed'] += indexed_count + except CCInternalException as e: + # Indexing failed after retries, return pagination info for manual retry + return _build_error_response( + stats, compact_stats, compact, batch_start_key, str(e), + ) # Update stats for current compact stats['total_providers_processed'] += compact_stats['providers_processed'] @@ -162,6 +170,9 @@ def populate_provider_documents(event: dict, context: LambdaContext): if last_key: dynamo_pagination['lastKey'] = last_key + # Save the key used to fetch this batch (for retry if indexing fails) + batch_start_key = last_key + # Query providers from the GSI result = data_client.get_providers_sorted_by_updated( compact=compact, @@ -195,7 +206,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): provider_user_records = data_client.get_provider_user_records( compact=compact, provider_id=provider_id, - consistent_read=False, # Eventual consistency is fine for indexing + consistent_read=True, ) # Generate API response object with all nested records @@ -235,14 +246,26 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Bulk index when batch is full if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index, stats) - compact_stats['providers_indexed'] += indexed_count - documents_to_index = [] + try: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) + compact_stats['providers_indexed'] += indexed_count + documents_to_index = [] + except CCInternalException as e: + # Indexing failed after retries, return pagination info for manual retry + return _build_error_response( + stats, compact_stats, compact, batch_start_key, str(e), + ) # Index any remaining documents for this compact if documents_to_index: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index, stats) - compact_stats['providers_indexed'] += indexed_count + try: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) + compact_stats['providers_indexed'] += indexed_count + except CCInternalException as e: + # Indexing failed after retries, return pagination info for manual retry + return _build_error_response( + stats, compact_stats, compact, batch_start_key, str(e), + ) # Update overall stats stats['total_providers_processed'] += compact_stats['providers_processed'] @@ -273,63 +296,93 @@ def populate_provider_documents(event: dict, context: LambdaContext): return stats -def _bulk_index_documents( - opensearch_client: OpenSearchClient, index_name: str, documents: list[dict], stats: dict -) -> int: +def _build_error_response( + stats: dict, compact_stats: dict, compact: str, batch_start_key: dict | None, error_message: str +) -> dict: + """ + Build an error response with pagination info for retry after indexing failure. + + :param stats: The overall statistics dictionary + :param compact_stats: The current compact's statistics + :param compact: The compact being processed when the error occurred + :param batch_start_key: The pagination key used to fetch the batch that failed to index + :param error_message: The error message from the failed indexing attempt + :return: Response dictionary with error info and pagination for retry + """ + logger.error( + 'Bulk indexing failed after retries, returning pagination info for retry', + compact=compact, + batch_start_key=batch_start_key, + error=error_message, + ) + + # Update stats for current compact + stats['total_providers_processed'] += compact_stats['providers_processed'] + stats['total_providers_indexed'] += compact_stats['providers_indexed'] + stats['total_providers_failed'] += compact_stats['providers_failed'] + if compact_stats['providers_processed'] > 0: + stats['compacts_processed'].append( + { + 'compact': compact, + **compact_stats, + } + ) + + # Return pagination info for retry - use batch_start_key so the failed batch is re-fetched + stats['completed'] = False + stats['resumeFrom'] = { + 'startingCompact': compact, + 'startingLastKey': batch_start_key, + } + stats['errors'].append( + { + 'compact': compact, + 'error': error_message, + } + ) + + return stats + + +def _bulk_index_documents(opensearch_client: OpenSearchClient, index_name: str, documents: list[dict]) -> int: """ Bulk index documents into OpenSearch. :param opensearch_client: The OpenSearch client :param index_name: The index to write to :param documents: List of documents to index - :param stats: Statistics dictionary to update with errors :return: Number of successfully indexed documents + :raises CCInternalException: If bulk indexing fails after max retry attempts """ if not documents: return 0 - try: - response = opensearch_client.bulk_index(index_name=index_name, documents=documents) - - # Check for errors in the bulk response - if response.get('errors'): - error_count = 0 - for item in response.get('items', []): - index_result = item.get('index', {}) - if index_result.get('error'): - error_count += 1 - logger.warning( - 'Bulk index item error', - document_id=index_result.get('_id'), - error=index_result.get('error'), - ) - logger.warning( - 'Bulk index completed with errors', - index_name=index_name, - total_documents=len(documents), - error_count=error_count, - ) - return len(documents) - error_count - - logger.info( - 'Indexed documents', + # This will raise CCInternalException if all retries fail + response = opensearch_client.bulk_index(index_name=index_name, documents=documents) + + # Check for errors in the bulk response (individual document failures, not connection issues) + if response.get('errors'): + error_count = 0 + for item in response.get('items', []): + index_result = item.get('index', {}) + if index_result.get('error'): + error_count += 1 + logger.warning( + 'Bulk index item error', + document_id=index_result.get('_id'), + error=index_result.get('error'), + ) + logger.warning( + 'Bulk index completed with errors', index_name=index_name, - document_count=len(documents), + total_documents=len(documents), + error_count=error_count, ) - return len(documents) + return len(documents) - error_count - except Exception as e: - logger.exception( - 'Failed to bulk index documents', - index_name=index_name, - document_count=len(documents), - error=str(e), - ) - stats['errors'].append( - { - 'index': index_name, - 'error': f'Bulk index failed: {str(e)}', - 'document_count': len(documents), - } - ) - return 0 + logger.info( + 'Indexed documents', + index_name=index_name, + document_count=len(documents), + ) + return len(documents) diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 235d0e5a0..9220a7f6b 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -1,6 +1,15 @@ +import time + import boto3 -from cc_common.config import config +from cc_common.config import config, logger +from cc_common.exceptions import CCInternalException from opensearchpy import AWSV4SignerAuth, OpenSearch, RequestsHttpConnection +from opensearchpy.exceptions import ConnectionTimeout, TransportError + +# Retry configuration for bulk indexing +MAX_RETRY_ATTEMPTS = 5 +INITIAL_BACKOFF_SECONDS = 1 +MAX_BACKOFF_SECONDS = 32 class OpenSearchClient: @@ -14,6 +23,7 @@ def __init__(self): verify_certs=True, connection_class=RequestsHttpConnection, pool_maxsize=20, + ) def create_index(self, index_name: str, index_mapping: dict) -> None: @@ -47,10 +57,15 @@ def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'pr """ Bulk index multiple documents into the specified index. + This method implements retry logic with exponential backoff to handle transient + connection issues (e.g., ConnectionTimeout, TransportError). If all retry attempts + fail, a CCInternalException is raised to signal the caller to handle the failure. + :param index_name: The name of the index to write to :param documents: List of documents to index :param id_field: The field name to use as the document ID (default: 'providerId') :return: The bulk response from OpenSearch + :raises CCInternalException: If all retry attempts fail due to connection issues """ if not documents: return {'items': [], 'errors': False} @@ -63,4 +78,51 @@ def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'pr # indices in the request body for security purposes. actions.append({'index': {'_id': doc[id_field]}}) actions.append(doc) - return self._client.bulk(body=actions, index=index_name) + + return self._bulk_index_with_retry(actions=actions, index_name=index_name, document_count=len(documents)) + + def _bulk_index_with_retry(self, actions: list, index_name: str, document_count: int) -> dict: + """ + Execute bulk index with retry logic and exponential backoff. + + :param actions: The bulk actions to execute + :param index_name: The name of the index to write to + :param document_count: Number of documents being indexed (for logging) + :return: The bulk response from OpenSearch + :raises CCInternalException: If all retry attempts fail + """ + last_exception = None + backoff_seconds = INITIAL_BACKOFF_SECONDS + + for attempt in range(1, MAX_RETRY_ATTEMPTS + 1): + try: + return self._client.bulk(body=actions, index=index_name, timeout=30) + except (ConnectionTimeout, TransportError) as e: + last_exception = e + if attempt < MAX_RETRY_ATTEMPTS: + logger.warning( + 'Bulk index attempt failed, retrying with backoff', + attempt=attempt, + max_attempts=MAX_RETRY_ATTEMPTS, + backoff_seconds=backoff_seconds, + index_name=index_name, + document_count=document_count, + error=str(e), + ) + time.sleep(backoff_seconds) + # Exponential backoff with cap + backoff_seconds = min(backoff_seconds * 2, MAX_BACKOFF_SECONDS) + else: + logger.error( + 'Bulk index failed after max retry attempts', + attempts=MAX_RETRY_ATTEMPTS, + index_name=index_name, + document_count=document_count, + error=str(e), + ) + + # All retry attempts failed + raise CCInternalException( + f'Failed to bulk index {document_count} documents to {index_name} after {MAX_RETRY_ATTEMPTS} attempts. ' + f'Last error: {last_exception}' + ) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 88074e873..44efdf432 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -251,3 +251,78 @@ def test_pagination_across_invocations_when_time_limit_reached(self, mock_opense bulk_index_calls = mock_client_instance.bulk_index.call_args_list self.assertEqual(self._generate_expected_call_for_document('octp'), bulk_index_calls[0]) self.assertEqual(self._generate_expected_call_for_document('coun'), bulk_index_calls[1]) + + @patch('handlers.populate_provider_documents.OpenSearchClient') + def test_returns_pagination_info_when_bulk_indexing_fails_after_retries(self, mock_opensearch_client): + """Test that the handler returns pagination info when bulk indexing fails after max retries. + + This test verifies: + 1. When CCInternalException is raised by bulk_index, the handler catches it + 2. The response includes resumeFrom with the batch_start_key for retry + 3. The developer can use this info to retry from the exact point of failure + """ + from cc_common.exceptions import CCInternalException + + from handlers.populate_provider_documents import TIME_THRESHOLD_MS, populate_provider_documents + + # Set up the mock opensearch client to raise CCInternalException on second compact + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + + # First compact (aslp) succeeds, second compact (octp) fails with CCInternalException + mock_client_instance.bulk_index.side_effect = [ + {'items': [], 'errors': False}, # aslp succeeds + CCInternalException('Connection timeout after 5 retries'), # octp fails + ] + + compacts = ['aslp', 'octp', 'coun'] + # Add a provider and license record for each compact + for compact in compacts: + self._put_test_provider_and_license_record_in_dynamodb_table(compact) + + # Mock the context to always return time above the cutoff threshold + mock_context = MagicMock() + mock_context.get_remaining_time_in_millis.return_value = TIME_THRESHOLD_MS + 60000 + + # Run the handler + result = populate_provider_documents({}, mock_context) + + # Verify the result indicates incomplete processing + self.assertFalse(result['completed']) + self.assertIn('resumeFrom', result) + + # Verify resumeFrom points to octp with the batch_start_key (None since it's the first batch) + self.assertEqual('octp', result['resumeFrom']['startingCompact']) + # startingLastKey should be None since it was the first batch of octp + self.assertIsNone(result['resumeFrom']['startingLastKey']) + + # Verify aslp was indexed but octp was not + self.assertEqual(1, result['total_providers_indexed']) + + # Verify errors list contains the failure info + self.assertEqual(1, len(result['errors'])) + self.assertEqual('octp', result['errors'][0]['compact']) + self.assertIn('Connection timeout', result['errors'][0]['error']) + + # Now verify that using resumeFrom allows completing the indexing + mock_opensearch_client.reset_mock() + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_index.return_value = {'items': [], 'errors': False} + + # Build the resume event from the first result + resume_event = { + 'startingCompact': result['resumeFrom']['startingCompact'], + 'startingLastKey': result['resumeFrom']['startingLastKey'], + } + + # Run the second invocation + second_result = populate_provider_documents(resume_event, mock_context) + + # Verify second invocation completed successfully + self.assertTrue(second_result['completed']) + self.assertNotIn('resumeFrom', second_result) + + # Verify octp and coun were indexed + self.assertEqual(2, second_result['total_providers_indexed']) + self.assertEqual(2, mock_client_instance.bulk_index.call_count) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index b63b4bc71..b9fb472e4 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -1,6 +1,8 @@ from unittest import TestCase from unittest.mock import MagicMock, patch +from opensearchpy.exceptions import ConnectionTimeout, TransportError + class TestOpenSearchClient(TestCase): """Test suite for OpenSearchClient to verify internal client calls.""" @@ -122,6 +124,7 @@ def test_bulk_index_calls_internal_client_with_expected_arguments(self): mock_internal_client.bulk.assert_called_once_with( body=expected_actions, index=index_name, + timeout=30 ) self.assertEqual(expected_response, result) @@ -147,6 +150,7 @@ def test_bulk_index_uses_custom_id_field(self): mock_internal_client.bulk.assert_called_once_with( body=expected_actions, index=index_name, + timeout=30 ) def test_bulk_index_returns_early_for_empty_documents(self): @@ -157,3 +161,101 @@ def test_bulk_index_returns_early_for_empty_documents(self): mock_internal_client.bulk.assert_not_called() self.assertEqual({'items': [], 'errors': False}, result) + + @patch('opensearch_client.time.sleep') + def test_bulk_index_retries_on_connection_timeout_and_succeeds(self, mock_sleep): + """Test that bulk_index retries on ConnectionTimeout and eventually succeeds.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + documents = [{'providerId': 'provider-1', 'givenName': 'John'}] + expected_response = {'errors': False, 'items': [{'index': {'_id': 'provider-1'}}]} + + # First two calls fail with ConnectionTimeout, third succeeds + mock_internal_client.bulk.side_effect = [ + ConnectionTimeout('Connection timed out', 503, 'some error'), + ConnectionTimeout('Connection timed out', 503, 'some error'), + expected_response, + ] + + result = client.bulk_index(index_name=index_name, documents=documents) + + # Verify bulk was called 3 times + self.assertEqual(3, mock_internal_client.bulk.call_count) + # Verify sleep was called with exponential backoff (1s, 2s) + self.assertEqual(2, mock_sleep.call_count) + mock_sleep.assert_any_call(1) + mock_sleep.assert_any_call(2) + # Verify we got the successful response + self.assertEqual(expected_response, result) + + @patch('opensearch_client.time.sleep') + def test_bulk_index_retries_on_transport_error_and_succeeds(self, mock_sleep): + """Test that bulk_index retries on TransportError and eventually succeeds.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + documents = [{'providerId': 'provider-1', 'givenName': 'John'}] + expected_response = {'errors': False, 'items': [{'index': {'_id': 'provider-1'}}]} + + # First call fails with TransportError, second succeeds + mock_internal_client.bulk.side_effect = [ + TransportError(503, 'ReadTimeout'), + expected_response, + ] + + result = client.bulk_index(index_name=index_name, documents=documents) + + # Verify bulk was called 2 times + self.assertEqual(2, mock_internal_client.bulk.call_count) + # Verify sleep was called once + self.assertEqual(1, mock_sleep.call_count) + self.assertEqual(expected_response, result) + + @patch('opensearch_client.time.sleep') + def test_bulk_index_raises_cc_internal_exception_after_max_retries(self, mock_sleep): + """Test that bulk_index raises CCInternalException after all retry attempts fail.""" + from cc_common.exceptions import CCInternalException + from opensearch_client import MAX_RETRY_ATTEMPTS + + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + documents = [{'providerId': 'provider-1', 'givenName': 'John'}] + + # All calls fail with ConnectionTimeout + mock_internal_client.bulk.side_effect = ConnectionTimeout('Connection timed out', 503, 'some error') + + with self.assertRaises(CCInternalException) as context: + client.bulk_index(index_name=index_name, documents=documents) + + # Verify bulk was called MAX_RETRY_ATTEMPTS times + self.assertEqual(MAX_RETRY_ATTEMPTS, mock_internal_client.bulk.call_count) + # Verify sleep was called MAX_RETRY_ATTEMPTS - 1 times (no sleep after last failure) + self.assertEqual(MAX_RETRY_ATTEMPTS - 1, mock_sleep.call_count) + # Verify the exception message contains useful info + self.assertIn('Failed to bulk index', str(context.exception)) + self.assertIn(index_name, str(context.exception)) + self.assertIn(str(MAX_RETRY_ATTEMPTS), str(context.exception)) + + @patch('opensearch_client.time.sleep') + def test_bulk_index_exponential_backoff_caps_at_max(self, mock_sleep): + """Test that exponential backoff is capped at MAX_BACKOFF_SECONDS.""" + from opensearch_client import MAX_BACKOFF_SECONDS + + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + documents = [{'providerId': 'provider-1', 'givenName': 'John'}] + + # All calls fail + mock_internal_client.bulk.side_effect = ConnectionTimeout('Connection timed out', 503, 'some error') + + with self.assertRaises(Exception): + client.bulk_index(index_name=index_name, documents=documents) + + # Verify backoff values: 1, 2, 4, 8 (all should be <= MAX_BACKOFF_SECONDS) + # With MAX_RETRY_ATTEMPTS = 5, we have 4 sleeps + sleep_calls = [call[0][0] for call in mock_sleep.call_args_list] + for sleep_value in sleep_calls: + self.assertLessEqual(sleep_value, MAX_BACKOFF_SECONDS) From 558efa170d459809f68f7e9a9667872e08a9913d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 4 Dec 2025 11:52:54 -0600 Subject: [PATCH 044/137] update doc to match current behavior --- backend/compact-connect/app_clients/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/app_clients/README.md b/backend/compact-connect/app_clients/README.md index 4573eead0..d1e04dd9f 100644 --- a/backend/compact-connect/app_clients/README.md +++ b/backend/compact-connect/app_clients/README.md @@ -66,7 +66,7 @@ jurisdiction. ```bash -python3 bin/create_app_client.py -e -u +python3 bin/create_app_client.py -u ``` **Interactive Process:** From 85d8a5b0833d9f5244c1a1130ae9c6845d7d788d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 4 Dec 2025 16:33:45 -0600 Subject: [PATCH 045/137] Add military status fields to mapping --- .../lambdas/python/search/handlers/manage_opensearch_indices.py | 2 ++ .../search/tests/function/test_manage_opensearch_indices.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index bcf1f01f8..6f90dc57b 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -226,6 +226,8 @@ def _get_provider_index_mapping(self) -> dict: 'providerFamGivMid': {'type': 'keyword'}, 'providerDateOfUpdate': {'type': 'date'}, 'birthMonthDay': {'type': 'keyword'}, + 'militaryStatus': {'type': 'keyword'}, + 'militaryStatusNote': {'type': 'text'}, # Nested arrays 'licenses': {'type': 'nested', 'properties': license_properties}, 'privileges': {'type': 'nested', 'properties': privilege_properties}, diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index 03e5a757d..dd1f5bdbc 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -212,6 +212,8 @@ def test_on_create_creates_indices_for_all_compacts_when_none_exist(self, mock_o }, 'type': 'nested', }, + 'militaryStatus': {'type': 'keyword'}, + 'militaryStatusNote': {'type': 'text'}, 'npi': {'type': 'keyword'}, 'privilegeJurisdictions': {'type': 'keyword'}, 'privileges': { From c34ab1b91c938f599f621832be0fb1861825fd91 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 4 Dec 2025 16:44:59 -0600 Subject: [PATCH 046/137] Formatting/linter --- .../handlers/populate_provider_documents.py | 27 ++++++++++--------- .../python/search/opensearch_client.py | 1 - .../test_populate_provider_documents.py | 9 +++---- .../tests/unit/test_opensearch_client.py | 15 +++-------- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 70f61a3bf..9c7bfec42 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -134,7 +134,11 @@ def populate_provider_documents(event: dict, context: LambdaContext): except CCInternalException as e: # Indexing failed after retries, return pagination info for manual retry return _build_error_response( - stats, compact_stats, compact, batch_start_key, str(e), + stats, + compact_stats, + compact, + batch_start_key, + str(e), ) # Update stats for current compact @@ -234,15 +238,6 @@ def populate_provider_documents(event: dict, context: LambdaContext): ) compact_stats['providers_failed'] += 1 continue - except Exception as e: - logger.exception( - 'Unexpected error processing provider', - provider_id=provider_id, - compact=compact, - error=str(e), - ) - compact_stats['providers_failed'] += 1 - continue # Bulk index when batch is full if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: @@ -253,7 +248,11 @@ def populate_provider_documents(event: dict, context: LambdaContext): except CCInternalException as e: # Indexing failed after retries, return pagination info for manual retry return _build_error_response( - stats, compact_stats, compact, batch_start_key, str(e), + stats, + compact_stats, + compact, + batch_start_key, + str(e), ) # Index any remaining documents for this compact @@ -264,7 +263,11 @@ def populate_provider_documents(event: dict, context: LambdaContext): except CCInternalException as e: # Indexing failed after retries, return pagination info for manual retry return _build_error_response( - stats, compact_stats, compact, batch_start_key, str(e), + stats, + compact_stats, + compact, + batch_start_key, + str(e), ) # Update overall stats diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 9220a7f6b..13565abfc 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -23,7 +23,6 @@ def __init__(self): verify_certs=True, connection_class=RequestsHttpConnection, pool_maxsize=20, - ) def create_index(self, index_name: str, index_mapping: dict) -> None: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 44efdf432..29e9f61e0 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -188,8 +188,8 @@ def test_pagination_across_invocations_when_time_limit_reached(self, mock_opense from handlers.populate_provider_documents import TIME_THRESHOLD_MS, populate_provider_documents # Time values for mocking (in milliseconds) - PLENTY_OF_TIME_MS = TIME_THRESHOLD_MS + 60000 # Above threshold, continue processing - LOW_TIME_MS = TIME_THRESHOLD_MS - 1000 # Below threshold, trigger timeout + time_before_cutoff = TIME_THRESHOLD_MS + 60000 # before cutoff time, continue processing + time_after_cutoff = TIME_THRESHOLD_MS - 1000 # after cutoff time, trigger timeout # Set up the mock opensearch client mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) @@ -204,7 +204,7 @@ def test_pagination_across_invocations_when_time_limit_reached(self, mock_opense # - Call 1: Processing aslp, plenty of time -> continue # - Call 2: About to process octp, low time -> timeout and return mock_context = MagicMock() - mock_context.get_remaining_time_in_millis.side_effect = [PLENTY_OF_TIME_MS, LOW_TIME_MS] + mock_context.get_remaining_time_in_millis.side_effect = [time_before_cutoff, time_after_cutoff] # Run the first invocation first_result = populate_provider_documents({}, mock_context) @@ -228,7 +228,7 @@ def test_pagination_across_invocations_when_time_limit_reached(self, mock_opense # Mock time to allow completion - needs enough calls for both octp and coun # - Call 1: Processing octp, plenty of time -> continue # - Call 2: Processing coun, plenty of time -> continue - mock_context.get_remaining_time_in_millis.side_effect = [PLENTY_OF_TIME_MS, PLENTY_OF_TIME_MS] + mock_context.get_remaining_time_in_millis.side_effect = [time_before_cutoff, time_before_cutoff] # Build the resume event from the first result resume_event = { @@ -262,7 +262,6 @@ def test_returns_pagination_info_when_bulk_indexing_fails_after_retries(self, mo 3. The developer can use this info to retry from the exact point of failure """ from cc_common.exceptions import CCInternalException - from handlers.populate_provider_documents import TIME_THRESHOLD_MS, populate_provider_documents # Set up the mock opensearch client to raise CCInternalException on second compact diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index b9fb472e4..73c48745a 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -1,6 +1,7 @@ from unittest import TestCase from unittest.mock import MagicMock, patch +from cc_common.exceptions import CCInternalException from opensearchpy.exceptions import ConnectionTimeout, TransportError @@ -121,11 +122,7 @@ def test_bulk_index_calls_internal_client_with_expected_arguments(self): {'index': {'_id': 'provider-2'}}, {'providerId': 'provider-2', 'givenName': 'Jane', 'familyName': 'Smith'}, ] - mock_internal_client.bulk.assert_called_once_with( - body=expected_actions, - index=index_name, - timeout=30 - ) + mock_internal_client.bulk.assert_called_once_with(body=expected_actions, index=index_name, timeout=30) self.assertEqual(expected_response, result) def test_bulk_index_uses_custom_id_field(self): @@ -147,11 +144,7 @@ def test_bulk_index_uses_custom_id_field(self): {'index': {'_id': 'custom-2'}}, {'customId': 'custom-2', 'name': 'Document 2'}, ] - mock_internal_client.bulk.assert_called_once_with( - body=expected_actions, - index=index_name, - timeout=30 - ) + mock_internal_client.bulk.assert_called_once_with(body=expected_actions, index=index_name, timeout=30) def test_bulk_index_returns_early_for_empty_documents(self): """Test that bulk_index returns early without calling the internal client for empty documents.""" @@ -251,7 +244,7 @@ def test_bulk_index_exponential_backoff_caps_at_max(self, mock_sleep): # All calls fail mock_internal_client.bulk.side_effect = ConnectionTimeout('Connection timed out', 503, 'some error') - with self.assertRaises(Exception): + with self.assertRaises(CCInternalException): client.bulk_index(index_name=index_name, documents=documents) # Verify backoff values: 1, 2, 4, 8 (all should be <= MAX_BACKOFF_SECONDS) From d552e57e36725cd435c8beee46355deac80b5787 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 4 Dec 2025 16:50:00 -0600 Subject: [PATCH 047/137] update dev requirement --- .../lambdas/python/purchases/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt index 7c95681ce..e83ce4bd3 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt @@ -167,7 +167,7 @@ urllib3==2.5.0 # docker # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto wheel==0.45.1 # via pip-tools From 7a3be2b7525db3e9aa0f4fb48c348156c3ac4ddd Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 4 Dec 2025 17:09:46 -0600 Subject: [PATCH 048/137] Update node dependency --- backend/compact-connect/lambdas/nodejs/package.json | 2 +- backend/compact-connect/lambdas/nodejs/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/lambdas/nodejs/package.json b/backend/compact-connect/lambdas/nodejs/package.json index 7276357dc..a4e7e7fa4 100644 --- a/backend/compact-connect/lambdas/nodejs/package.json +++ b/backend/compact-connect/lambdas/nodejs/package.json @@ -46,7 +46,7 @@ "@aws-sdk/client-sesv2": "^3.901.0", "@aws-sdk/util-dynamodb": "^3.901.0", "@jusdino-ia/email-builder": "^0.0.9-alpha.3", - "nodemailer": "^7.0.7", + "nodemailer": "^7.0.11", "zod": "^3.23.8" } } diff --git a/backend/compact-connect/lambdas/nodejs/yarn.lock b/backend/compact-connect/lambdas/nodejs/yarn.lock index 998a714d5..9121698ca 100644 --- a/backend/compact-connect/lambdas/nodejs/yarn.lock +++ b/backend/compact-connect/lambdas/nodejs/yarn.lock @@ -4510,10 +4510,10 @@ node-releases@^2.0.18: resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz" integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g== -nodemailer@^7.0.7: - version "7.0.9" - resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-7.0.9.tgz#fe5abd4173e08e01aa243c7cddd612ad8c6ccc18" - integrity sha512-9/Qm0qXIByEP8lEV2qOqcAW7bRpL8CR9jcTwk3NBnHJNmP9fIJ86g2fgmIXqHY+nj55ZEMwWqYAT2QTDpRUYiQ== +nodemailer@^7.0.11: + version "7.0.11" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-7.0.11.tgz#5f7b06afaec20073cff36bea92d1c7395cc3e512" + integrity sha512-gnXhNRE0FNhD7wPSCGhdNh46Hs6nm+uTyg+Kq0cZukNQiYdnCsoQjodNP9BQVG9XrcK/v6/MgpAPBUFyzh9pvw== normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" From f42e0018c2e5b2e1d6556969a89335a79150d571 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 4 Dec 2025 17:23:29 -0600 Subject: [PATCH 049/137] Update PR feedback --- .../python/search/handlers/populate_provider_documents.py | 2 +- .../stacks/search_persistent_stack/provider_search_domain.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 9c7bfec42..e74cb236a 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -47,7 +47,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, and bulk indexes them into the appropriate compact-specific OpenSearch index. - If processing cannot complete within 13 minutes, the function returns pagination + If processing cannot complete within 12 minutes, the function returns pagination information that can be passed as input to continue processing. :param event: Lambda event with optional pagination parameters: diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index bbb1b2512..318701a1b 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -417,7 +417,7 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): period=Duration.minutes(5), statistic='Average', ), - evaluation_periods=3, # 30 minutes sustained + evaluation_periods=3, # 15 minutes sustained threshold=70, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, From 9f390dff7f0701e40b6b0b2a71029deadf085005 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 10:09:32 -0600 Subject: [PATCH 050/137] Update engine to latest --- .../provider_search_domain.py | 89 ++++++++++++++++--- .../tests/app/test_search_persistent_stack.py | 51 +++++++++-- 2 files changed, 124 insertions(+), 16 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index 318701a1b..eacd4eefe 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -154,7 +154,7 @@ def __init__( self.domain = Domain( self, 'Domain', - version=EngineVersion.OPENSEARCH_3_1, + version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, # VPC configuration for network isolation vpc=vpc_stack.vpc, @@ -366,8 +366,10 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): These proactive thresholds give the DevOps team time to plan scaling activities: - Free Storage Space < 50% of allocated capacity - - JVM Memory Pressure > 70% - - CPU Utilization > 60% + - JVM Memory Pressure > 85% + - CPU Utilization > 70% + - Cluster Status (red/yellow) for critical and degraded states + - Automated Snapshot Failure for backup issues :param environment_name: The deployment environment name :param alarm_topic: The SNS topic to send alarm notifications to @@ -405,7 +407,7 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): ), ).add_alarm_action(SnsAction(alarm_topic)) - # Alarm: JVM Memory Pressure > 70% + # Alarm: JVM Memory Pressure > 85% # Sustained high memory pressure indicates need for instance scaling Alarm( self, @@ -415,20 +417,20 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): metric_name='JVMMemoryPressure', dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, period=Duration.minutes(5), - statistic='Average', + statistic='Maximum', ), evaluation_periods=3, # 15 minutes sustained - threshold=70, + threshold=85, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, alarm_description=( - f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 70%. ' + f'OpenSearch Domain {self.domain.domain_name} JVM memory pressure is above 85%. ' 'This indicates the cluster is using a significant portion of its heap memory. ' 'Consider scaling to larger instance types if pressure continues to increase.' ), ).add_alarm_action(SnsAction(alarm_topic)) - # Alarm: CPU Utilization > 60% + # Alarm: CPU Utilization > 70% # Sustained high CPU indicates need for more compute capacity Alarm( self, @@ -441,16 +443,83 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): statistic='Average', ), evaluation_periods=3, # 15 minutes sustained - threshold=60, + threshold=70, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, alarm_description=( - f'OpenSearch Domain {self.domain.domain_name} CPU utilization has been above 60% for 15 minutes. ' + f'OpenSearch Domain {self.domain.domain_name} CPU utilization has been above 70% for 15 minutes. ' 'This indicates sustained high load. Review metrics and consider scaling to larger instance types ' 'or adding more data nodes to distribute the load.' ), ).add_alarm_action(SnsAction(alarm_topic)) + # Alarm: Cluster Status RED - Critical + # Red status indicates critical issues requiring immediate attention + Alarm( + self, + 'ClusterStatusRedAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='ClusterStatus.red', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, + period=Duration.minutes(1), + statistic='Sum', + ), + evaluation_periods=1, # Alert immediately when red + threshold=1, + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} cluster status is RED. ' + 'This indicates critical issues requiring immediate attention. ' + 'Check cluster health and consider scaling if resource-constrained.' + ), + ).add_alarm_action(SnsAction(alarm_topic)) + + # Alarm: Cluster Status YELLOW - Degraded + # Yellow status indicates degraded state that should be monitored + Alarm( + self, + 'ClusterStatusYellowAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='ClusterStatus.yellow', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, + period=Duration.minutes(5), + statistic='Sum', + ), + evaluation_periods=1, # Alert when yellow status is detected + threshold=1, + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} cluster status is YELLOW. ' + 'This indicates degraded state. Monitor closely and consider scaling if persistent.' + ), + ).add_alarm_action(SnsAction(alarm_topic)) + + # Alarm: Automated Snapshot Failure + # Snapshot failures may indicate resource constraints or other issues + Alarm( + self, + 'AutomatedSnapshotFailureAlarm', + metric=Metric( + namespace='AWS/ES', + metric_name='AutomatedSnapshotFailure', + dimensions_map={'DomainName': self.domain.domain_name, 'ClientId': stack.account}, + period=Duration.hours(1), + statistic='Sum', + ), + evaluation_periods=1, + threshold=1, + comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + alarm_description=( + f'OpenSearch Domain {self.domain.domain_name} automated snapshot has failed. ' + 'This may indicate resource constraints or other issues requiring investigation.' + ), + ).add_alarm_action(SnsAction(alarm_topic)) + def _add_domain_suppressions(self, environment_name: str): """ Add CDK Nag suppressions for OpenSearch Domain configuration. diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 1f6948660..c34b3f2a2 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -44,7 +44,7 @@ def test_opensearch_version(self): search_template.has_resource_properties( 'AWS::OpenSearchService::Domain', { - 'EngineVersion': 'OpenSearch_3.1', + 'EngineVersion': 'OpenSearch_3.3', }, ) @@ -170,10 +170,13 @@ def test_capacity_alarms_configured(self): """ Test that capacity monitoring alarms are configured for proactive scaling. - Verifies three critical alarms: + Verifies six critical alarms: 1. Free Storage Space < 50% threshold - 2. JVM Memory Pressure > 70% threshold - 3. CPU Utilization > 60% threshold + 2. JVM Memory Pressure > 85% threshold + 3. CPU Utilization > 70% threshold + 4. Cluster Status RED for critical issues + 5. Cluster Status YELLOW for degraded state + 6. Automated Snapshot Failure for backup issues These alarms give DevOps team time to plan scaling activities before hitting limits. """ @@ -199,7 +202,7 @@ def test_capacity_alarms_configured(self): { 'MetricName': 'JVMMemoryPressure', 'Namespace': 'AWS/ES', - 'Threshold': 70, + 'Threshold': 85, 'ComparisonOperator': 'GreaterThanThreshold', 'EvaluationPeriods': 3, }, @@ -211,12 +214,48 @@ def test_capacity_alarms_configured(self): { 'MetricName': 'CPUUtilization', 'Namespace': 'AWS/ES', - 'Threshold': 60, + 'Threshold': 70, 'ComparisonOperator': 'GreaterThanThreshold', 'EvaluationPeriods': 3, # 15 minutes sustained }, ) + # Verify Cluster Status RED Alarm + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'ClusterStatus.red', + 'Namespace': 'AWS/ES', + 'Threshold': 1, + 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', + 'EvaluationPeriods': 1, + }, + ) + + # Verify Cluster Status YELLOW Alarm + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'ClusterStatus.yellow', + 'Namespace': 'AWS/ES', + 'Threshold': 1, + 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', + 'EvaluationPeriods': 1, + }, + ) + + # Verify Automated Snapshot Failure Alarm + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'AutomatedSnapshotFailure', + 'Namespace': 'AWS/ES', + 'Threshold': 1, + 'ComparisonOperator': 'GreaterThanThreshold', + 'EvaluationPeriods': 1, + }, + ) + def test_multi_index_queries_disabled(self): """ Test that multi-index queries are disabled for security. From 1f49f71e5074eded5527e7a1baa9dcfb4e737a78 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 13:57:22 -0600 Subject: [PATCH 051/137] Add endpoint to search privileges --- .../lambdas/python/search/handlers/search.py | 349 ++++++++++++++++++ .../search/handlers/search_providers.py | 109 ------ .../tests/function/test_search_privileges.py | 308 ++++++++++++++++ .../tests/function/test_search_providers.py | 58 +-- .../stacks/search_api_stack/v1_api/api.py | 12 +- .../search_api_stack/v1_api/api_model.py | 196 +++++++--- .../v1_api/privilege_search.py | 62 ++++ .../v1_api/provider_search.py | 2 +- .../search_persistent_stack/__init__.py | 6 +- ...providers_handler.py => search_handler.py} | 16 +- 10 files changed, 923 insertions(+), 195 deletions(-) create mode 100644 backend/compact-connect/lambdas/python/search/handlers/search.py delete mode 100644 backend/compact-connect/lambdas/python/search/handlers/search_providers.py create mode 100644 backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py create mode 100644 backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py rename backend/compact-connect/stacks/search_persistent_stack/{search_providers_handler.py => search_handler.py} (85%) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py new file mode 100644 index 000000000..fa0a70266 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -0,0 +1,349 @@ +from aws_lambda_powertools.utilities.typing import LambdaContext +from cc_common.config import logger +from cc_common.data_model.schema.provider.api import ( + ProviderGeneralResponseSchema, + SearchProvidersRequestSchema, + StatePrivilegeGeneralResponseSchema, +) +from cc_common.exceptions import CCInvalidRequestException +from cc_common.utils import api_handler +from marshmallow import ValidationError +from opensearch_client import OpenSearchClient + +# Default and maximum page sizes for search results +DEFAULT_SIZE = 10 +MAX_SIZE = 100 + + +@api_handler +def search_api_handler(event: dict, context: LambdaContext): + """ + Main entry point for search API. + Routes to the appropriate handler based on the HTTP method and resource path. + + :param event: Standard API Gateway event, API schema documented in the CDK ApiStack + :param context: Lambda context + """ + # Extract the HTTP method and resource path + http_method = event.get('httpMethod') + resource_path = event.get('resource') + + # Route to the appropriate handler + api_method = (http_method, resource_path) + match api_method: + case ('POST', '/v1/compacts/{compact}/providers/search'): + return _search_providers(event, context) + case ('POST', '/v1/compacts/{compact}/privileges/search'): + return _search_privileges(event, context) + + # If we get here, the method/resource combination is not supported + raise CCInvalidRequestException(f'Unsupported method or resource: {http_method} {resource_path}') + + +def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument + """ + Search providers using OpenSearch. + + This endpoint accepts an OpenSearch DSL query body and returns sanitized provider records. + Pagination follows OpenSearch DSL using `from`/`size` or `search_after` with `sort`. + + See: https://docs.opensearch.org/latest/search-plugins/searching-data/paginate/ + + :param event: Standard API Gateway event, API schema documented in the CDK ApiStack + :param LambdaContext context: + :return: Dictionary with providers array and pagination metadata + """ + compact = event['pathParameters']['compact'] + + # Parse and validate the request body using the schema + body = _parse_and_validate_request_body(event) + + # Build the OpenSearch search body + search_body = _build_opensearch_search_body(body) + + # Build the index name for this compact + index_name = f'compact_{compact}_providers' + + logger.info('Executing OpenSearch provider search', compact=compact, index_name=index_name) + + # Execute the search + client = OpenSearchClient() + response = client.search(index_name=index_name, body=search_body) + + # Extract hits from the response + hits_data = response.get('hits', {}) + hits = hits_data.get('hits', []) + total = hits_data.get('total', {}) + + # Sanitize the provider records using ProviderGeneralResponseSchema + general_schema = ProviderGeneralResponseSchema() + sanitized_providers = [] + last_sort = None + + for hit in hits: + source = hit.get('_source', {}) + try: + sanitized_provider = general_schema.load(source) + sanitized_providers.append(sanitized_provider) + # Track the sort values from the last hit for search_after pagination + last_sort = hit.get('sort') + except ValidationError as e: + # Log the error but continue processing other records + logger.warning( + 'Failed to sanitize provider record', + provider_id=source.get('providerId'), + errors=e.messages, + ) + + # Build response following OpenSearch DSL structure + response_body = { + 'providers': sanitized_providers, + 'total': total, + } + + # Include sort values from last hit to enable search_after pagination + if last_sort is not None: + response_body['lastSort'] = last_sort + + return response_body + + +def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument + """ + Search privileges using OpenSearch. + + This endpoint accepts an OpenSearch DSL query body and returns flattened privilege records. + Privileges are extracted from provider documents and combined with license data. + Pagination follows OpenSearch DSL using `from`/`size` or `search_after` with `sort`. + + :param event: Standard API Gateway event, API schema documented in the CDK ApiStack + :param LambdaContext context: + :return: Dictionary with privileges array and pagination metadata + """ + compact = event['pathParameters']['compact'] + + # Parse and validate the request body using the schema + body = _parse_and_validate_request_body(event) + + # Build the OpenSearch search body + search_body = _build_opensearch_search_body(body) + + # Build the index name for this compact + index_name = f'compact_{compact}_providers' + + logger.info('Executing OpenSearch privilege search', compact=compact, index_name=index_name) + + # Execute the search + client = OpenSearchClient() + response = client.search(index_name=index_name, body=search_body) + + # Extract hits from the response + hits_data = response.get('hits', {}) + hits = hits_data.get('hits', []) + total = hits_data.get('total', {}) + + # Extract and flatten privileges from provider records + flattened_privileges = [] + last_sort = None + privilege_schema = StatePrivilegeGeneralResponseSchema() + + for hit in hits: + provider = hit.get('_source', {}) + try: + # Extract privileges and flatten them with license data + provider_privileges = _extract_flattened_privileges(provider) + for flattened_privilege in provider_privileges: + try: + # Sanitize using StatePrivilegeGeneralResponseSchema + sanitized_privilege = privilege_schema.load(flattened_privilege) + flattened_privileges.append(sanitized_privilege) + except ValidationError as e: + logger.warning( + 'Failed to sanitize flattened privilege record', + provider_id=provider.get('providerId'), + privilege_id=flattened_privilege.get('privilegeId'), + errors=e.messages, + ) + # Track the sort values from the last hit for search_after pagination + last_sort = hit.get('sort') + except Exception as e: # noqa: BLE001 broad-exception-caught + logger.warning( + 'Failed to process provider privileges', + provider_id=provider.get('providerId'), + error=str(e), + ) + + # Build response following OpenSearch DSL structure + response_body = { + 'privileges': flattened_privileges, + 'total': total, + } + + # Include sort values from last hit to enable search_after pagination + if last_sort is not None: + response_body['lastSort'] = last_sort + + return response_body + + +def _parse_and_validate_request_body(event: dict) -> dict: + """ + Parse and validate the request body using the SearchProvidersRequestSchema. + + :param event: API Gateway event + :return: Validated request body + :raises CCInvalidRequestException: If the request body is invalid + """ + try: + schema = SearchProvidersRequestSchema() + return schema.loads(event['body']) + except ValidationError as e: + logger.warning('Invalid request body', errors=e.messages) + raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e + + +def _build_opensearch_search_body(body: dict) -> dict: + """ + Build the OpenSearch search body from the validated request. + + :param body: Validated request body + :return: OpenSearch search body + :raises CCInvalidRequestException: If search_after is used without sort + """ + search_body = { + 'query': body.get('query', {'match_all': {}}), + } + + # Add pagination parameters following OpenSearch DSL + # 'from_' in Python maps to 'from' in the JSON (due to data_key in schema) + from_param = body.get('from_') + if from_param is not None: + search_body['from'] = from_param + + size = body.get('size', DEFAULT_SIZE) + search_body['size'] = min(size, MAX_SIZE) + + # Add sort if provided - required for search_after pagination + sort = body.get('sort') + if sort is not None: + search_body['sort'] = sort + + # Add search_after for cursor-based pagination + search_after = body.get('search_after') + if search_after is not None: + search_body['search_after'] = search_after + # search_after requires sort to be specified + if 'sort' not in search_body: + raise CCInvalidRequestException('sort is required when using search_after pagination') + + return search_body + + +def _extract_flattened_privileges(provider: dict) -> list[dict]: + """ + Extract and flatten privileges from a provider document. + + This function combines privilege data with license data to create flattened + privilege records similar to what the state API returns. + + :param provider: Provider document from OpenSearch + :return: List of flattened privilege records + """ + privileges = provider.get('privileges', []) + licenses = provider.get('licenses', []) + + if not privileges: + return [] + + flattened_privileges = [] + + for privilege in privileges: + # Find matching license based on licenseJurisdiction and licenseType + matching_license = _find_matching_license( + licenses=licenses, + license_jurisdiction=privilege.get('licenseJurisdiction'), + license_type=privilege.get('licenseType'), + ) + + if matching_license is None: + logger.warning( + 'No matching license found for privilege', + provider_id=provider.get('providerId'), + privilege_id=privilege.get('privilegeId'), + license_jurisdiction=privilege.get('licenseJurisdiction'), + license_type=privilege.get('licenseType'), + ) + # Skip this privilege if no matching license is found + continue + + flattened_privilege = _create_flattened_privilege(privilege, matching_license, provider) + flattened_privileges.append(flattened_privilege) + + return flattened_privileges + + +def _find_matching_license(licenses: list[dict], license_jurisdiction: str, license_type: str) -> dict | None: + """ + Find a license that matches the given jurisdiction and license type. + + :param licenses: List of license records + :param license_jurisdiction: The jurisdiction to match + :param license_type: The license type to match + :return: The matching license or None if not found + """ + for license_record in licenses: + if ( + license_record.get('jurisdiction') == license_jurisdiction + and license_record.get('licenseType') == license_type + ): + return license_record + return None + + +def _create_flattened_privilege(privilege: dict, license_record: dict, provider: dict) -> dict: + """ + Create a flattened privilege record by combining privilege and license data. + + This mirrors the logic in state_api.py _create_flattened_privilege function. + + :param privilege: Privilege record + :param license_record: Matching license record + :param provider: Provider record (for email if registered) + :return: Flattened privilege record with combined data + """ + # Start with privilege data and set type + flattened = dict(privilege) + flattened['type'] = 'statePrivilege' + + # Add compactConnectRegisteredEmailAddress if present + if provider.get('compactConnectRegisteredEmailAddress') is not None: + flattened['compactConnectRegisteredEmailAddress'] = provider.get('compactConnectRegisteredEmailAddress') + + # Remove fields from license that would conflict with privilege fields + license_copy = dict(license_record) + conflicting_fields = { + 'providerId', + 'compact', + 'jurisdiction', + 'licenseType', + 'type', + 'pk', + 'sk', + 'dateOfIssuance', + 'dateOfRenewal', + 'dateOfUpdate', + 'dateOfExpiration', + 'status', + 'administratorSetStatus', + # Also remove nested objects that don't belong in flattened output + 'adverseActions', + 'investigations', + } + for field in conflicting_fields: + license_copy.pop(field, None) + + # Merge license data into flattened record + # License fields like givenName, familyName, npi, etc. get added + flattened.update(license_copy) + + return flattened diff --git a/backend/compact-connect/lambdas/python/search/handlers/search_providers.py b/backend/compact-connect/lambdas/python/search/handlers/search_providers.py deleted file mode 100644 index dda3b8deb..000000000 --- a/backend/compact-connect/lambdas/python/search/handlers/search_providers.py +++ /dev/null @@ -1,109 +0,0 @@ -from aws_lambda_powertools.utilities.typing import LambdaContext -from cc_common.config import logger -from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema, SearchProvidersRequestSchema -from cc_common.exceptions import CCInvalidRequestException -from cc_common.utils import api_handler -from marshmallow import ValidationError -from opensearch_client import OpenSearchClient - -# Default and maximum page sizes for search results -DEFAULT_SIZE = 10 -MAX_SIZE = 100 - - -@api_handler -def search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument - """ - Search providers using OpenSearch. - - This endpoint accepts an OpenSearch DSL query body and returns sanitized provider records. - Pagination follows OpenSearch DSL using `from`/`size` or `search_after` with `sort`. - - See: https://docs.opensearch.org/latest/search-plugins/searching-data/paginate/ - - :param event: Standard API Gateway event, API schema documented in the CDK ApiStack - :param LambdaContext context: - :return: Dictionary with providers array and pagination metadata - """ - compact = event['pathParameters']['compact'] - - # Parse and validate the request body using the schema - try: - schema = SearchProvidersRequestSchema() - body = schema.loads(event['body']) - except ValidationError as e: - logger.warning('Invalid request body', errors=e.messages) - raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e - - # Build the OpenSearch search body - pass through parameters directly - search_body = { - 'query': body.get('query', {'match_all': {}}), - } - - # Add pagination parameters following OpenSearch DSL - # 'from_' in Python maps to 'from' in the JSON (due to data_key in schema) - from_param = body.get('from_') - if from_param is not None: - search_body['from'] = from_param - - size = body.get('size', DEFAULT_SIZE) - search_body['size'] = min(size, MAX_SIZE) - - # Add sort if provided - required for search_after pagination - sort = body.get('sort') - if sort is not None: - search_body['sort'] = sort - - # Add search_after for cursor-based pagination - search_after = body.get('search_after') - if search_after is not None: - search_body['search_after'] = search_after - # search_after requires sort to be specified - if 'sort' not in search_body: - raise CCInvalidRequestException('sort is required when using search_after pagination') - - # Build the index name for this compact - index_name = f'compact_{compact}_providers' - - logger.info('Executing OpenSearch query', compact=compact, index_name=index_name) - - # Execute the search - client = OpenSearchClient() - response = client.search(index_name=index_name, body=search_body) - - # Extract hits from the response - hits_data = response.get('hits', {}) - hits = hits_data.get('hits', []) - total = hits_data.get('total', {}) - - # Sanitize the provider records using ProviderGeneralResponseSchema - general_schema = ProviderGeneralResponseSchema() - sanitized_providers = [] - last_sort = None - - for hit in hits: - source = hit.get('_source', {}) - try: - sanitized_provider = general_schema.load(source) - sanitized_providers.append(sanitized_provider) - # Track the sort values from the last hit for search_after pagination - last_sort = hit.get('sort') - except ValidationError as e: - # Log the error but continue processing other records - logger.warning( - 'Failed to sanitize provider record', - provider_id=source.get('providerId'), - errors=e.messages, - ) - - # Build response following OpenSearch DSL structure - response_body = { - 'providers': sanitized_providers, - 'total': total, - } - - # Include sort values from last hit to enable search_after pagination - if last_sort is not None: - response_body['lastSort'] = last_sort - - return response_body diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py new file mode 100644 index 000000000..b1ee26995 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -0,0 +1,308 @@ +import json +from unittest.mock import Mock, patch + +from moto import mock_aws + +from . import TstFunction + + +@mock_aws +class TestSearchPrivileges(TstFunction): + """Test suite for search_api_handler - privilege search functionality.""" + + def setUp(self): + super().setUp() + + def _create_api_event(self, compact: str, body: dict = None) -> dict: + """Create a standard API Gateway event for search_privileges.""" + return { + 'resource': '/v1/compacts/{compact}/privileges/search', + 'path': f'/v1/compacts/{compact}/privileges/search', + 'httpMethod': 'POST', + 'headers': { + 'accept': 'application/json', + 'content-type': 'application/json', + 'Content-Type': 'application/json', + 'origin': 'https://example.org', + 'Host': 'api.test.example.com', + }, + 'multiValueHeaders': {}, + 'queryStringParameters': None, + 'pathParameters': {'compact': compact}, + 'requestContext': { + 'resourcePath': '/v1/compacts/{compact}/privileges/search', + 'httpMethod': 'POST', + 'authorizer': { + 'claims': { + 'sub': 'test-user-id', + 'cognito:username': 'test-user', + } + }, + }, + 'body': json.dumps(body) if body else None, + 'isBase64Encoded': False, + } + + def _when_testing_mock_opensearch_client(self, mock_opensearch_client, search_response: dict = None): + """ + Configure the mock OpenSearchClient for testing. + + :param mock_opensearch_client: The patched OpenSearchClient class + :param search_response: The response to return from the search method + :return: The mock client instance + """ + if not search_response: + search_response = { + 'hits': { + 'total': {'value': 0, 'relation': 'eq'}, + 'hits': [], + } + } + + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.search.return_value = search_response + return mock_client_instance + + def _create_mock_provider_hit_with_privileges( + self, + provider_id: str = '00000000-0000-0000-0000-000000000001', + compact: str = 'aslp', + sort_values: list = None, + ) -> dict: + """Create a mock OpenSearch hit for a provider document with privileges and licenses.""" + hit = { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license-home', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': 'audiologist', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2020-01-01', + 'dateOfRenewal': '2024-01-01', + 'dateOfExpiration': '2025-12-31', + 'npi': '1234567890', + 'licenseNumber': 'AUD-12345', + } + ], + 'privileges': [ + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-001', + 'status': 'active', + } + ], + }, + } + if sort_values: + hit['sort'] = sort_values + return hit + + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_returns_flattened_privileges(self, mock_opensearch_client): + """Test that privilege search returns flattened privilege records.""" + from handlers.search import search_api_handler + + # Create a mock response with provider hits containing privileges + mock_hit = self._create_mock_provider_hit_with_privileges() + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [mock_hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + + # Verify response structure has 'privileges' instead of 'providers' + self.assertIn('privileges', body) + self.assertNotIn('providers', body) + self.assertEqual(1, len(body['privileges'])) + + # Verify the flattened privilege has both privilege and license fields + privilege = body['privileges'][0] + self.assertEqual('statePrivilege', privilege['type']) + self.assertEqual('00000000-0000-0000-0000-000000000001', privilege['providerId']) + self.assertEqual('ky', privilege['jurisdiction']) + self.assertEqual('oh', privilege['licenseJurisdiction']) + self.assertEqual('audiologist', privilege['licenseType']) + self.assertEqual('PRIV-001', privilege['privilegeId']) + self.assertEqual('active', privilege['status']) + + # Verify license fields were merged + self.assertEqual('John', privilege['givenName']) + self.assertEqual('Doe', privilege['familyName']) + self.assertEqual('1234567890', privilege['npi']) + self.assertEqual('AUD-12345', privilege['licenseNumber']) + + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_with_empty_results(self, mock_opensearch_client): + """Test that privilege search returns empty array when no results.""" + from handlers.search import search_api_handler + + search_response = { + 'hits': { + 'total': {'value': 0, 'relation': 'eq'}, + 'hits': [], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual({'privileges': [], 'total': {'relation': 'eq', 'value': 0}}, body) + + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_skips_provider_without_privileges(self, mock_opensearch_client): + """Test that providers without privileges don't add entries.""" + from handlers.search import search_api_handler + + # Create a provider hit without privileges + hit = { + '_index': 'compact_aslp_providers', + '_id': 'provider-1', + '_score': 1.0, + '_source': { + 'providerId': 'provider-1', + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': 'aslp', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'Jane', + 'familyName': 'Smith', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '03-20', + 'licenses': [], + 'privileges': [], + }, + } + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual(0, len(body['privileges'])) + + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_includes_last_sort(self, mock_opensearch_client): + """Test that lastSort is included in privilege search response.""" + from handlers.search import search_api_handler + + mock_hit = self._create_mock_provider_hit_with_privileges( + sort_values=['provider-uuid-123', '2024-01-15T10:30:00+00:00'] + ) + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [mock_hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event( + 'aslp', + body={ + 'query': {'match_all': {}}, + 'sort': [{'providerId': 'asc'}], + }, + ) + + response = search_api_handler(event, self.mock_context) + + body = json.loads(response['body']) + self.assertIn('lastSort', body) + self.assertEqual(['provider-uuid-123', '2024-01-15T10:30:00+00:00'], body['lastSort']) + + def test_unsupported_route_returns_400(self): + """Test that unsupported routes return a 400 error.""" + from handlers.search import search_api_handler + + # Create event with unsupported route + event = { + 'resource': '/v1/compacts/{compact}/unknown/search', + 'path': '/v1/compacts/aslp/unknown/search', + 'httpMethod': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'origin': 'https://example.org', + }, + 'multiValueHeaders': {}, + 'queryStringParameters': None, + 'pathParameters': {'compact': 'aslp'}, + 'requestContext': { + 'resourcePath': '/v1/compacts/{compact}/unknown/search', + 'httpMethod': 'POST', + 'authorizer': { + 'claims': { + 'sub': 'test-user-id', + 'cognito:username': 'test-user', + } + }, + }, + 'body': json.dumps({'query': {'match_all': {}}}), + 'isBase64Encoded': False, + } + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Unsupported method or resource', body['message']) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 6e67816e5..8af18ff08 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -8,7 +8,7 @@ @mock_aws class TestSearchProviders(TstFunction): - """Test suite for search_providers handler.""" + """Test suite for search_api_handler - provider search functionality.""" def setUp(self): super().setUp() @@ -16,7 +16,7 @@ def setUp(self): def _create_api_event(self, compact: str, body: dict = None) -> dict: """Create a standard API Gateway event for search_providers.""" return { - 'resource': f'/v1/compacts/{compact}/providers/search', + 'resource': '/v1/compacts/{compact}/providers/search', 'path': f'/v1/compacts/{compact}/providers/search', 'httpMethod': 'POST', 'headers': { @@ -30,7 +30,7 @@ def _create_api_event(self, compact: str, body: dict = None) -> dict: 'queryStringParameters': None, 'pathParameters': {'compact': compact}, 'requestContext': { - 'resourcePath': f'/v1/compacts/{compact}/providers/search', + 'resourcePath': '/v1/compacts/{compact}/providers/search', 'httpMethod': 'POST', 'authorizer': { 'claims': { @@ -95,17 +95,17 @@ def _create_mock_provider_hit( hit['sort'] = sort_values return hit - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_basic_search_with_match_all_query(self, mock_opensearch_client): """Test that a basic search with no query uses match_all.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create event with minimal body - just the required query field event = self._create_api_event(compact='aslp', body={'query': {'match_all': {}}}) - response = search_providers(event, self.mock_context) + response = search_api_handler(event, self.mock_context) # Verify OpenSearchClient was instantiated and search was called mock_opensearch_client.assert_called_once() @@ -121,10 +121,10 @@ def test_basic_search_with_match_all_query(self, mock_opensearch_client): body = json.loads(response['body']) self.assertEqual({'providers': [], 'total': {'relation': 'eq', 'value': 0}}, body) - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_with_custom_query(self, mock_opensearch_client): """Test that a custom OpenSearch query is passed through correctly.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) @@ -139,7 +139,7 @@ def test_search_with_custom_query(self, mock_opensearch_client): } event = self._create_api_event('aslp', body={'query': custom_query, 'from': 20}) - search_providers(event, self.mock_context) + search_api_handler(event, self.mock_context) # Verify the custom query was passed through mock_client_instance.search.assert_called_once_with( @@ -151,26 +151,26 @@ def test_search_with_custom_query(self, mock_opensearch_client): }, ) - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_size_capped_at_max(self, mock_opensearch_client): """Test that size parameter is capped at MAX_SIZE (100).""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) # Request size larger than MAX_SIZE event = self._create_api_event('aslp', body={'query': {'match_all': {}}, 'size': 500}) - search_providers(event, self.mock_context) + search_api_handler(event, self.mock_context) call_args = mock_client_instance.search.call_args search_body = call_args.kwargs['body'] self.assertEqual(100, search_body['size']) # Capped at MAX_SIZE - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_with_sort_parameter(self, mock_opensearch_client): """Test that sort parameter is included in the search body.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) @@ -185,7 +185,7 @@ def test_search_with_sort_parameter(self, mock_opensearch_client): }, ) - search_providers(event, self.mock_context) + search_api_handler(event, self.mock_context) mock_client_instance.search.assert_called_once_with( index_name='compact_aslp_providers', @@ -197,10 +197,10 @@ def test_search_with_sort_parameter(self, mock_opensearch_client): }, ) - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_after_without_sort_returns_400(self, mock_opensearch_client): """Test that search_after without sort raises an error.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler self._when_testing_mock_opensearch_client(mock_opensearch_client) @@ -213,7 +213,7 @@ def test_search_after_without_sort_returns_400(self, mock_opensearch_client): }, ) - response = search_providers(event, self.mock_context) + response = search_api_handler(event, self.mock_context) self.assertEqual(400, response['statusCode']) body = json.loads(response['body']) @@ -221,21 +221,21 @@ def test_search_after_without_sort_returns_400(self, mock_opensearch_client): def test_invalid_request_body_returns_400(self): """Test that an invalid request body returns a 400 error.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler # Create event with missing required 'query' field event = self._create_api_event('aslp', body={'size': 10}) - response = search_providers(event, self.mock_context) + response = search_api_handler(event, self.mock_context) self.assertEqual(400, response['statusCode']) body = json.loads(response['body']) self.assertIn('Invalid request', body['message']) - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_returns_sanitized_providers(self, mock_opensearch_client): """Test that provider records are sanitized through ProviderGeneralResponseSchema.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler # Create a mock response with provider hits mock_hit = self._create_mock_provider_hit() @@ -249,7 +249,7 @@ def test_search_returns_sanitized_providers(self, mock_opensearch_client): event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) - response = search_providers(event, self.mock_context) + response = search_api_handler(event, self.mock_context) self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) @@ -278,10 +278,10 @@ def test_search_returns_sanitized_providers(self, mock_opensearch_client): body, ) - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_response_includes_last_sort_for_pagination(self, mock_opensearch_client): """Test that lastSort is included in response for search_after pagination.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler # Create hits with sort values mock_hit = self._create_mock_provider_hit(sort_values=['provider-uuid-123', '2024-01-15T10:30:00+00:00']) @@ -301,16 +301,16 @@ def test_search_response_includes_last_sort_for_pagination(self, mock_opensearch }, ) - response = search_providers(event, self.mock_context) + response = search_api_handler(event, self.mock_context) body = json.loads(response['body']) self.assertIn('lastSort', body) self.assertEqual(['provider-uuid-123', '2024-01-15T10:30:00+00:00'], body['lastSort']) - @patch('handlers.search_providers.OpenSearchClient') + @patch('handlers.search.OpenSearchClient') def test_search_uses_correct_index_for_compact(self, mock_opensearch_client): """Test that the correct index name is used based on the compact parameter.""" - from handlers.search_providers import search_providers + from handlers.search import search_api_handler mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) @@ -319,7 +319,7 @@ def test_search_uses_correct_index_for_compact(self, mock_opensearch_client): mock_client_instance.reset_mock() event = self._create_api_event(compact, body={'query': {'match_all': {}}}) - search_providers(event, self.mock_context) + search_api_handler(event, self.mock_context) call_args = mock_client_instance.search.call_args self.assertEqual(f'compact_{compact}_providers', call_args.kwargs['index_name']) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py index 9d063f351..54c156f1c 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py @@ -3,6 +3,7 @@ from aws_cdk.aws_apigateway import AuthorizationType, IResource, MethodOptions from stacks import persistent_stack, search_persistent_stack +from stacks.search_api_stack.v1_api.privilege_search import PrivilegeSearch from stacks.search_api_stack.v1_api.provider_search import ProviderSearch from .api_model import ApiModel @@ -46,9 +47,18 @@ def __init__( # POST /v1/compacts/{compact}/providers providers_resource = self.compact_resource.add_resource('providers') - self.provider_management = ProviderSearch( + self.provider_search = ProviderSearch( resource=providers_resource, method_options=read_auth_method_options, search_persistent_stack=search_persistent_stack, api_model=self.api_model, ) + + # POST /v1/compacts/{compact}/privileges + privileges_resource = self.compact_resource.add_resource('privileges') + self.privilege_search = PrivilegeSearch( + resource=privileges_resource, + method_options=read_auth_method_options, + search_persistent_stack=search_persistent_stack, + api_model=self.api_model, + ) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py index 37f65ba21..72c350614 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py @@ -18,54 +18,87 @@ def __init__(self, api: cc_api.CCApi): self.api = api @property - def search_providers_request_model(self) -> Model: + def _common_search_request_schema(self) -> JsonSchema: """ - Return the search providers request model, which should only be created once per API. + Return the common search request schema used by both provider and privilege search endpoints. - This model closely mirrors OpenSearch DSL for pagination using search_after. + This schema closely mirrors OpenSearch DSL for pagination using search_after. See: https://docs.opensearch.org/latest/search-plugins/searching-data/paginate/ """ + return JsonSchema( + type=JsonSchemaType.OBJECT, + additional_properties=False, + required=['query'], + properties={ + 'query': JsonSchema( + type=JsonSchemaType.OBJECT, + description='The OpenSearch query body', + ), + 'from': JsonSchema( + type=JsonSchemaType.INTEGER, + minimum=0, + description='Starting document offset for pagination', + ), + 'size': JsonSchema( + type=JsonSchemaType.INTEGER, + minimum=1, + # setting low limit for now, as this search endpoint is only used by the UI client, + # and we don't anticipate needing to support more than 100 records per request + maximum=100, + description='Number of results to return', + ), + 'sort': JsonSchema( + type=JsonSchemaType.ARRAY, + description='Sort order for results (required for search_after pagination)', + items=JsonSchema(type=JsonSchemaType.OBJECT), + ), + 'search_after': JsonSchema( + type=JsonSchemaType.ARRAY, + description='Sort values from the last hit of the previous page for cursor-based pagination', + ), + }, + ) + + @property + def search_providers_request_model(self) -> Model: + """ + Return the search providers request model, which should only be created once per API. + """ if hasattr(self.api, '_v1_search_providers_request_model'): return self.api._v1_search_providers_request_model self.api._v1_search_providers_request_model = self.api.add_model( 'V1SearchProvidersRequestModel', description='Search providers request model following OpenSearch DSL', - schema=JsonSchema( - type=JsonSchemaType.OBJECT, - additional_properties=False, - required=['query'], - properties={ - 'query': JsonSchema( - type=JsonSchemaType.OBJECT, - description='The OpenSearch query body', - ), - 'from': JsonSchema( - type=JsonSchemaType.INTEGER, - minimum=0, - description='Starting document offset for pagination', - ), - 'size': JsonSchema( - type=JsonSchemaType.INTEGER, - minimum=1, - # setting low limit for now, as this search endpoint is only used by the UI client, - # and we don't anticipate needing to support more than 100 records per request - maximum=100, - description='Number of results to return', - ), - 'sort': JsonSchema( - type=JsonSchemaType.ARRAY, - description='Sort order for results (required for search_after pagination)', - items=JsonSchema(type=JsonSchemaType.OBJECT), - ), - 'search_after': JsonSchema( - type=JsonSchemaType.ARRAY, - description='Sort values from the last hit of the previous page for cursor-based pagination', - ), - }, - ), + schema=self._common_search_request_schema, ) return self.api._v1_search_providers_request_model + @property + def search_privileges_request_model(self) -> Model: + """ + Return the search privileges request model, which should only be created once per API. + """ + if hasattr(self.api, '_v1_search_privileges_request_model'): + return self.api._v1_search_privileges_request_model + self.api._v1_search_privileges_request_model = self.api.add_model( + 'V1SearchPrivilegesRequestModel', + description='Search privileges request model following OpenSearch DSL', + schema=self._common_search_request_schema, + ) + return self.api._v1_search_privileges_request_model + + @property + def _search_response_total_schema(self) -> JsonSchema: + """Return the common total hits schema used by search response models""" + return JsonSchema( + type=JsonSchemaType.OBJECT, + description='Total hits information from OpenSearch', + properties={ + 'value': JsonSchema(type=JsonSchemaType.INTEGER), + 'relation': JsonSchema(type=JsonSchemaType.STRING, enum=['eq', 'gte']), + }, + ) + @property def search_providers_response_model(self) -> Model: """Return the search providers response model, which should only be created once per API""" @@ -82,14 +115,7 @@ def search_providers_response_model(self) -> Model: type=JsonSchemaType.ARRAY, items=self._providers_response_schema, ), - 'total': JsonSchema( - type=JsonSchemaType.OBJECT, - description='Total hits information from OpenSearch', - properties={ - 'value': JsonSchema(type=JsonSchemaType.INTEGER), - 'relation': JsonSchema(type=JsonSchemaType.STRING, enum=['eq', 'gte']), - }, - ), + 'total': self._search_response_total_schema, 'lastSort': JsonSchema( type=JsonSchemaType.ARRAY, description='Sort values from the last hit to use with search_after for the next page', @@ -99,6 +125,88 @@ def search_providers_response_model(self) -> Model: ) return self.api._v1_search_providers_response_model + @property + def search_privileges_response_model(self) -> Model: + """Return the search privileges response model, which should only be created once per API""" + if hasattr(self.api, '_v1_search_privileges_response_model'): + return self.api._v1_search_privileges_response_model + self.api._v1_search_privileges_response_model = self.api.add_model( + 'V1SearchPrivilegesResponseModel', + description='Search privileges response model', + schema=JsonSchema( + type=JsonSchemaType.OBJECT, + required=['privileges', 'total'], + properties={ + 'privileges': JsonSchema( + type=JsonSchemaType.ARRAY, + items=self._flattened_privilege_response_schema, + ), + 'total': self._search_response_total_schema, + 'lastSort': JsonSchema( + type=JsonSchemaType.ARRAY, + description='Sort values from the last hit to use with search_after for the next page', + ), + }, + ), + ) + return self.api._v1_search_privileges_response_model + + @property + def _flattened_privilege_response_schema(self): + """ + Schema for flattened privilege response - combines privilege and license data. + This mirrors StatePrivilegeGeneralResponseSchema for the search API. + """ + stack: AppStack = AppStack.of(self.api) + + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'type', + 'providerId', + 'compact', + 'jurisdiction', + 'licenseType', + 'privilegeId', + 'status', + 'compactEligibility', + 'dateOfExpiration', + 'dateOfIssuance', + 'dateOfRenewal', + 'dateOfUpdate', + 'familyName', + 'givenName', + 'licenseJurisdiction', + 'licenseStatus', + ], + properties={ + 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['statePrivilege']), + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'jurisdiction': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions')), + 'licenseType': JsonSchema(type=JsonSchemaType.STRING), + 'privilegeId': JsonSchema(type=JsonSchemaType.STRING), + 'status': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + 'compactEligibility': JsonSchema(type=JsonSchemaType.STRING, enum=['eligible', 'ineligible']), + 'dateOfExpiration': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfIssuance': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfRenewal': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), + 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), + 'familyName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'givenName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'licenseJurisdiction': JsonSchema( + type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions') + ), + 'licenseStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), + # Optional fields + 'middleName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'suffix': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'licenseStatusName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'licenseNumber': JsonSchema(type=JsonSchemaType.STRING, max_length=100), + 'npi': JsonSchema(type=JsonSchemaType.STRING, pattern='^[0-9]{10}$'), + }, + ) + @property def _providers_response_schema(self): stack: AppStack = AppStack.of(self.api) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py new file mode 100644 index 000000000..4fc3701ef --- /dev/null +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from aws_cdk import Duration +from aws_cdk.aws_apigateway import LambdaIntegration, MethodOptions, MethodResponse, Resource + +from common_constructs.cc_api import CCApi +from stacks import search_persistent_stack + +from .api_model import ApiModel + + +class PrivilegeSearch: + """ + Endpoint for searching privileges in the OpenSearch domain. + """ + + def __init__( + self, + *, + resource: Resource, + method_options: MethodOptions, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + api_model: ApiModel, + ): + super().__init__() + + self.resource = resource + self.api: CCApi = resource.api + self.api_model = api_model + + self._add_search_privileges( + method_options=method_options, + search_persistent_stack=search_persistent_stack, + ) + + def _add_search_privileges( + self, + method_options: MethodOptions, + search_persistent_stack: search_persistent_stack.SearchPersistentStack, + ): + search_resource = self.resource.add_resource('search') + + # Get the search handler from the search persistent stack (same handler as provider search) + handler = search_persistent_stack.search_handler.handler + + search_resource.add_method( + 'POST', + request_validator=self.api.parameter_body_validator, + request_models={'application/json': self.api_model.search_privileges_request_model}, + method_responses=[ + MethodResponse( + status_code='200', + response_models={'application/json': self.api_model.search_privileges_response_model}, + ), + ], + integration=LambdaIntegration(handler, timeout=Duration.seconds(29)), + request_parameters={'method.request.header.Authorization': True}, + authorization_type=method_options.authorization_type, + authorizer=method_options.authorizer, + authorization_scopes=method_options.authorization_scopes, + ) + diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py index 29f516c1c..7e08dcb43 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py @@ -44,7 +44,7 @@ def _add_search_providers( search_resource = self.resource.add_resource('search') # Get the search providers handler from the search persistent stack - handler = search_persistent_stack.search_providers_handler.handler + handler = search_persistent_stack.search_handler.handler search_resource.add_method( 'POST', diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 9f39ad030..5d18fa555 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -10,7 +10,7 @@ from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain -from stacks.search_persistent_stack.search_providers_handler import SearchProvidersHandler +from stacks.search_persistent_stack.search_handler import SearchHandler from stacks.vpc_stack import VpcStack @@ -121,9 +121,9 @@ def __init__( ) # Create the search providers handler for API Gateway integration - self.search_providers_handler = SearchProvidersHandler( + self.search_handler = SearchHandler( self, - construct_id='searchProvidersHandler', + construct_id='searchHandler', opensearch_domain=self.provider_search_domain.domain, vpc_stack=vpc_stack, vpc_subnets=self.provider_search_domain.vpc_subnets, diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py similarity index 85% rename from backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py rename to backend/compact-connect/stacks/search_persistent_stack/search_handler.py index d38a1af9e..b51b340be 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/search_providers_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py @@ -14,12 +14,12 @@ from stacks.vpc_stack import VpcStack -class SearchProvidersHandler(Construct): +class SearchHandler(Construct): """ - Construct for the Search Providers Lambda function. + Construct for the Search Lambda function. This construct creates the Lambda function that handles search requests - against the OpenSearch domain for provider records. + against the OpenSearch domain for both provider and privilege records. """ def __init__( @@ -33,7 +33,7 @@ def __init__( alarm_topic: ITopic, ): """ - Initialize the SearchProvidersHandler construct. + Initialize the SearchHandler construct. :param scope: The scope of the construct :param construct_id: The id of the construct @@ -46,14 +46,14 @@ def __init__( super().__init__(scope, construct_id) stack = Stack.of(scope) - # Create Lambda function for searching providers + # Create Lambda function for searching providers and privileges self.handler = PythonFunction( self, 'SearchProvidersFunction', - description='Search providers handler for OpenSearch queries', - index=os.path.join('handlers', 'search_providers.py'), + description='Search handler for OpenSearch queries', + index=os.path.join('handlers', 'search.py'), lambda_dir='search', - handler='search_providers', + handler='search_api_handler', role=lambda_role, log_retention=RetentionDays.ONE_MONTH, environment={ From 31f605dec078b2f26411ff68058b30189452b0d9 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 14:02:30 -0600 Subject: [PATCH 052/137] logging/formatting --- .../lambdas/python/provider-data-v1/handlers/licenses.py | 1 + .../stacks/search_api_stack/v1_api/privilege_search.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py index 18ef1af65..59d8bd64f 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py +++ b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py @@ -75,6 +75,7 @@ def post_licenses(event: dict, context: LambdaContext): # noqa: ARG001 unused-a # verify that none of the SSN+LicenseType combinations are repeats within the same batch license_keys = [(license_record['ssn'], license_record['licenseType']) for license_record in licenses] if len(set(license_keys)) < len(license_keys): + logger.info('Duplicate SSNs detected in same request.', compact=compact, jurisdiction=jurisdiction) raise CCInvalidRequestCustomResponseException( response_body={ 'message': 'Invalid license records in request. See errors for more detail.', diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py index 4fc3701ef..27f8f6e77 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py @@ -59,4 +59,3 @@ def _add_search_privileges( authorizer=method_options.authorizer, authorization_scopes=method_options.authorization_scopes, ) - From 5737ccfd3dea8bf7e1052ccf88a6c65e0d365b89 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 14:09:35 -0600 Subject: [PATCH 053/137] Update purchase requirements --- .../python/purchases/requirements-dev.in | 1 + .../python/purchases/requirements-dev.txt | 58 ++++++++++--------- .../lambdas/python/purchases/requirements.in | 1 + .../lambdas/python/purchases/requirements.txt | 2 +- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/backend/compact-connect/lambdas/python/purchases/requirements-dev.in b/backend/compact-connect/lambdas/python/purchases/requirements-dev.in index 11892d0b0..c4c8851a6 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements-dev.in +++ b/backend/compact-connect/lambdas/python/purchases/requirements-dev.in @@ -14,3 +14,4 @@ boto3>=1.34.33, <2 cryptography>=46, <47 marshmallow>=3.21.3, <4.0.0 requests>=2.31.0, <3.0.0 +urllib3>=2.6.0, <3 diff --git a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt index e83ce4bd3..f4ac929f7 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt @@ -2,21 +2,21 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements-dev.in +# pip-compile --no-emit-index-url --no-strip-extras lambdas/python/purchases/requirements-dev.in # argon2-cffi==25.1.0 - # via -r requirements-dev.in + # via -r lambdas/python/purchases/requirements-dev.in argon2-cffi-bindings==25.1.0 # via argon2-cffi aws-lambda-powertools==3.23.0 - # via -r requirements-dev.in + # via -r lambdas/python/purchases/requirements-dev.in boolean-py==5.0 # via license-expression -boto3==1.40.76 +boto3==1.42.3 # via - # -r requirements-dev.in + # -r lambdas/python/purchases/requirements-dev.in # moto -botocore==1.40.76 +botocore==1.42.3 # via # boto3 # moto @@ -39,20 +39,20 @@ click==8.3.1 # via pip-tools coverage[toml]==7.12.0 # via - # -r requirements-dev.in + # -r lambdas/python/purchases/requirements-dev.in # pytest-cov cryptography==46.0.3 # via - # -r requirements-dev.in + # -r lambdas/python/purchases/requirements-dev.in # moto -cyclonedx-python-lib==9.1.0 +cyclonedx-python-lib==11.6.0 # via pip-audit defusedxml==0.7.1 # via py-serializable docker==7.1.0 # via moto faker==37.12.0 - # via -r requirements-dev.in + # via -r lambdas/python/purchases/requirements-dev.in filelock==3.20.0 # via cachecontrol idna==3.11 @@ -75,14 +75,14 @@ markupsafe==3.0.3 # jinja2 # werkzeug marshmallow==3.26.1 - # via -r requirements-dev.in + # via -r lambdas/python/purchases/requirements-dev.in mdurl==0.1.2 # via markdown-it-py -moto[dynamodb,s3]==5.1.17 - # via -r requirements-dev.in +moto[dynamodb,s3]==5.1.18 + # via -r lambdas/python/purchases/requirements-dev.in msgpack==1.1.2 # via cachecontrol -packageurl-python==0.17.5 +packageurl-python==0.17.6 # via cyclonedx-python-lib packaging==25.0 # via @@ -93,13 +93,13 @@ packaging==25.0 # pytest pip-api==0.0.34 # via pip-audit -pip-audit==2.9.0 - # via -r requirements-dev.in +pip-audit==2.10.0 + # via -r lambdas/python/purchases/requirements-dev.in pip-requirements-parser==32.0.1 # via pip-audit pip-tools==7.5.2 - # via -r requirements-dev.in -platformdirs==4.5.0 + # via -r lambdas/python/purchases/requirements-dev.in +platformdirs==4.5.1 # via pip-audit pluggy==1.6.0 # via @@ -123,10 +123,10 @@ pyproject-hooks==1.2.0 # pip-tools pytest==9.0.1 # via - # -r requirements-dev.in + # -r lambdas/python/purchases/requirements-dev.in # pytest-cov pytest-cov==7.0.0 - # via -r requirements-dev.in + # via -r lambdas/python/purchases/requirements-dev.in python-dateutil==2.9.0.post0 # via # botocore @@ -137,7 +137,7 @@ pyyaml==6.0.3 # responses requests==2.32.5 # via - # -r requirements-dev.in + # -r lambdas/python/purchases/requirements-dev.in # cachecontrol # docker # moto @@ -147,21 +147,25 @@ responses==0.25.8 # via moto rich==14.2.0 # via pip-audit -ruff==0.14.5 - # via -r requirements-dev.in -s3transfer==0.14.0 +ruff==0.14.8 + # via -r lambdas/python/purchases/requirements-dev.in +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil sortedcontainers==2.4.0 # via cyclonedx-python-lib -toml==0.10.2 +tomli==2.3.0 + # via pip-audit +tomli-w==1.2.0 # via pip-audit typing-extensions==4.15.0 - # via aws-lambda-powertools + # via + # aws-lambda-powertools + # cyclonedx-python-lib tzdata==2025.2 # via faker -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/purchases/requirements.in b/backend/compact-connect/lambdas/python/purchases/requirements.in index e5e0becf4..d32f4e91e 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements.in +++ b/backend/compact-connect/lambdas/python/purchases/requirements.in @@ -1,2 +1,3 @@ # common requirements are managed in the common-python requirements.in file authorizenet>=1.1.6, <2 +urllib3>=2.6.0, <3 diff --git a/backend/compact-connect/lambdas/python/purchases/requirements.txt b/backend/compact-connect/lambdas/python/purchases/requirements.txt index 6141bed67..786959633 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements.txt +++ b/backend/compact-connect/lambdas/python/purchases/requirements.txt @@ -18,5 +18,5 @@ pyxb-x==1.2.6.3 # via authorizenet requests==2.32.5 # via authorizenet -urllib3==2.5.0 +urllib3==2.6.0 # via requests From a23b092d19b5589164dbeaf24c5059f18b5b4376 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 14:14:01 -0600 Subject: [PATCH 054/137] Update requirements to latest --- .../python/cognito-backup/requirements-dev.txt | 6 +++--- .../lambdas/python/cognito-backup/requirements.txt | 6 +++--- .../lambdas/python/common/requirements-dev.txt | 14 +++++++------- .../lambdas/python/common/requirements.txt | 6 +++--- .../compact-configuration/requirements-dev.txt | 6 +++--- .../python/custom-resources/requirements-dev.txt | 6 +++--- .../python/data-events/requirements-dev.txt | 6 +++--- .../python/disaster-recovery/requirements-dev.txt | 6 +++--- .../python/provider-data-v1/requirements-dev.txt | 6 +++--- .../lambdas/python/search/requirements-dev.txt | 6 +++--- .../lambdas/python/search/requirements.txt | 2 +- .../staff-user-pre-token/requirements-dev.txt | 6 +++--- .../python/staff-users/requirements-dev.txt | 6 +++--- backend/compact-connect/requirements-dev.txt | 6 +++--- backend/compact-connect/requirements.txt | 4 ++-- 15 files changed, 46 insertions(+), 46 deletions(-) diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt index b289dda64..190ce43e5 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt @@ -6,11 +6,11 @@ # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements-dev.in -boto3==1.42.1 +boto3==1.42.3 # via # -r lambdas/python/cognito-backup/requirements-dev.in # moto -botocore==1.42.1 +botocore==1.42.3 # via # -r lambdas/python/cognito-backup/requirements-dev.in # boto3 @@ -77,7 +77,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # requests diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt index 077b34697..4ceae42b2 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt @@ -6,9 +6,9 @@ # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements.in -boto3==1.42.1 +boto3==1.42.3 # via -r lambdas/python/cognito-backup/requirements.in -botocore==1.42.1 +botocore==1.42.3 # via # -r lambdas/python/cognito-backup/requirements.in # boto3 @@ -26,5 +26,5 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.5.0 +urllib3==2.6.0 # via botocore diff --git a/backend/compact-connect/lambdas/python/common/requirements-dev.txt b/backend/compact-connect/lambdas/python/common/requirements-dev.txt index b6c13ba34..ce833c1ba 100644 --- a/backend/compact-connect/lambdas/python/common/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/common/requirements-dev.txt @@ -16,21 +16,21 @@ aws-sam-translator==1.105.0 # via cfn-lint aws-xray-sdk==2.15.0 # via moto -boto3==1.42.1 +boto3==1.42.3 # via # aws-sam-translator # moto -boto3-stubs[full]==1.41.5 +boto3-stubs[full]==1.42.3 # via -r lambdas/python/common/requirements-dev.in -boto3-stubs-full==1.41.5 +boto3-stubs-full==1.42.3 # via boto3-stubs -botocore==1.42.1 +botocore==1.42.3 # via # aws-xray-sdk # boto3 # moto # s3transfer -botocore-stubs==1.42.1 +botocore-stubs==1.42.3 # via boto3-stubs certifi==2025.11.12 # via requests @@ -150,7 +150,7 @@ six==1.17.0 # rfc3339-validator sympy==1.14.0 # via cfn-lint -types-awscrt==0.29.1 +types-awscrt==0.29.2 # via botocore-stubs types-s3transfer==0.15.0 # via boto3-stubs @@ -166,7 +166,7 @@ typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via faker -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/common/requirements.txt b/backend/compact-connect/lambdas/python/common/requirements.txt index 278d8c7be..96f03cba0 100644 --- a/backend/compact-connect/lambdas/python/common/requirements.txt +++ b/backend/compact-connect/lambdas/python/common/requirements.txt @@ -10,9 +10,9 @@ argon2-cffi-bindings==25.1.0 # via argon2-cffi aws-lambda-powertools==3.23.0 # via -r lambdas/python/common/requirements.in -boto3==1.42.1 +boto3==1.42.3 # via -r lambdas/python/common/requirements.in -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # s3transfer @@ -49,7 +49,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # requests diff --git a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt index 82f46dbab..6a33a909b 100644 --- a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/compact-configuration/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt index a6886ea70..603aca238 100644 --- a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/custom-resources/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt index 29a9d9928..ad67df8db 100644 --- a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/data-events/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt index 950742a88..92af676eb 100644 --- a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/disaster-recovery/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt index 287cb6c44..04088c6b2 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/provider-data-v1/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -62,7 +62,7 @@ six==1.17.0 # via python-dateutil tzdata==2025.2 # via faker -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.txt b/backend/compact-connect/lambdas/python/search/requirements-dev.txt index 5dd4157cd..a8162aa4e 100644 --- a/backend/compact-connect/lambdas/python/search/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/search/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -56,7 +56,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/search/requirements.txt b/backend/compact-connect/lambdas/python/search/requirements.txt index ed72ac2ec..96ec7e73b 100644 --- a/backend/compact-connect/lambdas/python/search/requirements.txt +++ b/backend/compact-connect/lambdas/python/search/requirements.txt @@ -30,7 +30,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via grpcio -urllib3==2.5.0 +urllib3==2.6.0 # via # opensearch-py # requests diff --git a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt index 0d413d5ef..9155ddfe0 100644 --- a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-user-pre-token/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt index a5c613557..88708c0c6 100644 --- a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-users/requirements-dev.in # -boto3==1.42.1 +boto3==1.42.3 # via moto -botocore==1.42.1 +botocore==1.42.3 # via # boto3 # moto @@ -66,7 +66,7 @@ six==1.17.0 # via python-dateutil tzdata==2025.2 # via faker -urllib3==2.5.0 +urllib3==2.6.0 # via # botocore # docker diff --git a/backend/compact-connect/requirements-dev.txt b/backend/compact-connect/requirements-dev.txt index 007dddc8f..3a57901a0 100644 --- a/backend/compact-connect/requirements-dev.txt +++ b/backend/compact-connect/requirements-dev.txt @@ -58,7 +58,7 @@ pip-requirements-parser==32.0.1 # via pip-audit pip-tools==7.5.2 # via -r requirements-dev.in -platformdirs==4.5.0 +platformdirs==4.5.1 # via pip-audit pluggy==1.6.0 # via @@ -88,7 +88,7 @@ requests==2.32.5 # pip-audit rich==14.2.0 # via pip-audit -ruff==0.14.7 +ruff==0.14.8 # via -r requirements-dev.in sortedcontainers==2.4.0 # via cyclonedx-python-lib @@ -100,7 +100,7 @@ typing-extensions==4.15.0 # via cyclonedx-python-lib tzdata==2025.2 # via faker -urllib3==2.5.0 +urllib3==2.6.0 # via requests wheel==0.45.1 # via pip-tools diff --git a/backend/compact-connect/requirements.txt b/backend/compact-connect/requirements.txt index 39829962e..ca0101983 100644 --- a/backend/compact-connect/requirements.txt +++ b/backend/compact-connect/requirements.txt @@ -12,11 +12,11 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-aws-lambda-python-alpha==2.231.0a0 +aws-cdk-aws-lambda-python-alpha==2.232.0a0 # via -r requirements.in aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.231.0 +aws-cdk-lib==2.232.0 # via # -r requirements.in # aws-cdk-aws-lambda-python-alpha From 509438aaf8ff3cb60177d214a97682bc970ef6f6 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 15:49:58 -0600 Subject: [PATCH 055/137] Add notes about blue/green deployments --- .../stacks/search_persistent_stack/__init__.py | 8 ++++++-- .../search_persistent_stack/provider_search_domain.py | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 5d18fa555..290285036 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -28,8 +28,12 @@ class SearchPersistentStack(AppStack): - Non-prod (sandbox/test/beta): t3.small.search, 1 node - Prod: m7g.medium.search, 3 master + 3 data nodes (with standby) - Note: Prod deployment is currently conditional and will not deploy until the full - advanced search API is implemented. + IMPORTANT NOTE: Updating the OpenSearch domain may require a blue/green deployment, which is known to get stuck + on occasion requiring AWS support intervention (every time we attempted to update the engine version during + development, the deployment never completed). If you intend to update any field that will require a blue/green + deployment as described here: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html + Note that worse case scenario, you may have to delete the entire stack, re-deploy it, and re-index all the data from + the provider table. In light of this, DO NOT place any resources in this stack that should never be deleted. """ def __init__( diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index eacd4eefe..9856671d3 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -154,6 +154,15 @@ def __init__( self.domain = Domain( self, 'Domain', + # IMPORTANT NOTE: updating the engine version requires a blue/green deployment, which is known to get stuck + # on occasion requiring AWS support intervention. If you intend to update this field, or any other field + # that will require a blue/green deployment as described here: + # https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html + # You should consider working with stakeholders to schedule a maintenance window during low-traffic periods + # where advanced search may become inaccessible during the update. During development, we found that if a + # blue/green deployment became stuck, the search endpoints were still able to serve data, but the + # CloudFormation deployment would fail waiting for the domain to become active. Worst case scenario, + # both the search API and search persistent stacks needed to be destroyed, redeployed, and re-indexed. version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, # VPC configuration for network isolation From 9d3f59b35d50fdc373bb95e5a369f8504e445eef Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 16:02:35 -0600 Subject: [PATCH 056/137] PR feedback --- .../lambdas/python/purchases/requirements.in | 1 + .../lambdas/python/search/handlers/search.py | 2 +- .../provider_search_domain.py | 81 +++++++++---------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/backend/compact-connect/lambdas/python/purchases/requirements.in b/backend/compact-connect/lambdas/python/purchases/requirements.in index d32f4e91e..3ddeb3993 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements.in +++ b/backend/compact-connect/lambdas/python/purchases/requirements.in @@ -1,3 +1,4 @@ # common requirements are managed in the common-python requirements.in file authorizenet>=1.1.6, <2 +# explicitly setting this transitive dependency to pick vulnerability patch urllib3>=2.6.0, <3 diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index fa0a70266..0c2223bfd 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -196,7 +196,7 @@ def _parse_and_validate_request_body(event: dict) -> dict: """ try: schema = SearchProvidersRequestSchema() - return schema.loads(event['body']) + return schema.loads(event.get('body', '{}')) except ValidationError as e: logger.warning('Invalid request body', errors=e.messages) raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index 9856671d3..903daa1ce 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -521,7 +521,7 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): ), evaluation_periods=1, threshold=1, - comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, alarm_description=( f'OpenSearch Domain {self.domain.domain_name} automated snapshot has failed. ' @@ -587,47 +587,46 @@ def _add_access_policy_lambda_suppressions(self): """ stack = Stack.of(self) - # Find auto-generated Lambda constructs by looking for children with IDs starting with 'AWS' - # These are created by CDK's AwsCustomResource for managing domain access policies - for child in stack.node.children: - if child.node.id.startswith('AWS'): - NagSuppressions.add_resource_suppressions( - child, - suppressions=[ - { - 'id': 'AwsSolutions-L1', - 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' - 'OpenSearch domain access policies. We cannot specify the runtime version.', - }, - { - 'id': 'AwsSolutions-IAM4', - 'appliesTo': [ - 'Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' - ], - 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' - 'OpenSearch domain access policies. It uses the standard execution role.', - }, - { - 'id': 'AwsSolutions-IAM5', - 'appliesTo': ['Action::kms:Describe*', 'Action::kms:List*'], - 'reason': 'This is an AWS-managed custom resource Lambda that requires KMS permissions to ' - 'access the encryption key used by the OpenSearch domain.', - }, - { - 'id': 'HIPAA.Security-LambdaDLQ', - 'reason': 'This is an AWS-managed custom resource Lambda used only during deployment to ' - 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' - 'functions.', - }, - { - 'id': 'HIPAA.Security-LambdaInsideVPC', - 'reason': 'This is an AWS-managed custom resource Lambda that needs internet access to ' - 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' - 'required.', - }, + # Suppress for the auto-generated Lambda function + # The construct ID is auto-generated by CDK, so we need to suppress at the stack level + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{stack.node.path}/AWS679f53fac002430cb0da5b7982bd2287', + suppressions=[ + { + 'id': 'AwsSolutions-L1', + 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' + 'OpenSearch domain access policies. We cannot specify the runtime version.', + }, + { + 'id': 'AwsSolutions-IAM4', + 'appliesTo': [ + 'Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' ], - apply_to_children=True, - ) + 'reason': 'This is an AWS-managed custom resource Lambda created by CDK to manage ' + 'OpenSearch domain access policies. It uses the standard execution role.', + }, + { + 'id': 'AwsSolutions-IAM5', + 'appliesTo': ['Action::kms:Describe*', 'Action::kms:List*'], + 'reason': 'This is an AWS-managed custom resource Lambda that requires KMS permissions to ' + 'access the encryption key used by the OpenSearch domain.', + }, + { + 'id': 'HIPAA.Security-LambdaDLQ', + 'reason': 'This is an AWS-managed custom resource Lambda used only during deployment to ' + 'manage OpenSearch access policies. A DLQ is not necessary for deployment-time ' + 'functions.', + }, + { + 'id': 'HIPAA.Security-LambdaInsideVPC', + 'reason': 'This is an AWS-managed custom resource Lambda that needs internet access to ' + 'manage OpenSearch domain access policies via AWS APIs. VPC placement is not ' + 'required.', + }, + ], + apply_to_children=True, + ) def _add_lambda_role_suppressions(self, lambda_role: IRole): """ From 8e572aaedcd128548bd710acbdf090b25c71476f Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 17:38:21 -0600 Subject: [PATCH 057/137] Add documentation for search functionality --- backend/compact-connect/docs/design/README.md | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/backend/compact-connect/docs/design/README.md b/backend/compact-connect/docs/design/README.md index 84d89add1..8189d8da3 100644 --- a/backend/compact-connect/docs/design/README.md +++ b/backend/compact-connect/docs/design/README.md @@ -10,6 +10,8 @@ Look here for continued documentation of the back-end design, as it progresses. - **[Privileges](#privileges)** - **[Attestations](#attestations)** - **[Transaction History Reporting](#transaction-history-reporting)** +- **[Advanced Data Search](#advanced-data-search)** +- **[CI/CD Pipelines](#cicd-pipelines)** - **[Audit Logging](#audit-logging)** ## Compacts and Jurisdictions @@ -675,6 +677,224 @@ For this reason, we use the batch settlement time as the timestamp for the trans transaction history table. This ensures that any transactions that are in a batch which fails to settle will eventually be processed and stored in the transaction history table. + +## Advanced Data Search +[Back to top](#backend-design) + +To support advanced search capability of providers and privilege records, this project leverages +[AWS OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html). +Provider data from the provider DynamoDB table is indexed within an OpenSearch Domain (Cluster), which is then +queryable by staff users through the Search API (search.compactconnect.org). The OpenSearch resources are deployed within a Virtual Private Cloud (VPC) to provide a layer of network security. + +### Architecture Overview + +The search infrastructure consists of several key components: + +1. **OpenSearch Domain**: A managed OpenSearch cluster deployed within a VPC +2. **Search API**: API Gateway endpoints backed by Lambda functions for querying the domain +3. **Index Manager**: A CloudFormation custom resource that creates and manages indices +4. **Populate Handler**: A Lambda function for bulk indexing all provider data from DynamoDB + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ API Gateway │────▶│ Search Lambda │────▶│ OpenSearch │ +│ (Search API) │ │ (in VPC) │ │ Domain (VPC) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + ▲ + │ +┌─────────────────┐ ┌─────────────────┐ │ +│ DynamoDB │────▶│ Populate Lambda │──────────────┘ +│ (Provider Table)│ │ (in VPC) │ +└─────────────────┘ └─────────────────┘ +``` + +### OpenSearch Domain Configuration + +The OpenSearch domain is configured differently based on environment: + +| Environment | Instance Type | Data Nodes | Master Nodes | Replicas | EBS Size | +|-------------|---------------|------------|--------------|----------|----------| +| Non-prod (sandbox/test/beta) | t3.small.search | 1 | None | 0 | 10 GB | +| Production | m7g.medium.search | 3 | 3 (with standby) | 1 | 25 GB | + +### Index Structure + +Provider documents are stored in compact-specific indices with the naming convention: `compact_{compact}_providers` +(e.g., `compact_aslp_providers`). + +#### Index Mapping + +Each provider document contains all information you would see from the provider detail api endpoint with `readGeneral` permission: + +**Top-Level Provider Fields:** +- `providerId` (keyword): Unique provider identifier +- `givenName`, `middleName`, `familyName` (text with ASCII folding): Provider name fields +- `licenseJurisdiction` (keyword): Home license jurisdiction +- `licenseStatus` (keyword): Current license status (active/inactive) +- `compactEligibility` (keyword): Compact eligibility status +- `dateOfExpiration` (date): License expiration date +- `npi` (keyword): National Provider Identifier +- `privilegeJurisdictions` (keyword array): Jurisdictions where provider has privileges + +**Nested Objects:** +- `licenses`: Array of license records with full license details +- `privileges`: Array of privilege records with privilege details +- `militaryAffiliations`: Array of military affiliation records + +The index uses a custom ASCII-folding analyzer for name fields, which allows searching for names with international +characters using their ASCII equivalents (e.g., searching "Jose" matches "José"). + +### Search API Endpoints + +The Search API provides two endpoints for querying the OpenSearch domain: + +#### Provider Search +``` +POST /v1/compacts/{compact}/providers/search +``` + +Returns provider records matching the query. Response includes the full provider document with licenses, privileges, +and military affiliations. + +#### Privilege Search +``` +POST /v1/compacts/{compact}/privileges/search +``` + +Returns flattened privilege records. This endpoint queries the same provider index but extracts and flattens +privileges, combining privilege data with license data to provide a denormalized view suitable for privilege-focused +reports and exports. + +### Request/Response Format + +Both endpoints accept OpenSearch DSL query bodies with pagination support: + +**Request Body:** +```json +{ + "query": { + "bool": { + "must": [ + { "match": { "givenName": "John" }}, + { "term": { "licenseStatus": "active" }} + ] + } + }, + "size": 100, + "sort": [{ "providerId": "asc" }], + "search_after": ["previous-provider-id"] +} +``` + +**Provider Search Response:** +```json +{ + "providers": [ + { + "providerId": "...", + "givenName": "John", + "familyName": "Doe", + "licenses": [...], + "privileges": [...] + } + ], + "total": { "value": 150, "relation": "eq" }, + "lastSort": ["current-provider-id"] +} +``` + +**Privilege Search Response:** +```json +{ + "privileges": [ + { + "type": "statePrivilege", + "providerId": "...", + "jurisdiction": "ky", + "licenseJurisdiction": "oh", + "givenName": "John", + "familyName": "Doe", + "privilegeId": "PRIV-001", + "status": "active" + } + ], + "total": { "value": 50, "relation": "eq" }, + "lastSort": ["current-provider-id"] +} +``` + +### Pagination + +The API supports two pagination strategies following OpenSearch DSL conventions: + +1. **Offset Pagination** (`from`/`size`): Simple but limited to 10,000 results + ```json + { "from": 0, "size": 100 } + ``` + +2. **Cursor-Based Pagination** (`search_after`): Recommended for large result sets + ```json + { + "sort": [{ "providerId": "asc" }], + "search_after": ["last-provider-id-from-previous-page"] + } + ``` + +**Note**: When using `search_after`, a `sort` field is required. The `lastSort` value in the response can be passed +as `search_after` in the next request. + +**Limits:** +- Maximum page size: 100 records per request +- Default page size: 10 records + +### Data Indexing + +#### Initial Population / Re-indexing + +The `populate_provider_documents` Lambda function handles bulk indexing of provider data from DynamoDB into +OpenSearch. This function is invoked manually through the AWS Console for: +- Initial data population when the search infrastructure is first deployed +- Full re-indexing if data becomes out of sync + +The function: +1. Scans the provider table using the `providerDateOfUpdate` GSI +2. Retrieves complete provider records for each provider +3. Sanitizes data using `ProviderGeneralResponseSchema` +4. Bulk indexes documents in batches of 1,000 + +**Resumable Processing**: If the function approaches the 15-minute Lambda timeout, it returns pagination information in the +`resumeFrom` field that can be passed as lambda input to continue processing: + +```json +{ + "startingCompact": "aslp", + "startingLastKey": {"pk": "...", "sk": "..."} +} +``` + +#### Index Management + +The `IndexManagerCustomResource` is a CloudFormation custom resource that creates compact-specific indices when the +stack is deployed. It ensures indices exist with the correct mapping before any indexing operations begin. + +### Monitoring and Alarms + +The search infrastructure includes CloudWatch alarms for capacity monitoring. If these alarms get triggered, review +usage metrics to determine if the Domain needs to be scaled up: + +- **CPU Utilization**: Alerts when CPU exceeds threshold +- **Memory Pressure**: Monitors JVM memory pressure +- **Storage Space**: Alerts on low disk space +- **Cluster Health**: Monitors yellow/red cluster status + +### Important OpenSearch Domain Maintenance Note +**WARNING**: Updating the OpenSearch domain may require a blue/green deployment, which has been known to get stuck +and require AWS support intervention. If you need to update any field that triggers a blue/green deployment (see +[AWS documentation](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html)), +be prepared for worst-case scenario of deleting the entire search stack, re-deploying it, and re-indexing all data +from the provider table. You should work with stakeholders to schedule a maintanence window during low traffic periods +for major updates where advanced search may be temporarily unavailable. + ## CI/CD Pipelines This project leverages AWS CodePipeline to deploy the backend and frontend infrastructure. See the From ded27a0c347037cbc44e49bebb256ef0eda16e17 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 08:41:43 -0600 Subject: [PATCH 058/137] fix snapshot for test --- .../compact-connect/tests/app/test_search_persistent_stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index c34b3f2a2..0639e3f65 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -251,7 +251,7 @@ def test_capacity_alarms_configured(self): 'MetricName': 'AutomatedSnapshotFailure', 'Namespace': 'AWS/ES', 'Threshold': 1, - 'ComparisonOperator': 'GreaterThanThreshold', + 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'EvaluationPeriods': 1, }, ) From 8997a9675157ea212da64e433e0619adad6b8a22 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 12:18:21 -0600 Subject: [PATCH 059/137] update requirements to latest --- .../cognito-backup/requirements-dev.txt | 10 +++++----- .../python/cognito-backup/requirements.txt | 8 ++++---- .../python/common/requirements-dev.txt | 19 +++++++++---------- .../lambdas/python/common/requirements.txt | 8 ++++---- .../requirements-dev.txt | 8 ++++---- .../compact-configuration/requirements.txt | 2 +- .../custom-resources/requirements-dev.txt | 8 ++++---- .../python/custom-resources/requirements.txt | 2 +- .../python/data-events/requirements-dev.txt | 8 ++++---- .../python/data-events/requirements.txt | 2 +- .../disaster-recovery/requirements-dev.txt | 8 ++++---- .../python/disaster-recovery/requirements.txt | 2 +- .../provider-data-v1/requirements-dev.txt | 8 ++++---- .../python/provider-data-v1/requirements.txt | 2 +- .../python/search/requirements-dev.txt | 8 ++++---- .../lambdas/python/search/requirements.txt | 6 +++--- .../staff-user-pre-token/requirements-dev.txt | 8 ++++---- .../staff-user-pre-token/requirements.txt | 2 +- .../python/staff-users/requirements-dev.txt | 8 ++++---- .../python/staff-users/requirements.txt | 2 +- backend/compact-connect/requirements-dev.txt | 10 ++++------ backend/compact-connect/requirements.txt | 6 +++--- 22 files changed, 71 insertions(+), 74 deletions(-) diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt index 190ce43e5..c2f0c39df 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt @@ -1,16 +1,16 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/cognito-backup/requirements-dev.in # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements-dev.in -boto3==1.42.3 +boto3==1.42.4 # via # -r lambdas/python/cognito-backup/requirements-dev.in # moto -botocore==1.42.3 +botocore==1.42.4 # via # -r lambdas/python/cognito-backup/requirements-dev.in # boto3 @@ -55,7 +55,7 @@ pycparser==2.23 # via cffi pygments==2.19.2 # via pytest -pytest==9.0.1 +pytest==9.0.2 # via -r lambdas/python/cognito-backup/requirements-dev.in python-dateutil==2.9.0.post0 # via @@ -77,7 +77,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # requests diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt index 4ceae42b2..15c38b047 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/cognito-backup/requirements.in # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements.in -boto3==1.42.3 +boto3==1.42.4 # via -r lambdas/python/cognito-backup/requirements.in -botocore==1.42.3 +botocore==1.42.4 # via # -r lambdas/python/cognito-backup/requirements.in # boto3 @@ -26,5 +26,5 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.6.0 +urllib3==2.6.1 # via botocore diff --git a/backend/compact-connect/lambdas/python/common/requirements-dev.txt b/backend/compact-connect/lambdas/python/common/requirements-dev.txt index ce833c1ba..f3001523e 100644 --- a/backend/compact-connect/lambdas/python/common/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/common/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/common/requirements-dev.in @@ -16,21 +16,21 @@ aws-sam-translator==1.105.0 # via cfn-lint aws-xray-sdk==2.15.0 # via moto -boto3==1.42.3 +boto3==1.42.4 # via # aws-sam-translator # moto -boto3-stubs[full]==1.42.3 +boto3-stubs[full]==1.42.4 # via -r lambdas/python/common/requirements-dev.in -boto3-stubs-full==1.42.3 +boto3-stubs-full==1.42.4 # via boto3-stubs -botocore==1.42.3 +botocore==1.42.4 # via # aws-xray-sdk # boto3 # moto # s3transfer -botocore-stubs==1.42.3 +botocore-stubs==1.42.4 # via boto3-stubs certifi==2025.11.12 # via requests @@ -91,7 +91,7 @@ mpmath==1.3.0 # via sympy multipart==1.3.0 # via moto -networkx==3.6 +networkx==3.6.1 # via cfn-lint openapi-schema-validator==0.6.3 # via openapi-spec-validator @@ -152,7 +152,7 @@ sympy==1.14.0 # via cfn-lint types-awscrt==0.29.2 # via botocore-stubs -types-s3transfer==0.15.0 +types-s3transfer==0.16.0 # via boto3-stubs typing-extensions==4.15.0 # via @@ -160,13 +160,12 @@ typing-extensions==4.15.0 # cfn-lint # pydantic # pydantic-core - # referencing # typing-inspection typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via faker -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/common/requirements.txt b/backend/compact-connect/lambdas/python/common/requirements.txt index 96f03cba0..886da2a81 100644 --- a/backend/compact-connect/lambdas/python/common/requirements.txt +++ b/backend/compact-connect/lambdas/python/common/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/common/requirements.in @@ -10,9 +10,9 @@ argon2-cffi-bindings==25.1.0 # via argon2-cffi aws-lambda-powertools==3.23.0 # via -r lambdas/python/common/requirements.in -boto3==1.42.3 +boto3==1.42.4 # via -r lambdas/python/common/requirements.in -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # s3transfer @@ -49,7 +49,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # requests diff --git a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt index 6a33a909b..d6a37154b 100644 --- a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/compact-configuration/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt b/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt index 9eafb5559..79d227dbe 100644 --- a/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt +++ b/backend/compact-connect/lambdas/python/compact-configuration/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/compact-configuration/requirements.in diff --git a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt index 603aca238..0a5be4023 100644 --- a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/custom-resources/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/custom-resources/requirements.txt b/backend/compact-connect/lambdas/python/custom-resources/requirements.txt index e1110d66e..2a8810a14 100644 --- a/backend/compact-connect/lambdas/python/custom-resources/requirements.txt +++ b/backend/compact-connect/lambdas/python/custom-resources/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/custom-resources/requirements.in diff --git a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt index ad67df8db..6d028c569 100644 --- a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/data-events/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/data-events/requirements.txt b/backend/compact-connect/lambdas/python/data-events/requirements.txt index 70343d550..7a1fc37aa 100644 --- a/backend/compact-connect/lambdas/python/data-events/requirements.txt +++ b/backend/compact-connect/lambdas/python/data-events/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/data-events/requirements.in diff --git a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt index 92af676eb..716b90bfa 100644 --- a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/disaster-recovery/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt b/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt index 1366cbf3b..9ad49d395 100644 --- a/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt +++ b/backend/compact-connect/lambdas/python/disaster-recovery/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/disaster-recovery/requirements.in diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt index 04088c6b2..c801c6500 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/provider-data-v1/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -62,7 +62,7 @@ six==1.17.0 # via python-dateutil tzdata==2025.2 # via faker -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt b/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt index 8ca619fb1..f9665c63b 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt +++ b/backend/compact-connect/lambdas/python/provider-data-v1/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/provider-data-v1/requirements.in diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.txt b/backend/compact-connect/lambdas/python/search/requirements-dev.txt index a8162aa4e..6a8162bab 100644 --- a/backend/compact-connect/lambdas/python/search/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/search/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -56,7 +56,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/search/requirements.txt b/backend/compact-connect/lambdas/python/search/requirements.txt index 96ec7e73b..6ff061bc8 100644 --- a/backend/compact-connect/lambdas/python/search/requirements.txt +++ b/backend/compact-connect/lambdas/python/search/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements.in @@ -20,7 +20,7 @@ opensearch-protobufs==0.19.0 # via opensearch-py opensearch-py==3.1.0 # via -r lambdas/python/search/requirements.in -protobuf==6.33.1 +protobuf==6.33.2 # via opensearch-protobufs python-dateutil==2.9.0.post0 # via opensearch-py @@ -30,7 +30,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via grpcio -urllib3==2.6.0 +urllib3==2.6.1 # via # opensearch-py # requests diff --git a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt index 9155ddfe0..2401d3526 100644 --- a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-user-pre-token/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt index e4844a1f4..e7cabe5cb 100644 --- a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt +++ b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-user-pre-token/requirements.in diff --git a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt index 88708c0c6..f022d6c90 100644 --- a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-users/requirements-dev.in # -boto3==1.42.3 +boto3==1.42.4 # via moto -botocore==1.42.3 +botocore==1.42.4 # via # boto3 # moto @@ -66,7 +66,7 @@ six==1.17.0 # via python-dateutil tzdata==2025.2 # via faker -urllib3==2.6.0 +urllib3==2.6.1 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/staff-users/requirements.txt b/backend/compact-connect/lambdas/python/staff-users/requirements.txt index 84457061f..d32dda42a 100644 --- a/backend/compact-connect/lambdas/python/staff-users/requirements.txt +++ b/backend/compact-connect/lambdas/python/staff-users/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-users/requirements.in diff --git a/backend/compact-connect/requirements-dev.txt b/backend/compact-connect/requirements-dev.txt index 3a57901a0..060dbd80e 100644 --- a/backend/compact-connect/requirements-dev.txt +++ b/backend/compact-connect/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras requirements-dev.in @@ -18,7 +18,7 @@ charset-normalizer==3.4.4 # via requests click==8.3.1 # via pip-tools -coverage[toml]==7.12.0 +coverage[toml]==7.13.0 # via # -r requirements-dev.in # pytest-cov @@ -76,7 +76,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==9.0.1 +pytest==9.0.2 # via # -r requirements-dev.in # pytest-cov @@ -96,11 +96,9 @@ tomli==2.3.0 # via pip-audit tomli-w==1.2.0 # via pip-audit -typing-extensions==4.15.0 - # via cyclonedx-python-lib tzdata==2025.2 # via faker -urllib3==2.6.0 +urllib3==2.6.1 # via requests wheel==0.45.1 # via pip-tools diff --git a/backend/compact-connect/requirements.txt b/backend/compact-connect/requirements.txt index ca0101983..8eb6df186 100644 --- a/backend/compact-connect/requirements.txt +++ b/backend/compact-connect/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # # pip-compile --no-emit-index-url --no-strip-extras requirements.in @@ -12,11 +12,11 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-aws-lambda-python-alpha==2.232.0a0 +aws-cdk-aws-lambda-python-alpha==2.232.1a0 # via -r requirements.in aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.232.0 +aws-cdk-lib==2.232.1 # via # -r requirements.in # aws-cdk-aws-lambda-python-alpha From 64b51eb1d5fbdbf35a206f62f4dda4b12295700c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 14:33:40 -0600 Subject: [PATCH 060/137] PR feedback - using index aliases - Passing in number of shards/replicas as properties to custom resource - Using alarm topic from persistent stack --- .../handlers/manage_opensearch_indices.py | 85 ++++++++-- .../python/search/opensearch_client.py | 8 + .../test_manage_opensearch_indices.py | 157 ++++++++++++++---- .../tests/unit/test_opensearch_client.py | 35 ++++ .../search_persistent_stack/__init__.py | 17 +- .../search_persistent_stack/index_manager.py | 41 ++++- .../provider_search_domain.py | 2 +- 7 files changed, 282 insertions(+), 63 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index 6f90dc57b..e148cb417 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -2,23 +2,51 @@ from custom_resource_handler import CustomResourceHandler, CustomResourceResponse from opensearch_client import OpenSearchClient +# Initial index version for new deployments +INITIAL_INDEX_VERSION = 'v1' + class OpenSearchIndexManager(CustomResourceHandler): """ Custom resource handler to create OpenSearch indices for compacts. + + Creates versioned indices (e.g., compact_aslp_providers_v1) with aliases + (e.g., compact_aslp_providers) to enable safe blue-green migrations for + future mapping changes. Queries use the alias, allowing the underlying + index to be swapped without application changes. + See https://docs.opensearch.org/latest/im-plugin/index-alias/ """ - def on_create(self, _properties: dict) -> CustomResourceResponse | None: + def on_create(self, properties: dict) -> CustomResourceResponse | None: """ - Create the indices on creation. + Create the versioned indices and aliases on creation. """ logger.info('Connecting to OpenSearch domain') client = OpenSearchClient() + # Get index configuration from custom resource properties + number_of_shards = int(properties['numberOfShards']) + number_of_replicas = int(properties['numberOfReplicas']) + + logger.info( + 'Index configuration', + number_of_shards=number_of_shards, + number_of_replicas=number_of_replicas, + ) + compacts = config.compacts for compact in compacts: - index_name = f'compact_{compact}_providers' - self._create_provider_index(client, index_name) + # Create versioned index name (e.g., compact_aslp_providers_v1) + index_name = f'compact_{compact}_providers_{INITIAL_INDEX_VERSION}' + # Create alias name (e.g., compact_aslp_providers) + alias_name = f'compact_{compact}_providers' + self._create_provider_index_with_alias( + client=client, + index_name=index_name, + alias_name=alias_name, + number_of_shards=number_of_shards, + number_of_replicas=number_of_replicas, + ) def on_update(self, properties: dict) -> CustomResourceResponse | None: """ @@ -30,20 +58,53 @@ def on_delete(self, _properties: dict) -> CustomResourceResponse | None: No-op on delete. """ - def _create_provider_index(self, client: OpenSearchClient, index_name: str) -> None: + def _create_provider_index_with_alias( + self, + client: OpenSearchClient, + index_name: str, + alias_name: str, + number_of_shards: int, + number_of_replicas: int, + ) -> None: """ - Create the provider index in OpenSearch if it doesn't exist. + Create the provider index and alias in OpenSearch if they don't exist. + + :param client: The OpenSearch client + :param index_name: The versioned index name (e.g., compact_aslp_providers_v1) + :param alias_name: The alias name (e.g., compact_aslp_providers) + :param number_of_shards: Number of primary shards for the index + :param number_of_replicas: Number of replica shards for the index """ + # Check if the alias already exists (meaning an index version is already set up) + if client.alias_exists(alias_name): + logger.info(f"Alias '{alias_name}' already exists. Skipping index and alias creation.") + return + + # Check if the index already exists (edge case: index exists but alias doesn't) if client.index_exists(index_name): - logger.info(f"Index '{index_name}' already exists. Skipping creation.") + logger.info(f"Index '{index_name}' already exists. Creating alias only.") + client.create_alias(index_name, alias_name) + logger.info(f"Alias '{alias_name}' -> '{index_name}' created successfully.") return + + # Create the index with the specified configuration logger.info(f"Creating index '{index_name}'...") - client.create_index(index_name, self._get_provider_index_mapping()) + index_mapping = self._get_provider_index_mapping(number_of_shards, number_of_replicas) + client.create_index(index_name, index_mapping) logger.info(f"Index '{index_name}' created successfully.") - def _get_provider_index_mapping(self) -> dict: + # Create the alias pointing to the new index + logger.info(f"Creating alias '{alias_name}' -> '{index_name}'...") + client.create_alias(index_name, alias_name) + logger.info(f"Alias '{alias_name}' -> '{index_name}' created successfully.") + + def _get_provider_index_mapping(self, number_of_shards: int, number_of_replicas: int) -> dict: """ Define the index mapping for provider documents. + + :param number_of_shards: Number of primary shards for the index + :param number_of_replicas: Number of replica shards for the index + :return: The index mapping dictionary """ # Nested schema for AttestationVersion attestation_version_properties = { @@ -171,10 +232,8 @@ def _get_provider_index_mapping(self) -> dict: return { 'settings': { 'index': { - 'number_of_shards': 1, - # no replicas for non-prod envs (since there is only one data node) - # one replica for prod - 'number_of_replicas': 0 if config.environment_name != 'prod' else 1, + 'number_of_shards': number_of_shards, + 'number_of_replicas': number_of_replicas, }, 'analysis': { # this custom analyzer is recommended by Opensearch when you have international character diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 13565abfc..dd0573e9c 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -31,6 +31,14 @@ def create_index(self, index_name: str, index_mapping: dict) -> None: def index_exists(self, index_name: str) -> bool: return self._client.indices.exists(index=index_name) + def alias_exists(self, alias_name: str) -> bool: + """Check if an alias exists.""" + return self._client.indices.exists_alias(name=alias_name) + + def create_alias(self, index_name: str, alias_name: str) -> None: + """Create an alias pointing to the specified index.""" + self._client.indices.put_alias(index=index_name, name=alias_name) + def search(self, index_name: str, body: dict) -> dict: """ Execute a search query against the specified index. diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index dd1f5bdbc..19e0a95c4 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -14,18 +14,29 @@ def setUp(self): def _create_event(self, request_type: str, properties: dict = None) -> dict: """Create a CloudFormation custom resource event.""" + default_properties = { + 'numberOfShards': 1, + 'numberOfReplicas': 0, + } + if properties: + default_properties.update(properties) return { 'RequestType': request_type, - 'ResourceProperties': properties or {}, + 'ResourceProperties': default_properties, } def _when_testing_mock_opensearch_client( - self, mock_opensearch_client, index_exists_return_value: bool | dict = False + self, + mock_opensearch_client, + alias_exists_return_value: bool | dict = False, + index_exists_return_value: bool | dict = False, ): """ Configure the mock OpenSearchClient for testing. :param mock_opensearch_client: The patched OpenSearchClient class + :param alias_exists_return_value: Either a boolean (applied to all aliases) + or a dict mapping alias names to booleans :param index_exists_return_value: Either a boolean (applied to all indices) or a dict mapping index names to booleans :return: The mock client instance @@ -33,7 +44,15 @@ def _when_testing_mock_opensearch_client( mock_client_instance = Mock() mock_opensearch_client.return_value = mock_client_instance - # If a dict is provided, use side_effect to return different values per index + # Configure alias_exists mock + if isinstance(alias_exists_return_value, dict): + mock_client_instance.alias_exists.side_effect = lambda alias_name: alias_exists_return_value.get( + alias_name, False + ) + else: + mock_client_instance.alias_exists.return_value = alias_exists_return_value + + # Configure index_exists mock if isinstance(index_exists_return_value, dict): mock_client_instance.index_exists.side_effect = lambda index_name: index_exists_return_value.get( index_name, False @@ -44,17 +63,19 @@ def _when_testing_mock_opensearch_client( return mock_client_instance @patch('handlers.manage_opensearch_indices.OpenSearchClient') - def test_on_create_creates_indices_for_all_compacts_when_none_exist(self, mock_opensearch_client): - """Test that on_create creates indices for all compacts when they don't exist.""" + def test_on_create_creates_versioned_indices_and_aliases_for_all_compacts_when_none_exist( + self, mock_opensearch_client + ): + """Test that on_create creates versioned indices and aliases for all compacts when they don't exist.""" from handlers.manage_opensearch_indices import on_event - # Set up the mock opensearch client - no indices exist + # Set up the mock opensearch client - no aliases or indices exist mock_client_instance = self._when_testing_mock_opensearch_client( - mock_opensearch_client, index_exists_return_value=False + mock_opensearch_client, alias_exists_return_value=False, index_exists_return_value=False ) - # Create the event for a 'Create' request - event = self._create_event('Create') + # Create the event for a 'Create' request with explicit shard/replica configuration + event = self._create_event('Create', {'numberOfShards': 2, 'numberOfReplicas': 1}) # Call the handler on_event(event, self.mock_context) @@ -62,29 +83,41 @@ def test_on_create_creates_indices_for_all_compacts_when_none_exist(self, mock_o # Assert that the OpenSearchClient was instantiated mock_opensearch_client.assert_called_once() - # Assert that index_exists was called for each compact - expected_index_exists_calls = [ + # Assert that alias_exists was called for each compact + expected_alias_exists_calls = [ call('compact_aslp_providers'), call('compact_octp_providers'), call('compact_coun_providers'), ] - mock_client_instance.index_exists.assert_has_calls(expected_index_exists_calls, any_order=False) - self.assertEqual(3, mock_client_instance.index_exists.call_count) + mock_client_instance.alias_exists.assert_has_calls(expected_alias_exists_calls, any_order=False) + self.assertEqual(3, mock_client_instance.alias_exists.call_count) - # Assert that create_index was called for each compact + # Assert that create_index was called for each compact with versioned names self.assertEqual(3, mock_client_instance.create_index.call_count) - # Verify the index names in create_index calls + # Verify the versioned index names in create_index calls create_index_calls = mock_client_instance.create_index.call_args_list index_names_created = [call_args[0][0] for call_args in create_index_calls] self.assertEqual( - ['compact_aslp_providers', 'compact_octp_providers', 'compact_coun_providers'], + ['compact_aslp_providers_v1', 'compact_octp_providers_v1', 'compact_coun_providers_v1'], index_names_created, ) - # Verify the mapping was passed to create_index + # Assert that create_alias was called for each compact + self.assertEqual(3, mock_client_instance.create_alias.call_count) + expected_alias_calls = [ + call('compact_aslp_providers_v1', 'compact_aslp_providers'), + call('compact_octp_providers_v1', 'compact_octp_providers'), + call('compact_coun_providers_v1', 'compact_coun_providers'), + ] + mock_client_instance.create_alias.assert_has_calls(expected_alias_calls, any_order=False) + + # Verify the mapping was passed to create_index with correct shard/replica configuration for call_args in create_index_calls: index_mapping = call_args[0][1] + # Verify the index settings use the provided shard/replica values + self.assertEqual(2, index_mapping['settings']['index']['number_of_shards']) + self.assertEqual(1, index_mapping['settings']['index']['number_of_replicas']) # Verify the mapping has the expected structure self.assertEqual( { @@ -296,20 +329,20 @@ def test_on_create_creates_indices_for_all_compacts_when_none_exist(self, mock_o }, 'filter': {'custom_ascii_folding': {'preserve_original': True, 'type': 'asciifolding'}}, }, - 'index': {'number_of_replicas': 0, 'number_of_shards': 1}, + 'index': {'number_of_replicas': 1, 'number_of_shards': 2}, }, }, index_mapping, ) @patch('handlers.manage_opensearch_indices.OpenSearchClient') - def test_on_create_skips_index_creation_when_all_indices_exist(self, mock_opensearch_client): - """Test that on_create skips index creation when indices already exist.""" + def test_on_create_skips_index_and_alias_creation_when_all_aliases_exist(self, mock_opensearch_client): + """Test that on_create skips index and alias creation when aliases already exist.""" from handlers.manage_opensearch_indices import on_event - # Set up the mock opensearch client - all indices exist + # Set up the mock opensearch client - all aliases exist (meaning indices are already set up) mock_client_instance = self._when_testing_mock_opensearch_client( - mock_opensearch_client, index_exists_return_value=True + mock_opensearch_client, alias_exists_return_value=True ) # Create the event for a 'Create' request @@ -321,25 +354,32 @@ def test_on_create_skips_index_creation_when_all_indices_exist(self, mock_opense # Assert that the OpenSearchClient was instantiated mock_opensearch_client.assert_called_once() - # Assert that index_exists was called for each compact - self.assertEqual(3, mock_client_instance.index_exists.call_count) + # Assert that alias_exists was called for each compact + self.assertEqual(3, mock_client_instance.alias_exists.call_count) - # Assert that create_index was NOT called since indices already exist + # Assert that index_exists was NOT called since aliases already exist + mock_client_instance.index_exists.assert_not_called() + + # Assert that create_index was NOT called since aliases already exist mock_client_instance.create_index.assert_not_called() + # Assert that create_alias was NOT called since aliases already exist + mock_client_instance.create_alias.assert_not_called() + @patch('handlers.manage_opensearch_indices.OpenSearchClient') - def test_on_create_only_creates_missing_indices(self, mock_opensearch_client): - """Test that on_create only creates indices that don't exist.""" + def test_on_create_only_creates_missing_indices_and_aliases(self, mock_opensearch_client): + """Test that on_create only creates indices and aliases that don't exist.""" from handlers.manage_opensearch_indices import on_event - # Set up the mock opensearch client - only aslp index exists + # Set up the mock opensearch client - only aslp alias exists mock_client_instance = self._when_testing_mock_opensearch_client( mock_opensearch_client, - index_exists_return_value={ + alias_exists_return_value={ 'compact_aslp_providers': True, 'compact_octp_providers': False, 'compact_coun_providers': False, }, + index_exists_return_value=False, ) # Create the event for a 'Create' request @@ -348,16 +388,67 @@ def test_on_create_only_creates_missing_indices(self, mock_opensearch_client): # Call the handler on_event(event, self.mock_context) - # Assert that index_exists was called for each compact - self.assertEqual(3, mock_client_instance.index_exists.call_count) + # Assert that alias_exists was called for each compact + self.assertEqual(3, mock_client_instance.alias_exists.call_count) + + # Assert that index_exists was called only for missing aliases (octp and coun) + self.assertEqual(2, mock_client_instance.index_exists.call_count) # Assert that create_index was called only for missing indices (octp and coun) self.assertEqual(2, mock_client_instance.create_index.call_count) - # Verify the correct indices were created + # Verify the correct versioned indices were created create_index_calls = mock_client_instance.create_index.call_args_list index_names_created = [call_args[0][0] for call_args in create_index_calls] - self.assertEqual(['compact_octp_providers', 'compact_coun_providers'], index_names_created) + self.assertEqual(['compact_octp_providers_v1', 'compact_coun_providers_v1'], index_names_created) + + # Assert that create_alias was called only for missing aliases (octp and coun) + self.assertEqual(2, mock_client_instance.create_alias.call_count) + expected_alias_calls = [ + call('compact_octp_providers_v1', 'compact_octp_providers'), + call('compact_coun_providers_v1', 'compact_coun_providers'), + ] + mock_client_instance.create_alias.assert_has_calls(expected_alias_calls, any_order=False) + + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_create_creates_alias_only_when_index_exists_but_alias_does_not(self, mock_opensearch_client): + """Test that on_create creates only the alias when the index exists but the alias doesn't.""" + from handlers.manage_opensearch_indices import on_event + + # Set up the mock opensearch client - index exists but alias doesn't (edge case) + mock_client_instance = self._when_testing_mock_opensearch_client( + mock_opensearch_client, + alias_exists_return_value=False, + index_exists_return_value={ + 'compact_aslp_providers_v1': True, + 'compact_octp_providers_v1': True, + 'compact_coun_providers_v1': True, + }, + ) + + # Create the event for a 'Create' request + event = self._create_event('Create') + + # Call the handler + on_event(event, self.mock_context) + + # Assert that alias_exists was called for each compact + self.assertEqual(3, mock_client_instance.alias_exists.call_count) + + # Assert that index_exists was called for each compact + self.assertEqual(3, mock_client_instance.index_exists.call_count) + + # Assert that create_index was NOT called since indices already exist + mock_client_instance.create_index.assert_not_called() + + # Assert that create_alias was called for each compact (to create the missing aliases) + self.assertEqual(3, mock_client_instance.create_alias.call_count) + expected_alias_calls = [ + call('compact_aslp_providers_v1', 'compact_aslp_providers'), + call('compact_octp_providers_v1', 'compact_octp_providers'), + call('compact_coun_providers_v1', 'compact_coun_providers'), + ] + mock_client_instance.create_alias.assert_has_calls(expected_alias_calls, any_order=False) @patch('handlers.manage_opensearch_indices.OpenSearchClient') def test_on_update_is_noop(self, mock_opensearch_client): diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index 73c48745a..bb4ba3130 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -52,6 +52,41 @@ def test_index_exists_calls_internal_client_with_expected_arguments(self): mock_internal_client.indices.exists.assert_called_once_with(index=index_name) self.assertTrue(result) + def test_alias_exists_calls_internal_client_with_expected_arguments(self): + """Test that alias_exists calls the internal client's indices.exists_alias method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + alias_name = 'test_alias' + mock_internal_client.indices.exists_alias.return_value = True + + result = client.alias_exists(alias_name=alias_name) + + mock_internal_client.indices.exists_alias.assert_called_once_with(name=alias_name) + self.assertTrue(result) + + def test_alias_exists_returns_false_when_alias_does_not_exist(self): + """Test that alias_exists returns False when the alias does not exist.""" + client, mock_internal_client = self._create_client_with_mock() + + alias_name = 'nonexistent_alias' + mock_internal_client.indices.exists_alias.return_value = False + + result = client.alias_exists(alias_name=alias_name) + + mock_internal_client.indices.exists_alias.assert_called_once_with(name=alias_name) + self.assertFalse(result) + + def test_create_alias_calls_internal_client_with_expected_arguments(self): + """Test that create_alias calls the internal client's indices.put_alias method correctly.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index_v1' + alias_name = 'test_alias' + + client.create_alias(index_name=index_name, alias_name=alias_name) + + mock_internal_client.indices.put_alias.assert_called_once_with(index=index_name, name=alias_name) + def test_search_calls_internal_client_with_expected_arguments(self): """Test that search calls the internal client's search method correctly.""" client, mock_internal_client = self._create_client_with_mock() diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 290285036..19a285d1d 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -87,16 +87,6 @@ def __init__( removal_policy=removal_policy, ) - # Create alarm topic for OpenSearch capacity and health monitoring - notifications = environment_context.get('notifications', {}) - self.alarm_topic = AlarmTopic( - self, - 'SearchAlarmTopic', - master_key=search_alarm_encryption_key, - email_subscriptions=notifications.get('email', []), - slack_subscriptions=notifications.get('slack', []), - ) - # Create the OpenSearch domain and associated resources self.provider_search_domain = ProviderSearchDomain( self, @@ -104,7 +94,7 @@ def __init__( environment_name=environment_name, vpc_stack=vpc_stack, compact_abbreviations=persistent_stack.get_list_of_compact_abbreviations(), - alarm_topic=self.alarm_topic, + alarm_topic=persistent_stack.alarm_topic, ingest_lambda_role=self.opensearch_ingest_lambda_role, index_manager_lambda_role=self.opensearch_index_manager_lambda_role, search_api_lambda_role=self.search_api_lambda_role, @@ -122,6 +112,7 @@ def __init__( vpc_stack=vpc_stack, vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_index_manager_lambda_role, + environment_name=environment_name, ) # Create the search providers handler for API Gateway integration @@ -132,7 +123,7 @@ def __init__( vpc_stack=vpc_stack, vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.search_api_lambda_role, - alarm_topic=self.alarm_topic, + alarm_topic=persistent_stack.alarm_topic, ) # Create the populate provider documents handler for manual invocation @@ -145,5 +136,5 @@ def __init__( vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, - alarm_topic=self.alarm_topic, + alarm_topic=persistent_stack.alarm_topic, ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index e06c4c0b5..394ec5f98 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -10,16 +10,26 @@ from common_constructs.stack import Stack from constructs import Construct +from common_constructs.constants import PROD_ENV_NAME from common_constructs.python_function import PythonFunction from stacks.vpc_stack import VpcStack +# Index configuration constants +# Non-prod environments use a single data node, so no replicas are needed +# Production uses 3 data nodes across 3 AZs, so 1 replica ensures data availability +NON_PROD_NUMBER_OF_SHARDS = 1 +NON_PROD_NUMBER_OF_REPLICAS = 0 +PROD_NUMBER_OF_SHARDS = 1 +PROD_NUMBER_OF_REPLICAS = 1 + class IndexManagerCustomResource(Construct): """ Custom resource for managing OpenSearch indices. This construct creates a CloudFormation custom resource that populates the OpenSearch Domain with the needed - provider indices. + provider indices. Indices are created with versioned names (e.g., compact_aslp_providers_v1) and aliases + (e.g., compact_aslp_providers) to enable safe blue-green migrations in the future. """ def __init__( @@ -30,6 +40,7 @@ def __init__( vpc_stack: VpcStack, vpc_subnets: SubnetSelection, lambda_role: IRole, + environment_name: str, ): """ Initialize the IndexManagerCustomResource construct. @@ -39,6 +50,8 @@ def __init__( :param opensearch_domain: The reference to the OpenSearch domain resource :param vpc_stack: The VPC stack :param vpc_subnets: The VPC subnets + :param lambda_role: The IAM role for the Lambda function + :param environment_name: The deployment environment name (e.g., 'prod', 'test') """ super().__init__(scope, construct_id) stack = Stack.of(scope) @@ -155,12 +168,34 @@ def __init__( ], ) + # Determine index configuration based on environment + number_of_shards, number_of_replicas = self._get_index_configuration(environment_name) + # Create custom resource for managing indices - # This custom resource will create the 'compact_{compact}_providers' indices - # with the appropriate mappings once the domain is ready + # This custom resource will create versioned indices (e.g., 'compact_aslp_providers_v1') + # with aliases (e.g., 'compact_aslp_providers') for each compact. + # The alias abstraction enables safe blue-green migrations for future mapping changes. self.index_manager = CustomResource( self, 'IndexManagerCustomResource', resource_type='Custom::IndexManager', service_token=provider.service_token, + properties={ + 'numberOfShards': number_of_shards, + 'numberOfReplicas': number_of_replicas, + }, ) + + def _get_index_configuration(self, environment_name: str) -> tuple[int, int]: + """ + Determine OpenSearch index configuration based on environment. + + Non-prod environments use a single data node, so no replicas are needed. + Production uses 3 data nodes across 3 AZs, so 1 replica ensures data availability. + + :param environment_name: The deployment environment name + :return: Tuple of (number_of_shards, number_of_replicas) + """ + if environment_name == PROD_ENV_NAME: + return PROD_NUMBER_OF_SHARDS, PROD_NUMBER_OF_REPLICAS + return NON_PROD_NUMBER_OF_SHARDS, NON_PROD_NUMBER_OF_REPLICAS diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index 903daa1ce..fe693cb6f 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -182,7 +182,7 @@ def __init__( # Advanced security options advanced_options={ # Prevent queries from accessing multiple indices in a single request - # This is a security control to ensure queries are scoped to a single index + # This is a security control to ensure queries are scoped to a single index, and thus a single compact 'rest.action.multi.allow_explicit_index': 'false', }, logging=LoggingOptions( From 7d0bf76e2f63f085e9db26da0226647f6a54529a Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 14:38:24 -0600 Subject: [PATCH 061/137] remove unused vars --- .../stacks/search_persistent_stack/__init__.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 19a285d1d..5f14744e7 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,11 +1,7 @@ -from aws_cdk import RemovalPolicy from aws_cdk.aws_iam import Role, ServicePrincipal -from aws_cdk.aws_kms import Key -from common_constructs.alarm_topic import AlarmTopic from common_constructs.stack import AppStack from constructs import Construct -from common_constructs.constants import PROD_ENV_NAME from stacks.persistent_stack import PersistentStack from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler @@ -51,9 +47,6 @@ def __init__( scope, construct_id, environment_context=environment_context, environment_name=environment_name, **kwargs ) - # Determine removal policy based on environment - removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY - # Create IAM roles for Lambda functions that need OpenSearch access self.opensearch_ingest_lambda_role = Role( self, @@ -78,15 +71,6 @@ def __init__( description='IAM role for Search API Lambda functions that need read access to OpenSearch Domain', ) - # Create dedicated KMS key for alarm topic encryption - search_alarm_encryption_key = Key( - self, - 'SearchAlarmEncryptionKey', - enable_key_rotation=True, - alias=f'{self.stack_name}-search-alarm-encryption-key', - removal_policy=removal_policy, - ) - # Create the OpenSearch domain and associated resources self.provider_search_domain = ProviderSearchDomain( self, From 68b4fe5ea3a5ed49376df214d9fa9712f229e98b Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 16:46:53 -0600 Subject: [PATCH 062/137] Only return matching privileges on nested searches --- .../data_model/schema/provider/api.py | 8 +- .../lambdas/python/search/handlers/search.py | 84 +++- .../tests/function/test_search_privileges.py | 445 ++++++++++++++++++ .../tests/function/test_search_providers.py | 20 +- 4 files changed, 530 insertions(+), 27 deletions(-) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index 9561cb078..ec4c072ee 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -1,9 +1,9 @@ # ruff: noqa: N801, N815, ARG002 invalid-name unused-argument from datetime import timedelta -from marshmallow import ValidationError, validates_schema +from marshmallow import ValidationError, validates_schema, Schema from marshmallow.fields import UUID, Date, DateTime, Email, Integer, List, Nested, Raw, String -from marshmallow.validate import Length, OneOf, Regexp +from marshmallow.validate import Length, OneOf, Regexp, Range from cc_common.data_model.schema.base_record import ForgivingSchema from cc_common.data_model.schema.common import CCRequestSchema @@ -450,7 +450,7 @@ class StateProviderDetailGeneralResponseSchema(ForgivingSchema): providerUIUrl = String(required=True, allow_none=False) -class SearchProvidersRequestSchema(CCRequestSchema): +class SearchProvidersRequestSchema(Schema): """ Schema for advanced search providers requests. @@ -470,7 +470,7 @@ class SearchProvidersRequestSchema(CCRequestSchema): # Pagination parameters following OpenSearch DSL # 'from' is a reserved word in Python, so we use 'from_' with data_key='from' from_ = Integer(required=False, allow_none=False, data_key='from') - size = Integer(required=False, allow_none=False) + size = Integer(required=False, allow_none=False, validate=Range(min=1, max=100)) # Sort order - required when using search_after pagination # Example: [{"providerId": "asc"}, {"dateOfUpdate": "desc"}] diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 0c2223bfd..d26db636b 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -5,14 +5,15 @@ SearchProvidersRequestSchema, StatePrivilegeGeneralResponseSchema, ) -from cc_common.exceptions import CCInvalidRequestException +from cc_common.exceptions import CCInvalidRequestException, CCInvalidRequestCustomResponseException from cc_common.utils import api_handler from marshmallow import ValidationError from opensearch_client import OpenSearchClient # Default and maximum page sizes for search results -DEFAULT_SIZE = 10 -MAX_SIZE = 100 +MAX_PROVIDER_PAGE_SIZE = 100 +PRIVILEGE_SEARCH_PAGE_SIZE = 2000 +MAX_MATCH_TOTAL_ALLOWED = 10000 @api_handler @@ -59,7 +60,7 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus body = _parse_and_validate_request_body(event) # Build the OpenSearch search body - search_body = _build_opensearch_search_body(body) + search_body = _build_opensearch_search_body(body, size_override=MAX_PROVIDER_PAGE_SIZE) # Build the index name for this compact index_name = f'compact_{compact}_providers' @@ -116,6 +117,20 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu Privileges are extracted from provider documents and combined with license data. Pagination follows OpenSearch DSL using `from`/`size` or `search_after` with `sort`. + If the query includes a nested query on privileges with `inner_hits`, only the matched + privileges will be returned. Otherwise, all privileges for matching providers are returned. + + Example nested query with inner_hits: + { + "query": { + "nested": { + "path": "privileges", + "query": { "term": { "privileges.jurisdiction": "ky" } }, + "inner_hits": {} + } + } + } + :param event: Standard API Gateway event, API schema documented in the CDK ApiStack :param LambdaContext context: :return: Dictionary with privileges array and pagination metadata @@ -126,7 +141,7 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu body = _parse_and_validate_request_body(event) # Build the OpenSearch search body - search_body = _build_opensearch_search_body(body) + search_body = _build_opensearch_search_body(body, size_override=PRIVILEGE_SEARCH_PAGE_SIZE) # Build the index name for this compact index_name = f'compact_{compact}_providers' @@ -140,7 +155,15 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu # Extract hits from the response hits_data = response.get('hits', {}) hits = hits_data.get('hits', []) - total = hits_data.get('total', {}) + total = hits_data['total'] + + if total['value'] >= MAX_MATCH_TOTAL_ALLOWED: + logger.info('request scope too large for current implementation, returning 400 with custom response') + raise CCInvalidRequestCustomResponseException( + response_body={ + 'message': 'Search scope too broad. Please narrow your search.', + } + ) # Extract and flatten privileges from provider records flattened_privileges = [] @@ -150,8 +173,23 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu for hit in hits: provider = hit.get('_source', {}) try: - # Extract privileges and flatten them with license data - provider_privileges = _extract_flattened_privileges(provider) + # Check if inner_hits are present for privileges + # If so, use only the matched privileges; otherwise, use all privileges + inner_hits = hit.get('inner_hits', {}) + privileges_inner_hits = inner_hits.get('privileges', {}).get('hits', {}).get('hits', []) + + if privileges_inner_hits: + # Use only the privileges that matched the nested query + matched_privileges = [ih.get('_source', {}) for ih in privileges_inner_hits] + provider_privileges = _extract_flattened_privileges_from_list( + privileges=matched_privileges, + licenses=provider.get('licenses', []), + provider=provider, + ) + else: + # No inner_hits, return all privileges for this provider + provider_privileges = _extract_flattened_privileges(provider) + for flattened_privilege in provider_privileges: try: # Sanitize using StatePrivilegeGeneralResponseSchema @@ -202,7 +240,7 @@ def _parse_and_validate_request_body(event: dict) -> dict: raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e -def _build_opensearch_search_body(body: dict) -> dict: +def _build_opensearch_search_body(body: dict, size_override: int) -> dict: """ Build the OpenSearch search body from the validated request. @@ -220,8 +258,7 @@ def _build_opensearch_search_body(body: dict) -> dict: if from_param is not None: search_body['from'] = from_param - size = body.get('size', DEFAULT_SIZE) - search_body['size'] = min(size, MAX_SIZE) + search_body['size'] = body.get('size', size_override) # Add sort if provided - required for search_after pagination sort = body.get('sort') @@ -241,7 +278,7 @@ def _build_opensearch_search_body(body: dict) -> dict: def _extract_flattened_privileges(provider: dict) -> list[dict]: """ - Extract and flatten privileges from a provider document. + Extract and flatten all privileges from a provider document. This function combines privilege data with license data to create flattened privilege records similar to what the state API returns. @@ -252,6 +289,29 @@ def _extract_flattened_privileges(provider: dict) -> list[dict]: privileges = provider.get('privileges', []) licenses = provider.get('licenses', []) + return _extract_flattened_privileges_from_list( + privileges=privileges, + licenses=licenses, + provider=provider, + ) + + +def _extract_flattened_privileges_from_list( + privileges: list[dict], + licenses: list[dict], + provider: dict, +) -> list[dict]: + """ + Flatten a list of privileges by combining with license data. + + This function is used both for extracting all privileges from a provider document + and for processing only the matched privileges from inner_hits. + + :param privileges: List of privilege records to flatten + :param licenses: List of license records from the provider + :param provider: Provider document (for email and provider_id logging) + :return: List of flattened privilege records + """ if not privileges: return [] diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index b1ee26995..9be2e0b15 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -271,6 +271,451 @@ def test_privilege_search_includes_last_sort(self, mock_opensearch_client): self.assertIn('lastSort', body) self.assertEqual(['provider-uuid-123', '2024-01-15T10:30:00+00:00'], body['lastSort']) + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_with_inner_hits_returns_only_matched_privileges(self, mock_opensearch_client): + """Test that when inner_hits are present, only matched privileges are returned.""" + from handlers.search import search_api_handler + + provider_id = '00000000-0000-0000-0000-000000000001' + compact = 'aslp' + + # Create a provider with multiple privileges but inner_hits only matches one + hit = { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license-home', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': 'audiologist', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2020-01-01', + 'dateOfRenewal': '2024-01-01', + 'dateOfExpiration': '2025-12-31', + 'npi': '1234567890', + 'licenseNumber': 'AUD-12345', + } + ], + # Provider has THREE privileges + 'privileges': [ + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-KY-001', + 'status': 'active', + }, + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ne', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-02-01', + 'dateOfRenewal': '2024-02-01', + 'dateOfExpiration': '2025-02-01', + 'dateOfUpdate': '2024-02-01T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-NE-001', + 'status': 'active', + }, + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'co', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-03-01', + 'dateOfRenewal': '2024-03-01', + 'dateOfExpiration': '2025-03-01', + 'dateOfUpdate': '2024-03-01T10:30:00+00:00', + 'administratorSetStatus': 'inactive', + 'privilegeId': 'PRIV-CO-001', + 'status': 'inactive', + }, + ], + }, + # inner_hits only contains the KY privilege (simulating a nested query for jurisdiction: ky) + 'inner_hits': { + 'privileges': { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [ + { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_nested': {'field': 'privileges', 'offset': 0}, + '_score': 1.0, + '_source': { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-KY-001', + 'status': 'active', + }, + } + ], + } + } + }, + } + + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + + # Should only return 1 privilege (from inner_hits), not all 3 + self.assertEqual(1, len(body['privileges'])) + self.assertEqual('PRIV-KY-001', body['privileges'][0]['privilegeId']) + self.assertEqual('ky', body['privileges'][0]['jurisdiction']) + + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_with_multiple_inner_hits_returns_all_matched(self, mock_opensearch_client): + """Test that when inner_hits contains multiple matches, all are returned.""" + from handlers.search import search_api_handler + + provider_id = '00000000-0000-0000-0000-000000000001' + compact = 'aslp' + + # Create a provider with multiple privileges, inner_hits matches two of them + hit = { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license-home', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': 'audiologist', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2020-01-01', + 'dateOfRenewal': '2024-01-01', + 'dateOfExpiration': '2025-12-31', + 'npi': '1234567890', + 'licenseNumber': 'AUD-12345', + } + ], + # Provider has THREE privileges + 'privileges': [ + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-KY-001', + 'status': 'active', + }, + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ne', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-02-01', + 'dateOfRenewal': '2024-02-01', + 'dateOfExpiration': '2025-02-01', + 'dateOfUpdate': '2024-02-01T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-NE-001', + 'status': 'active', + }, + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'co', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-03-01', + 'dateOfRenewal': '2024-03-01', + 'dateOfExpiration': '2025-03-01', + 'dateOfUpdate': '2024-03-01T10:30:00+00:00', + 'administratorSetStatus': 'inactive', + 'privilegeId': 'PRIV-CO-001', + 'status': 'inactive', + }, + ], + }, + # inner_hits contains TWO active privileges (simulating nested query for status: active) + 'inner_hits': { + 'privileges': { + 'hits': { + 'total': {'value': 2, 'relation': 'eq'}, + 'hits': [ + { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_nested': {'field': 'privileges', 'offset': 0}, + '_score': 1.0, + '_source': { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-KY-001', + 'status': 'active', + }, + }, + { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_nested': {'field': 'privileges', 'offset': 1}, + '_score': 1.0, + '_source': { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ne', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-02-01', + 'dateOfRenewal': '2024-02-01', + 'dateOfExpiration': '2025-02-01', + 'dateOfUpdate': '2024-02-01T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-NE-001', + 'status': 'active', + }, + }, + ], + } + } + }, + } + + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + + # Should return 2 privileges (from inner_hits), not all 3 + self.assertEqual(2, len(body['privileges'])) + privilege_ids = [p['privilegeId'] for p in body['privileges']] + self.assertIn('PRIV-KY-001', privilege_ids) + self.assertIn('PRIV-NE-001', privilege_ids) + self.assertNotIn('PRIV-CO-001', privilege_ids) + + @patch('handlers.search.OpenSearchClient') + def test_privilege_search_without_inner_hits_returns_all_privileges(self, mock_opensearch_client): + """Test that without inner_hits, all privileges for matching providers are returned.""" + from handlers.search import search_api_handler + + provider_id = '00000000-0000-0000-0000-000000000001' + compact = 'aslp' + + # Create a provider with multiple privileges and NO inner_hits + hit = { + '_index': f'compact_{compact}_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license-home', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': 'audiologist', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2020-01-01', + 'dateOfRenewal': '2024-01-01', + 'dateOfExpiration': '2025-12-31', + 'npi': '1234567890', + 'licenseNumber': 'AUD-12345', + } + ], + # Provider has THREE privileges + 'privileges': [ + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-KY-001', + 'status': 'active', + }, + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'ne', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-02-01', + 'dateOfRenewal': '2024-02-01', + 'dateOfExpiration': '2025-02-01', + 'dateOfUpdate': '2024-02-01T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-NE-001', + 'status': 'active', + }, + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': compact, + 'jurisdiction': 'co', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-03-01', + 'dateOfRenewal': '2024-03-01', + 'dateOfExpiration': '2025-03-01', + 'dateOfUpdate': '2024-03-01T10:30:00+00:00', + 'administratorSetStatus': 'inactive', + 'privilegeId': 'PRIV-CO-001', + 'status': 'inactive', + }, + ], + }, + # No inner_hits - regular query without nested + } + + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + body = json.loads(response['body']) + + # Should return ALL 3 privileges since there are no inner_hits + self.assertEqual(3, len(body['privileges'])) + privilege_ids = [p['privilegeId'] for p in body['privileges']] + self.assertIn('PRIV-KY-001', privilege_ids) + self.assertIn('PRIV-NE-001', privilege_ids) + self.assertIn('PRIV-CO-001', privilege_ids) + def test_unsupported_route_returns_400(self): """Test that unsupported routes return a 400 error.""" from handlers.search import search_api_handler diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 8af18ff08..3f3d1596a 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -113,7 +113,7 @@ def test_basic_search_with_match_all_query(self, mock_opensearch_client): # Verify the search was called with correct parameters mock_client_instance.search.assert_called_once_with( - index_name='compact_aslp_providers', body={'query': {'match_all': {}}, 'size': 10} + index_name='compact_aslp_providers', body={'query': {'match_all': {}}, 'size': 100} ) # Verify response structure @@ -146,7 +146,7 @@ def test_search_with_custom_query(self, mock_opensearch_client): index_name='compact_aslp_providers', body={ 'query': {'bool': {'must': [{'match': {'givenName': 'John'}}, {'term': {'licenseStatus': 'active'}}]}}, - 'size': 10, + 'size': 100, 'from': 20, }, ) @@ -155,17 +155,15 @@ def test_search_with_custom_query(self, mock_opensearch_client): def test_search_size_capped_at_max(self, mock_opensearch_client): """Test that size parameter is capped at MAX_SIZE (100).""" from handlers.search import search_api_handler - - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) - # Request size larger than MAX_SIZE event = self._create_api_event('aslp', body={'query': {'match_all': {}}, 'size': 500}) - search_api_handler(event, self.mock_context) - - call_args = mock_client_instance.search.call_args - search_body = call_args.kwargs['body'] - self.assertEqual(100, search_body['size']) # Capped at MAX_SIZE + result = search_api_handler(event, self.mock_context) + self.assertEqual(400, result['statusCode']) + self.assertEqual({"message": + "Invalid request: " + "{'size': ['Must be greater than or equal to 1 and less than or equal to 100.']}"}, + json.loads(result['body'])) @patch('handlers.search.OpenSearchClient') def test_search_with_sort_parameter(self, mock_opensearch_client): @@ -191,7 +189,7 @@ def test_search_with_sort_parameter(self, mock_opensearch_client): index_name='compact_aslp_providers', body={ 'query': {'match_all': {}}, - 'size': 10, + 'size': 100, 'sort': sort_config, 'search_after': search_after_values, }, From e403c092918b505f5387bf8cc378946bf86c7690 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 16:47:56 -0600 Subject: [PATCH 063/137] formatting --- .../lambdas/python/search/handlers/search.py | 2 +- .../search/tests/function/test_search_providers.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index d26db636b..435cccd13 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -5,7 +5,7 @@ SearchProvidersRequestSchema, StatePrivilegeGeneralResponseSchema, ) -from cc_common.exceptions import CCInvalidRequestException, CCInvalidRequestCustomResponseException +from cc_common.exceptions import CCInvalidRequestCustomResponseException, CCInvalidRequestException from cc_common.utils import api_handler from marshmallow import ValidationError from opensearch_client import OpenSearchClient diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 3f3d1596a..e8925230a 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -155,15 +155,19 @@ def test_search_with_custom_query(self, mock_opensearch_client): def test_search_size_capped_at_max(self, mock_opensearch_client): """Test that size parameter is capped at MAX_SIZE (100).""" from handlers.search import search_api_handler + # Request size larger than MAX_SIZE event = self._create_api_event('aslp', body={'query': {'match_all': {}}, 'size': 500}) result = search_api_handler(event, self.mock_context) self.assertEqual(400, result['statusCode']) - self.assertEqual({"message": - "Invalid request: " - "{'size': ['Must be greater than or equal to 1 and less than or equal to 100.']}"}, - json.loads(result['body'])) + self.assertEqual( + { + 'message': 'Invalid request: ' + "{'size': ['Must be greater than or equal to 1 and less than or equal to 100.']}" + }, + json.loads(result['body']), + ) @patch('handlers.search.OpenSearchClient') def test_search_with_sort_parameter(self, mock_opensearch_client): From 65181e3bd5b6773fd843c5e343989ea9d425520d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 17:46:06 -0600 Subject: [PATCH 064/137] Support privilege CSV exports We have determined to only support CSV exports for privilege search results. This adds the needed infra and logic to convert privilege search results into a csv file. --- .../lambdas/python/common/cc_common/config.py | 4 + .../data_model/schema/provider/api.py | 4 +- .../lambdas/python/search/handlers/search.py | 173 +++++++++++-- .../lambdas/python/search/tests/__init__.py | 1 + .../python/search/tests/function/__init__.py | 7 + .../tests/function/test_search_privileges.py | 235 +++++++++++++----- .../search_api_stack/v1_api/api_model.py | 106 +++----- .../v1_api/privilege_search.py | 8 +- .../search_persistent_stack/__init__.py | 10 + .../export_results_bucket.py | 70 ++++++ .../search_persistent_stack/search_handler.py | 14 +- 11 files changed, 459 insertions(+), 173 deletions(-) create mode 100644 backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py diff --git a/backend/compact-connect/lambdas/python/common/cc_common/config.py b/backend/compact-connect/lambdas/python/common/cc_common/config.py index e7a170b71..e952271fa 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/config.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/config.py @@ -288,6 +288,10 @@ def transaction_client(self): def transaction_reports_bucket_name(self): return os.environ['TRANSACTION_REPORTS_BUCKET_NAME'] + @property + def export_results_bucket_name(self): + return os.environ['EXPORT_RESULTS_BUCKET_NAME'] + @property def transaction_history_table_name(self): return os.environ['TRANSACTION_HISTORY_TABLE_NAME'] diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index ec4c072ee..9c47b143a 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -1,9 +1,9 @@ # ruff: noqa: N801, N815, ARG002 invalid-name unused-argument from datetime import timedelta -from marshmallow import ValidationError, validates_schema, Schema +from marshmallow import Schema, ValidationError, validates_schema from marshmallow.fields import UUID, Date, DateTime, Email, Integer, List, Nested, Raw, String -from marshmallow.validate import Length, OneOf, Regexp, Range +from marshmallow.validate import Length, OneOf, Range, Regexp from cc_common.data_model.schema.base_record import ForgivingSchema from cc_common.data_model.schema.common import CCRequestSchema diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 435cccd13..b2dd25fd0 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -1,5 +1,8 @@ +import csv +import io + from aws_lambda_powertools.utilities.typing import LambdaContext -from cc_common.config import logger +from cc_common.config import config, logger from cc_common.data_model.schema.provider.api import ( ProviderGeneralResponseSchema, SearchProvidersRequestSchema, @@ -15,6 +18,34 @@ PRIVILEGE_SEARCH_PAGE_SIZE = 2000 MAX_MATCH_TOTAL_ALLOWED = 10000 +# Presigned URL expiration time in seconds (1 minute) +PRESIGNED_URL_EXPIRATION_SECONDS = 60 + +# CSV field names for privilege export +PRIVILEGE_CSV_FIELDS = [ + 'type', + 'providerId', + 'compact', + 'jurisdiction', + 'licenseType', + 'privilegeId', + 'status', + 'compactEligibility', + 'dateOfExpiration', + 'dateOfIssuance', + 'dateOfRenewal', + 'dateOfUpdate', + 'familyName', + 'givenName', + 'middleName', + 'suffix', + 'licenseJurisdiction', + 'licenseStatus', + 'licenseStatusName', + 'licenseNumber', + 'npi', +] + @api_handler def search_api_handler(event: dict, context: LambdaContext): @@ -34,8 +65,8 @@ def search_api_handler(event: dict, context: LambdaContext): match api_method: case ('POST', '/v1/compacts/{compact}/providers/search'): return _search_providers(event, context) - case ('POST', '/v1/compacts/{compact}/privileges/search'): - return _search_privileges(event, context) + case ('POST', '/v1/compacts/{compact}/privileges/export'): + return _export_privileges(event, context) # If we get here, the method/resource combination is not supported raise CCInvalidRequestException(f'Unsupported method or resource: {http_method} {resource_path}') @@ -109,13 +140,12 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus return response_body -def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument +def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument """ - Search privileges using OpenSearch. + Export privileges to a CSV file in S3 and return a presigned URL for download. - This endpoint accepts an OpenSearch DSL query body and returns flattened privilege records. - Privileges are extracted from provider documents and combined with license data. - Pagination follows OpenSearch DSL using `from`/`size` or `search_after` with `sort`. + This endpoint accepts an OpenSearch DSL query body, retrieves all matching privilege records, + converts them to CSV format, stores the file in S3, and returns a presigned URL for download. If the query includes a nested query on privileges with `inner_hits`, only the matched privileges will be returned. Otherwise, all privileges for matching providers are returned. @@ -133,20 +163,23 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu :param event: Standard API Gateway event, API schema documented in the CDK ApiStack :param LambdaContext context: - :return: Dictionary with privileges array and pagination metadata + :return: Dictionary with fileUrl containing presigned URL to download the CSV file """ compact = event['pathParameters']['compact'] + # Get the caller's cognito user id + caller_user_id = _get_caller_user_id(event) + # Parse and validate the request body using the schema - body = _parse_and_validate_request_body(event) + body = _parse_and_validate_export_request_body(event) - # Build the OpenSearch search body - search_body = _build_opensearch_search_body(body, size_override=PRIVILEGE_SEARCH_PAGE_SIZE) + # Build the OpenSearch search body (no pagination for export) + search_body = _build_export_search_body(body) # Build the index name for this compact index_name = f'compact_{compact}_providers' - logger.info('Executing OpenSearch privilege search', compact=compact, index_name=index_name) + logger.info('Executing OpenSearch privilege export', compact=compact, index_name=index_name) # Execute the search client = OpenSearchClient() @@ -167,7 +200,6 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu # Extract and flatten privileges from provider records flattened_privileges = [] - last_sort = None privilege_schema = StatePrivilegeGeneralResponseSchema() for hit in hits: @@ -202,8 +234,6 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu privilege_id=flattened_privilege.get('privilegeId'), errors=e.messages, ) - # Track the sort values from the last hit for search_after pagination - last_sort = hit.get('sort') except Exception as e: # noqa: BLE001 broad-exception-caught logger.warning( 'Failed to process provider privileges', @@ -211,17 +241,37 @@ def _search_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu error=str(e), ) - # Build response following OpenSearch DSL structure - response_body = { - 'privileges': flattened_privileges, - 'total': total, - } + logger.info('Found privileges to export', count=len(flattened_privileges)) - # Include sort values from last hit to enable search_after pagination - if last_sort is not None: - response_body['lastSort'] = last_sort + # Generate CSV content from the flattened privileges + csv_content = _generate_csv_content(flattened_privileges) - return response_body + # Generate S3 key path + request_datetime = config.current_standard_datetime.isoformat() + s3_key = f'compact/{compact}/privilegeSearch/caller/{caller_user_id}/time/{request_datetime}/export.csv' + + # Upload CSV to S3 + logger.info('Uploading CSV to S3', bucket=config.export_results_bucket_name, key=s3_key) + config.s3_client.put_object( + Bucket=config.export_results_bucket_name, + Key=s3_key, + Body=csv_content.encode('utf-8'), + ContentType='text/csv', + ) + + # Generate presigned URL for download + presigned_url = config.s3_client.generate_presigned_url( + 'get_object', + Params={ + 'Bucket': config.export_results_bucket_name, + 'Key': s3_key, + }, + ExpiresIn=PRESIGNED_URL_EXPIRATION_SECONDS, + ) + + logger.info('Generated presigned URL for export', url_expires_in=PRESIGNED_URL_EXPIRATION_SECONDS) + + return {'fileUrl': presigned_url} def _parse_and_validate_request_body(event: dict) -> dict: @@ -240,6 +290,44 @@ def _parse_and_validate_request_body(event: dict) -> dict: raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e +def _parse_and_validate_export_request_body(event: dict) -> dict: + """ + Parse and validate the request body for export endpoints. + + Export endpoints only accept the query parameter, no pagination. + + :param event: API Gateway event + :return: Validated request body with query + :raises CCInvalidRequestException: If the request body is invalid + """ + import json + + try: + body = json.loads(event.get('body', '{}')) + if 'query' not in body: + raise CCInvalidRequestException('Request body must contain a query') + return body + except json.JSONDecodeError as e: + logger.warning('Invalid JSON in request body', error=str(e)) + raise CCInvalidRequestException('Invalid JSON in request body') from e + + +def _get_caller_user_id(event: dict) -> str: + """ + Get the caller's cognito user id from the event. + + :param event: API Gateway event + :return: The caller's user id (sub claim from cognito token) + :raises CCInvalidRequestException: If user id cannot be extracted + """ + try: + return event['requestContext']['authorizer']['claims']['sub'] + except (KeyError, TypeError) as e: + logger.warning('Could not extract user id from event', error=str(e)) + # For public endpoints without authorization, use 'anonymous' + return 'anonymous' + + def _build_opensearch_search_body(body: dict, size_override: int) -> dict: """ Build the OpenSearch search body from the validated request. @@ -276,6 +364,41 @@ def _build_opensearch_search_body(body: dict, size_override: int) -> dict: return search_body +def _build_export_search_body(body: dict) -> dict: + """ + Build the OpenSearch search body for export requests. + + Export requests do not support pagination - they return all results up to MAX_MATCH_TOTAL_ALLOWED. + + :param body: Validated request body + :return: OpenSearch search body + """ + return { + 'query': body.get('query', {'match_all': {}}), + 'size': PRIVILEGE_SEARCH_PAGE_SIZE, + } + + +def _generate_csv_content(privileges: list[dict]) -> str: + """ + Generate CSV content from a list of privilege records. + + :param privileges: List of flattened privilege records + :return: CSV content as a string + """ + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=PRIVILEGE_CSV_FIELDS, extrasaction='ignore') + + # Write header row + writer.writeheader() + + # Write data rows + for privilege in privileges: + writer.writerow(privilege) + + return output.getvalue() + + def _extract_flattened_privileges(provider: dict) -> list[dict]: """ Extract and flatten all privileges from a provider document. diff --git a/backend/compact-connect/lambdas/python/search/tests/__init__.py b/backend/compact-connect/lambdas/python/search/tests/__init__.py index 6b11fbe2f..53d1a14ab 100644 --- a/backend/compact-connect/lambdas/python/search/tests/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/__init__.py @@ -23,6 +23,7 @@ def setUpClass(cls): 'PROV_FAM_GIV_MID_INDEX_NAME': 'providerFamGivMid', 'LICENSE_GSI_NAME': 'licenseGSI', 'OPENSEARCH_HOST_ENDPOINT': 'vpc-providersearchd-5bzuqxhpxffk-w6dkpddu.us-east-1.es.amazonaws.com', + 'EXPORT_RESULTS_BUCKET_NAME': 'test-export-results-bucket', 'JURISDICTIONS': json.dumps( [ 'al', diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py index 0703f14a6..7d03f001b 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -26,9 +26,16 @@ def setUp(self): # noqa: N801 invalid-name def build_resources(self): self.create_provider_table() + self.create_export_results_bucket() def delete_resources(self): self._provider_table.delete() + # must delete all objects in the bucket before deleting the bucket + self._bucket.objects.delete() + self._bucket.delete() + def create_export_results_bucket(self): + """Create the mock S3 bucket for export results""" + self._bucket = boto3.resource('s3').create_bucket(Bucket=os.environ['EXPORT_RESULTS_BUCKET_NAME']) def create_provider_table(self): self._provider_table = boto3.resource('dynamodb').create_table( diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 9be2e0b15..88e8b2054 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -7,17 +7,17 @@ @mock_aws -class TestSearchPrivileges(TstFunction): - """Test suite for search_api_handler - privilege search functionality.""" +class TestExportPrivileges(TstFunction): + """Test suite for search_api_handler - privilege export functionality.""" def setUp(self): super().setUp() def _create_api_event(self, compact: str, body: dict = None) -> dict: - """Create a standard API Gateway event for search_privileges.""" + """Create a standard API Gateway event for export_privileges.""" return { - 'resource': '/v1/compacts/{compact}/privileges/search', - 'path': f'/v1/compacts/{compact}/privileges/search', + 'resource': '/v1/compacts/{compact}/privileges/export', + 'path': f'/v1/compacts/{compact}/privileges/export', 'httpMethod': 'POST', 'headers': { 'accept': 'application/json', @@ -30,7 +30,7 @@ def _create_api_event(self, compact: str, body: dict = None) -> dict: 'queryStringParameters': None, 'pathParameters': {'compact': compact}, 'requestContext': { - 'resourcePath': '/v1/compacts/{compact}/privileges/search', + 'resourcePath': '/v1/compacts/{compact}/privileges/export', 'httpMethod': 'POST', 'authorizer': { 'claims': { @@ -134,8 +134,8 @@ def _create_mock_provider_hit_with_privileges( return hit @patch('handlers.search.OpenSearchClient') - def test_privilege_search_returns_flattened_privileges(self, mock_opensearch_client): - """Test that privilege search returns flattened privilege records.""" + def test_privilege_export_returns_presigned_url(self, mock_opensearch_client): + """Test that privilege export returns a presigned URL to a CSV file.""" from handlers.search import search_api_handler # Create a mock response with provider hits containing privileges @@ -155,30 +155,38 @@ def test_privilege_search_returns_flattened_privileges(self, mock_opensearch_cli self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - # Verify response structure has 'privileges' instead of 'providers' - self.assertIn('privileges', body) - self.assertNotIn('providers', body) - self.assertEqual(1, len(body['privileges'])) - - # Verify the flattened privilege has both privilege and license fields - privilege = body['privileges'][0] - self.assertEqual('statePrivilege', privilege['type']) - self.assertEqual('00000000-0000-0000-0000-000000000001', privilege['providerId']) - self.assertEqual('ky', privilege['jurisdiction']) - self.assertEqual('oh', privilege['licenseJurisdiction']) - self.assertEqual('audiologist', privilege['licenseType']) - self.assertEqual('PRIV-001', privilege['privilegeId']) - self.assertEqual('active', privilege['status']) - - # Verify license fields were merged - self.assertEqual('John', privilege['givenName']) - self.assertEqual('Doe', privilege['familyName']) - self.assertEqual('1234567890', privilege['npi']) - self.assertEqual('AUD-12345', privilege['licenseNumber']) + # Verify response contains fileUrl + self.assertIn('fileUrl', body) + self.assertIsInstance(body['fileUrl'], str) + # Verify the URL contains expected parts + self.assertIn('test-export-results-bucket', body['fileUrl']) + self.assertIn('compact/aslp/privilegeSearch', body['fileUrl']) + self.assertIn('test-user-id', body['fileUrl']) # caller user id from event + self.assertIn('export.csv', body['fileUrl']) + + # Verify the CSV file was uploaded to S3 by checking the bucket + import boto3 + + s3_client = boto3.client('s3') + response = s3_client.list_objects_v2( + Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' + ) + self.assertEqual(1, response['KeyCount']) + + # Get the CSV content and verify it contains the expected data + key = response['Contents'][0]['Key'] + csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) + csv_content = csv_obj['Body'].read().decode('utf-8') + + # Verify CSV contains header and data + self.assertIn('type,providerId,compact,jurisdiction', csv_content) + self.assertIn('statePrivilege', csv_content) + self.assertIn('00000000-0000-0000-0000-000000000001', csv_content) + self.assertIn('PRIV-001', csv_content) @patch('handlers.search.OpenSearchClient') - def test_privilege_search_with_empty_results(self, mock_opensearch_client): - """Test that privilege search returns empty array when no results.""" + def test_privilege_export_with_empty_results_returns_empty_csv(self, mock_opensearch_client): + """Test that privilege export with no results returns a presigned URL to an empty CSV.""" from handlers.search import search_api_handler search_response = { @@ -195,11 +203,29 @@ def test_privilege_search_with_empty_results(self, mock_opensearch_client): self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - self.assertEqual({'privileges': [], 'total': {'relation': 'eq', 'value': 0}}, body) + + # Verify response contains fileUrl + self.assertIn('fileUrl', body) + + # Verify the CSV file was uploaded with only headers + import boto3 + + s3_client = boto3.client('s3') + response = s3_client.list_objects_v2( + Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' + ) + key = response['Contents'][0]['Key'] + csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) + csv_content = csv_obj['Body'].read().decode('utf-8') + + # CSV should have header row but no data rows + lines = csv_content.strip().split('\n') + self.assertEqual(1, len(lines)) # Only header row + self.assertIn('type,providerId,compact', lines[0]) @patch('handlers.search.OpenSearchClient') - def test_privilege_search_skips_provider_without_privileges(self, mock_opensearch_client): - """Test that providers without privileges don't add entries.""" + def test_privilege_export_skips_provider_without_privileges(self, mock_opensearch_client): + """Test that providers without privileges result in empty CSV data rows.""" from handlers.search import search_api_handler # Create a provider hit without privileges @@ -239,16 +265,30 @@ def test_privilege_search_skips_provider_without_privileges(self, mock_opensearc self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - self.assertEqual(0, len(body['privileges'])) + + # Verify response contains fileUrl + self.assertIn('fileUrl', body) + + # Verify the CSV has only headers (no data rows) + import boto3 + + s3_client = boto3.client('s3') + response = s3_client.list_objects_v2( + Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' + ) + key = response['Contents'][0]['Key'] + csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) + csv_content = csv_obj['Body'].read().decode('utf-8') + + lines = csv_content.strip().split('\n') + self.assertEqual(1, len(lines)) # Only header row @patch('handlers.search.OpenSearchClient') - def test_privilege_search_includes_last_sort(self, mock_opensearch_client): - """Test that lastSort is included in privilege search response.""" + def test_privilege_export_anonymous_user_when_no_auth(self, mock_opensearch_client): + """Test that export uses 'anonymous' when no auth claims are present.""" from handlers.search import search_api_handler - mock_hit = self._create_mock_provider_hit_with_privileges( - sort_values=['provider-uuid-123', '2024-01-15T10:30:00+00:00'] - ) + mock_hit = self._create_mock_provider_hit_with_privileges() search_response = { 'hits': { 'total': {'value': 1, 'relation': 'eq'}, @@ -257,23 +297,39 @@ def test_privilege_search_includes_last_sort(self, mock_opensearch_client): } self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) - event = self._create_api_event( - 'aslp', - body={ - 'query': {'match_all': {}}, - 'sort': [{'providerId': 'asc'}], + # Create event without auth claims + event = { + 'resource': '/v1/compacts/{compact}/privileges/export', + 'path': '/v1/compacts/aslp/privileges/export', + 'httpMethod': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'origin': 'https://example.org', }, - ) + 'multiValueHeaders': {}, + 'queryStringParameters': None, + 'pathParameters': {'compact': 'aslp'}, + 'requestContext': { + 'resourcePath': '/v1/compacts/{compact}/privileges/export', + 'httpMethod': 'POST', + # No authorizer + }, + 'body': json.dumps({'query': {'match_all': {}}}), + 'isBase64Encoded': False, + } response = search_api_handler(event, self.mock_context) + self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - self.assertIn('lastSort', body) - self.assertEqual(['provider-uuid-123', '2024-01-15T10:30:00+00:00'], body['lastSort']) + + # Verify the URL contains 'anonymous' instead of user id + self.assertIn('fileUrl', body) + self.assertIn('caller/anonymous/', body['fileUrl']) @patch('handlers.search.OpenSearchClient') - def test_privilege_search_with_inner_hits_returns_only_matched_privileges(self, mock_opensearch_client): - """Test that when inner_hits are present, only matched privileges are returned.""" + def test_privilege_export_with_inner_hits_exports_only_matched_privileges(self, mock_opensearch_client): + """Test that when inner_hits are present, only matched privileges are exported to CSV.""" from handlers.search import search_api_handler provider_id = '00000000-0000-0000-0000-000000000001' @@ -416,14 +472,29 @@ def test_privilege_search_with_inner_hits_returns_only_matched_privileges(self, self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - # Should only return 1 privilege (from inner_hits), not all 3 - self.assertEqual(1, len(body['privileges'])) - self.assertEqual('PRIV-KY-001', body['privileges'][0]['privilegeId']) - self.assertEqual('ky', body['privileges'][0]['jurisdiction']) + # Verify response contains fileUrl + self.assertIn('fileUrl', body) + + # Verify the CSV contains only the matched privilege + import boto3 + + s3_client = boto3.client('s3') + response = s3_client.list_objects_v2( + Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' + ) + key = response['Contents'][0]['Key'] + csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) + csv_content = csv_obj['Body'].read().decode('utf-8') + + lines = csv_content.strip().split('\n') + self.assertEqual(2, len(lines)) # Header + 1 data row + self.assertIn('PRIV-KY-001', csv_content) + self.assertNotIn('PRIV-NE-001', csv_content) + self.assertNotIn('PRIV-CO-001', csv_content) @patch('handlers.search.OpenSearchClient') - def test_privilege_search_with_multiple_inner_hits_returns_all_matched(self, mock_opensearch_client): - """Test that when inner_hits contains multiple matches, all are returned.""" + def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, mock_opensearch_client): + """Test that when inner_hits contains multiple matches, all are exported to CSV.""" from handlers.search import search_api_handler provider_id = '00000000-0000-0000-0000-000000000001' @@ -587,16 +658,29 @@ def test_privilege_search_with_multiple_inner_hits_returns_all_matched(self, moc self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - # Should return 2 privileges (from inner_hits), not all 3 - self.assertEqual(2, len(body['privileges'])) - privilege_ids = [p['privilegeId'] for p in body['privileges']] - self.assertIn('PRIV-KY-001', privilege_ids) - self.assertIn('PRIV-NE-001', privilege_ids) - self.assertNotIn('PRIV-CO-001', privilege_ids) + # Verify response contains fileUrl + self.assertIn('fileUrl', body) + + # Verify the CSV contains only the 2 matched privileges + import boto3 + + s3_client = boto3.client('s3') + response = s3_client.list_objects_v2( + Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' + ) + key = response['Contents'][0]['Key'] + csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) + csv_content = csv_obj['Body'].read().decode('utf-8') + + lines = csv_content.strip().split('\n') + self.assertEqual(3, len(lines)) # Header + 2 data rows + self.assertIn('PRIV-KY-001', csv_content) + self.assertIn('PRIV-NE-001', csv_content) + self.assertNotIn('PRIV-CO-001', csv_content) @patch('handlers.search.OpenSearchClient') - def test_privilege_search_without_inner_hits_returns_all_privileges(self, mock_opensearch_client): - """Test that without inner_hits, all privileges for matching providers are returned.""" + def test_privilege_export_without_inner_hits_exports_all_privileges(self, mock_opensearch_client): + """Test that without inner_hits, all privileges for matching providers are exported.""" from handlers.search import search_api_handler provider_id = '00000000-0000-0000-0000-000000000001' @@ -709,12 +793,25 @@ def test_privilege_search_without_inner_hits_returns_all_privileges(self, mock_o self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - # Should return ALL 3 privileges since there are no inner_hits - self.assertEqual(3, len(body['privileges'])) - privilege_ids = [p['privilegeId'] for p in body['privileges']] - self.assertIn('PRIV-KY-001', privilege_ids) - self.assertIn('PRIV-NE-001', privilege_ids) - self.assertIn('PRIV-CO-001', privilege_ids) + # Verify response contains fileUrl + self.assertIn('fileUrl', body) + + # Verify the CSV contains all 3 privileges + import boto3 + + s3_client = boto3.client('s3') + response = s3_client.list_objects_v2( + Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' + ) + key = response['Contents'][0]['Key'] + csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) + csv_content = csv_obj['Body'].read().decode('utf-8') + + lines = csv_content.strip().split('\n') + self.assertEqual(4, len(lines)) # Header + 3 data rows + self.assertIn('PRIV-KY-001', csv_content) + self.assertIn('PRIV-NE-001', csv_content) + self.assertIn('PRIV-CO-001', csv_content) def test_unsupported_route_returns_400(self): """Test that unsupported routes return a 400 error.""" diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py index 72c350614..4f764c94d 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api_model.py @@ -73,17 +73,40 @@ def search_providers_request_model(self) -> Model: ) return self.api._v1_search_providers_request_model + @property + def _export_privileges_request_schema(self) -> JsonSchema: + """ + Return the export privileges request schema. + + This schema is similar to the search request schema but without pagination parameters. + The export endpoint does not support pagination - it returns all results as a CSV file. + """ + return JsonSchema( + type=JsonSchemaType.OBJECT, + additional_properties=False, + required=['query'], + properties={ + 'query': JsonSchema( + type=JsonSchemaType.OBJECT, + description='The OpenSearch query body', + ), + }, + ) + @property def search_privileges_request_model(self) -> Model: """ - Return the search privileges request model, which should only be created once per API. + Return the export privileges request model, which should only be created once per API. + + This model is used for the privilege export endpoint and does not include + pagination parameters (size, from, search_after). """ if hasattr(self.api, '_v1_search_privileges_request_model'): return self.api._v1_search_privileges_request_model self.api._v1_search_privileges_request_model = self.api.add_model( - 'V1SearchPrivilegesRequestModel', - description='Search privileges request model following OpenSearch DSL', - schema=self._common_search_request_schema, + 'V1ExportPrivilegesRequestModel', + description='Export privileges request model - query only, no pagination', + schema=self._export_privileges_request_schema, ) return self.api._v1_search_privileges_request_model @@ -127,86 +150,25 @@ def search_providers_response_model(self) -> Model: @property def search_privileges_response_model(self) -> Model: - """Return the search privileges response model, which should only be created once per API""" + """Return the export privileges response model, which should only be created once per API""" if hasattr(self.api, '_v1_search_privileges_response_model'): return self.api._v1_search_privileges_response_model self.api._v1_search_privileges_response_model = self.api.add_model( - 'V1SearchPrivilegesResponseModel', - description='Search privileges response model', + 'V1ExportPrivilegesResponseModel', + description='Export privileges response model with presigned URL to CSV file', schema=JsonSchema( type=JsonSchemaType.OBJECT, - required=['privileges', 'total'], + required=['fileUrl'], properties={ - 'privileges': JsonSchema( - type=JsonSchemaType.ARRAY, - items=self._flattened_privilege_response_schema, - ), - 'total': self._search_response_total_schema, - 'lastSort': JsonSchema( - type=JsonSchemaType.ARRAY, - description='Sort values from the last hit to use with search_after for the next page', + 'fileUrl': JsonSchema( + type=JsonSchemaType.STRING, + description='Presigned URL to download the CSV file containing the export results', ), }, ), ) return self.api._v1_search_privileges_response_model - @property - def _flattened_privilege_response_schema(self): - """ - Schema for flattened privilege response - combines privilege and license data. - This mirrors StatePrivilegeGeneralResponseSchema for the search API. - """ - stack: AppStack = AppStack.of(self.api) - - return JsonSchema( - type=JsonSchemaType.OBJECT, - required=[ - 'type', - 'providerId', - 'compact', - 'jurisdiction', - 'licenseType', - 'privilegeId', - 'status', - 'compactEligibility', - 'dateOfExpiration', - 'dateOfIssuance', - 'dateOfRenewal', - 'dateOfUpdate', - 'familyName', - 'givenName', - 'licenseJurisdiction', - 'licenseStatus', - ], - properties={ - 'type': JsonSchema(type=JsonSchemaType.STRING, enum=['statePrivilege']), - 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), - 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), - 'jurisdiction': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions')), - 'licenseType': JsonSchema(type=JsonSchemaType.STRING), - 'privilegeId': JsonSchema(type=JsonSchemaType.STRING), - 'status': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), - 'compactEligibility': JsonSchema(type=JsonSchemaType.STRING, enum=['eligible', 'ineligible']), - 'dateOfExpiration': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), - 'dateOfIssuance': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), - 'dateOfRenewal': JsonSchema(type=JsonSchemaType.STRING, format='date', pattern=cc_api.YMD_FORMAT), - 'dateOfUpdate': JsonSchema(type=JsonSchemaType.STRING, format='date-time'), - 'familyName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), - 'givenName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), - 'licenseJurisdiction': JsonSchema( - type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions') - ), - 'licenseStatus': JsonSchema(type=JsonSchemaType.STRING, enum=['active', 'inactive']), - # Optional fields - 'middleName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), - 'suffix': JsonSchema(type=JsonSchemaType.STRING, max_length=100), - 'licenseStatusName': JsonSchema(type=JsonSchemaType.STRING, max_length=100), - 'licenseNumber': JsonSchema(type=JsonSchemaType.STRING, max_length=100), - 'npi': JsonSchema(type=JsonSchemaType.STRING, pattern='^[0-9]{10}$'), - }, - ) - @property def _providers_response_schema(self): stack: AppStack = AppStack.of(self.api) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py index 27f8f6e77..31f120edb 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py @@ -28,22 +28,22 @@ def __init__( self.api: CCApi = resource.api self.api_model = api_model - self._add_search_privileges( + self._add_export_privileges( method_options=method_options, search_persistent_stack=search_persistent_stack, ) - def _add_search_privileges( + def _add_export_privileges( self, method_options: MethodOptions, search_persistent_stack: search_persistent_stack.SearchPersistentStack, ): - search_resource = self.resource.add_resource('search') + export_resource = self.resource.add_resource('export') # Get the search handler from the search persistent stack (same handler as provider search) handler = search_persistent_stack.search_handler.handler - search_resource.add_method( + privilege_search = export_resource.add_method( 'POST', request_validator=self.api.parameter_body_validator, request_models={'application/json': self.api_model.search_privileges_request_model}, diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 5f14744e7..3983c4998 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -3,6 +3,7 @@ from constructs import Construct from stacks.persistent_stack import PersistentStack +from stacks.search_persistent_stack.export_results_bucket import ExportResultsBucket from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain @@ -88,6 +89,14 @@ def __init__( self.domain = self.provider_search_domain.domain self.opensearch_encryption_key = self.provider_search_domain.encryption_key + # Create the export results bucket for temporary CSV files + self.export_results_bucket = ExportResultsBucket( + self, + 'ExportResultsBucket', + access_logs_bucket=persistent_stack.access_logs_bucket, + encryption_key=persistent_stack.shared_encryption_key, + ) + # Create the index manager custom resource self.index_manager_custom_resource = IndexManagerCustomResource( self, @@ -108,6 +117,7 @@ def __init__( vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.search_api_lambda_role, alarm_topic=persistent_stack.alarm_topic, + export_results_bucket=self.export_results_bucket, ) # Create the populate provider documents handler for manual invocation diff --git a/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py b/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py new file mode 100644 index 000000000..acbf1da08 --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from aws_cdk import Duration +from aws_cdk.aws_kms import IKey +from aws_cdk.aws_s3 import BucketEncryption, CorsRule, HttpMethods, LifecycleRule +from cdk_nag import NagSuppressions +from common_constructs.access_logs_bucket import AccessLogsBucket +from common_constructs.bucket import Bucket +from constructs import Construct + + +class ExportResultsBucket(Bucket): + """ + S3 bucket to store temporary CSV export result files. + + Files stored in this bucket are automatically deleted after 1 day + since they are only needed for the duration of the download. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + *, + access_logs_bucket: AccessLogsBucket, + encryption_key: IKey, + **kwargs, + ): + super().__init__( + scope, + construct_id, + encryption=BucketEncryption.KMS, + encryption_key=encryption_key, + server_access_logs_bucket=access_logs_bucket, + # Versioning is not needed for temporary export files + versioned=False, + cors=[ + CorsRule( + allowed_methods=[HttpMethods.GET], + allowed_origins=['*'], + allowed_headers=['*'], + ), + ], + # Automatically delete objects after 1 day + lifecycle_rules=[ + LifecycleRule( + id='DeleteExportFilesAfterOneDay', + enabled=True, + expiration=Duration.days(1), + ), + ], + **kwargs, + ) + + NagSuppressions.add_resource_suppressions( + self, + suppressions=[ + { + 'id': 'HIPAA.Security-S3BucketReplicationEnabled', + 'reason': 'This bucket houses transitory export data only that is deleted after 1 day. ' + 'Replication to a backup bucket is unhelpful.', + }, + { + 'id': 'HIPAA.Security-S3BucketVersioningEnabled', + 'reason': 'This bucket houses transitory export data only. ' + 'Version history is not needed for temporary files.', + }, + ], + ) + diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_handler.py b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py index b51b340be..0a03c6f8e 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/search_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py @@ -5,6 +5,7 @@ from aws_cdk.aws_iam import IRole from aws_cdk.aws_logs import RetentionDays from aws_cdk.aws_opensearchservice import Domain +from aws_cdk.aws_s3 import IBucket from aws_cdk.aws_sns import ITopic from cdk_nag import NagSuppressions from common_constructs.stack import Stack @@ -31,6 +32,7 @@ def __init__( vpc_subnets: SubnetSelection, lambda_role: IRole, alarm_topic: ITopic, + export_results_bucket: IBucket, ): """ Initialize the SearchHandler construct. @@ -42,6 +44,7 @@ def __init__( :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function :param alarm_topic: The SNS topic for alarms + :param export_results_bucket: The S3 bucket for storing export result CSV files """ super().__init__(scope, construct_id) stack = Stack.of(scope) @@ -58,6 +61,7 @@ def __init__( log_retention=RetentionDays.ONE_MONTH, environment={ 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, + 'EXPORT_RESULTS_BUCKET_NAME': export_results_bucket.bucket_name, **stack.common_env_vars, }, timeout=Duration.seconds(29), @@ -71,6 +75,12 @@ def __init__( # Grant the handler read access to the OpenSearch domain opensearch_domain.grant_read(self.handler) + # Grant the handler write access to the export results bucket + export_results_bucket.grant_write(self.handler) + + # Grant the handler permission to generate presigned URLs for the export results bucket + export_results_bucket.grant_read(self.handler) + # Add CDK Nag suppressions for the Lambda function's IAM role NagSuppressions.add_resource_suppressions_by_path( stack, @@ -80,7 +90,9 @@ def __init__( 'id': 'AwsSolutions-IAM5', 'reason': 'The grant_read method requires wildcard permissions on the OpenSearch domain to ' 'read from indices. This is appropriate for a search function that needs to query ' - 'provider indices in the domain.', + 'provider indices in the domain. Additionally, grant_write and grant_read on the S3 bucket ' + 'use wildcard permissions for object-level operations which is required for writing and ' + 'generating presigned URLs for export result CSV files.', }, ], ) From 0421b2ae267876e274bdabb38a2a35468443b900 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 8 Dec 2025 17:47:06 -0600 Subject: [PATCH 065/137] formatting --- .../lambdas/python/search/tests/function/__init__.py | 1 + .../stacks/search_persistent_stack/export_results_bucket.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py index 7d03f001b..9044cc9c8 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -33,6 +33,7 @@ def delete_resources(self): # must delete all objects in the bucket before deleting the bucket self._bucket.objects.delete() self._bucket.delete() + def create_export_results_bucket(self): """Create the mock S3 bucket for export results""" self._bucket = boto3.resource('s3').create_bucket(Bucket=os.environ['EXPORT_RESULTS_BUCKET_NAME']) diff --git a/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py b/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py index acbf1da08..2f143a873 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py +++ b/backend/compact-connect/stacks/search_persistent_stack/export_results_bucket.py @@ -67,4 +67,3 @@ def __init__( }, ], ) - From c4f307738d992e6a3afd2a4322b2303449e1d222 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 09:23:14 -0600 Subject: [PATCH 066/137] update comment --- .../stacks/search_persistent_stack/provider_search_domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index fe693cb6f..f7a17b681 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -121,7 +121,7 @@ def __init__( ) # Create CloudWatch Logs resource policy to allow OpenSearch to write logs - # This is done manually to avoid CDK creating an auto-generated Lambda function + # This is set here to avoid CDK creating an auto-generated Lambda function # The resource ARNs must include ':*' to grant permissions on log streams within the log groups ResourcePolicy( self, From 74d3361881f147c5f2a3209b6b0c52ceb5da26db Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 09:36:48 -0600 Subject: [PATCH 067/137] PR feedback --- .../common/cc_common/data_model/schema/provider/api.py | 6 +++--- .../python/search/tests/function/test_search_providers.py | 1 + .../stacks/search_persistent_stack/__init__.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index 9c47b143a..043477ba3 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -1,7 +1,7 @@ # ruff: noqa: N801, N815, ARG002 invalid-name unused-argument from datetime import timedelta -from marshmallow import Schema, ValidationError, validates_schema +from marshmallow import ValidationError, validates_schema from marshmallow.fields import UUID, Date, DateTime, Email, Integer, List, Nested, Raw, String from marshmallow.validate import Length, OneOf, Range, Regexp @@ -450,7 +450,7 @@ class StateProviderDetailGeneralResponseSchema(ForgivingSchema): providerUIUrl = String(required=True, allow_none=False) -class SearchProvidersRequestSchema(Schema): +class SearchProvidersRequestSchema(CCRequestSchema): """ Schema for advanced search providers requests. @@ -469,7 +469,7 @@ class SearchProvidersRequestSchema(Schema): # Pagination parameters following OpenSearch DSL # 'from' is a reserved word in Python, so we use 'from_' with data_key='from' - from_ = Integer(required=False, allow_none=False, data_key='from') + from_ = Integer(required=False, allow_none=False, data_key='from', validate=Range(min=0, max=9900)) size = Integer(required=False, allow_none=False, validate=Range(min=1, max=100)) # Sort order - required when using search_after pagination diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index e8925230a..8b75c2ba7 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -168,6 +168,7 @@ def test_search_size_capped_at_max(self, mock_opensearch_client): }, json.loads(result['body']), ) + mock_opensearch_client.assert_not_called() @patch('handlers.search.OpenSearchClient') def test_search_with_sort_parameter(self, mock_opensearch_client): diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 3983c4998..62be3fc05 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -63,7 +63,7 @@ def __init__( description='IAM role for index manager Lambda function that needs read/write access to OpenSearch Domain', ) - # Create IAM role for Lambda functions access OpenSearch through API + # Create IAM role for Lambda functions that access OpenSearch through API # this role only needs read access self.search_api_lambda_role = Role( self, From d2394d716bcfa65ac771c9e90be6939f8e7cc5cf Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 10:18:31 -0600 Subject: [PATCH 068/137] Return 404 if no matches found --- .../lambdas/python/search/handlers/search.py | 14 +- .../tests/function/test_search_privileges.py | 253 ++---------------- 2 files changed, 29 insertions(+), 238 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index b2dd25fd0..d7cb32685 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -8,7 +8,11 @@ SearchProvidersRequestSchema, StatePrivilegeGeneralResponseSchema, ) -from cc_common.exceptions import CCInvalidRequestCustomResponseException, CCInvalidRequestException +from cc_common.exceptions import ( + CCInvalidRequestCustomResponseException, + CCInvalidRequestException, + CCNotFoundException, +) from cc_common.utils import api_handler from marshmallow import ValidationError from opensearch_client import OpenSearchClient @@ -46,7 +50,7 @@ 'npi', ] - +# TODO - add auth wrapper to check for readGeneral scope after testing @api_handler def search_api_handler(event: dict, context: LambdaContext): """ @@ -243,6 +247,10 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu logger.info('Found privileges to export', count=len(flattened_privileges)) + # If no privileges were found, return 404 + if not flattened_privileges: + raise CCNotFoundException('The search parameters did not match any privileges.') + # Generate CSV content from the flattened privileges csv_content = _generate_csv_content(flattened_privileges) @@ -324,7 +332,7 @@ def _get_caller_user_id(event: dict) -> str: return event['requestContext']['authorizer']['claims']['sub'] except (KeyError, TypeError) as e: logger.warning('Could not extract user id from event', error=str(e)) - # For public endpoints without authorization, use 'anonymous' + # TODO - remove this after testing and raise errors return 'anonymous' diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 88e8b2054..5c71ebc30 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -185,8 +185,8 @@ def test_privilege_export_returns_presigned_url(self, mock_opensearch_client): self.assertIn('PRIV-001', csv_content) @patch('handlers.search.OpenSearchClient') - def test_privilege_export_with_empty_results_returns_empty_csv(self, mock_opensearch_client): - """Test that privilege export with no results returns a presigned URL to an empty CSV.""" + def test_privilege_export_with_empty_results_returns_404(self, mock_opensearch_client): + """Test that privilege export with no results returns a 404 error.""" from handlers.search import search_api_handler search_response = { @@ -201,31 +201,26 @@ def test_privilege_export_with_empty_results_returns_empty_csv(self, mock_opense response = search_api_handler(event, self.mock_context) - self.assertEqual(200, response['statusCode']) + self.assertEqual(404, response['statusCode']) body = json.loads(response['body']) - # Verify response contains fileUrl - self.assertIn('fileUrl', body) + # Verify response contains error message + self.assertIn('message', body) + self.assertEqual('The search parameters did not match any privileges.', body['message']) - # Verify the CSV file was uploaded with only headers + # Verify no CSV file was uploaded to S3 import boto3 s3_client = boto3.client('s3') response = s3_client.list_objects_v2( Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' ) - key = response['Contents'][0]['Key'] - csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) - csv_content = csv_obj['Body'].read().decode('utf-8') - - # CSV should have header row but no data rows - lines = csv_content.strip().split('\n') - self.assertEqual(1, len(lines)) # Only header row - self.assertIn('type,providerId,compact', lines[0]) + # Should have no objects + self.assertEqual(0, response.get('KeyCount', 0)) @patch('handlers.search.OpenSearchClient') - def test_privilege_export_skips_provider_without_privileges(self, mock_opensearch_client): - """Test that providers without privileges result in empty CSV data rows.""" + def test_privilege_export_skips_provider_without_privileges_returns_404(self, mock_opensearch_client): + """Test that providers without privileges result in a 404 error.""" from handlers.search import search_api_handler # Create a provider hit without privileges @@ -263,234 +258,22 @@ def test_privilege_export_skips_provider_without_privileges(self, mock_opensearc response = search_api_handler(event, self.mock_context) - self.assertEqual(200, response['statusCode']) + self.assertEqual(404, response['statusCode']) body = json.loads(response['body']) - # Verify response contains fileUrl - self.assertIn('fileUrl', body) + # Verify response contains error message + self.assertIn('message', body) + self.assertEqual('The search parameters did not match any privileges.', body['message']) - # Verify the CSV has only headers (no data rows) + # Verify no CSV file was uploaded to S3 import boto3 s3_client = boto3.client('s3') response = s3_client.list_objects_v2( Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' ) - key = response['Contents'][0]['Key'] - csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) - csv_content = csv_obj['Body'].read().decode('utf-8') - - lines = csv_content.strip().split('\n') - self.assertEqual(1, len(lines)) # Only header row - - @patch('handlers.search.OpenSearchClient') - def test_privilege_export_anonymous_user_when_no_auth(self, mock_opensearch_client): - """Test that export uses 'anonymous' when no auth claims are present.""" - from handlers.search import search_api_handler - - mock_hit = self._create_mock_provider_hit_with_privileges() - search_response = { - 'hits': { - 'total': {'value': 1, 'relation': 'eq'}, - 'hits': [mock_hit], - } - } - self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) - - # Create event without auth claims - event = { - 'resource': '/v1/compacts/{compact}/privileges/export', - 'path': '/v1/compacts/aslp/privileges/export', - 'httpMethod': 'POST', - 'headers': { - 'Content-Type': 'application/json', - 'origin': 'https://example.org', - }, - 'multiValueHeaders': {}, - 'queryStringParameters': None, - 'pathParameters': {'compact': 'aslp'}, - 'requestContext': { - 'resourcePath': '/v1/compacts/{compact}/privileges/export', - 'httpMethod': 'POST', - # No authorizer - }, - 'body': json.dumps({'query': {'match_all': {}}}), - 'isBase64Encoded': False, - } - - response = search_api_handler(event, self.mock_context) - - self.assertEqual(200, response['statusCode']) - body = json.loads(response['body']) - - # Verify the URL contains 'anonymous' instead of user id - self.assertIn('fileUrl', body) - self.assertIn('caller/anonymous/', body['fileUrl']) - - @patch('handlers.search.OpenSearchClient') - def test_privilege_export_with_inner_hits_exports_only_matched_privileges(self, mock_opensearch_client): - """Test that when inner_hits are present, only matched privileges are exported to CSV.""" - from handlers.search import search_api_handler - - provider_id = '00000000-0000-0000-0000-000000000001' - compact = 'aslp' - - # Create a provider with multiple privileges but inner_hits only matches one - hit = { - '_index': f'compact_{compact}_providers', - '_id': provider_id, - '_score': 1.0, - '_source': { - 'providerId': provider_id, - 'type': 'provider', - 'dateOfUpdate': '2024-01-15T10:30:00+00:00', - 'compact': compact, - 'licenseJurisdiction': 'oh', - 'licenseStatus': 'active', - 'compactEligibility': 'eligible', - 'givenName': 'John', - 'familyName': 'Doe', - 'dateOfExpiration': '2025-12-31', - 'jurisdictionUploadedLicenseStatus': 'active', - 'jurisdictionUploadedCompactEligibility': 'eligible', - 'birthMonthDay': '06-15', - 'licenses': [ - { - 'providerId': provider_id, - 'type': 'license-home', - 'dateOfUpdate': '2024-01-15T10:30:00+00:00', - 'compact': compact, - 'jurisdiction': 'oh', - 'licenseType': 'audiologist', - 'licenseStatus': 'active', - 'compactEligibility': 'eligible', - 'jurisdictionUploadedLicenseStatus': 'active', - 'jurisdictionUploadedCompactEligibility': 'eligible', - 'givenName': 'John', - 'familyName': 'Doe', - 'dateOfIssuance': '2020-01-01', - 'dateOfRenewal': '2024-01-01', - 'dateOfExpiration': '2025-12-31', - 'npi': '1234567890', - 'licenseNumber': 'AUD-12345', - } - ], - # Provider has THREE privileges - 'privileges': [ - { - 'type': 'privilege', - 'providerId': provider_id, - 'compact': compact, - 'jurisdiction': 'ky', - 'licenseJurisdiction': 'oh', - 'licenseType': 'audiologist', - 'dateOfIssuance': '2024-01-15', - 'dateOfRenewal': '2024-01-15', - 'dateOfExpiration': '2025-01-15', - 'dateOfUpdate': '2024-01-15T10:30:00+00:00', - 'administratorSetStatus': 'active', - 'privilegeId': 'PRIV-KY-001', - 'status': 'active', - }, - { - 'type': 'privilege', - 'providerId': provider_id, - 'compact': compact, - 'jurisdiction': 'ne', - 'licenseJurisdiction': 'oh', - 'licenseType': 'audiologist', - 'dateOfIssuance': '2024-02-01', - 'dateOfRenewal': '2024-02-01', - 'dateOfExpiration': '2025-02-01', - 'dateOfUpdate': '2024-02-01T10:30:00+00:00', - 'administratorSetStatus': 'active', - 'privilegeId': 'PRIV-NE-001', - 'status': 'active', - }, - { - 'type': 'privilege', - 'providerId': provider_id, - 'compact': compact, - 'jurisdiction': 'co', - 'licenseJurisdiction': 'oh', - 'licenseType': 'audiologist', - 'dateOfIssuance': '2024-03-01', - 'dateOfRenewal': '2024-03-01', - 'dateOfExpiration': '2025-03-01', - 'dateOfUpdate': '2024-03-01T10:30:00+00:00', - 'administratorSetStatus': 'inactive', - 'privilegeId': 'PRIV-CO-001', - 'status': 'inactive', - }, - ], - }, - # inner_hits only contains the KY privilege (simulating a nested query for jurisdiction: ky) - 'inner_hits': { - 'privileges': { - 'hits': { - 'total': {'value': 1, 'relation': 'eq'}, - 'hits': [ - { - '_index': f'compact_{compact}_providers', - '_id': provider_id, - '_nested': {'field': 'privileges', 'offset': 0}, - '_score': 1.0, - '_source': { - 'type': 'privilege', - 'providerId': provider_id, - 'compact': compact, - 'jurisdiction': 'ky', - 'licenseJurisdiction': 'oh', - 'licenseType': 'audiologist', - 'dateOfIssuance': '2024-01-15', - 'dateOfRenewal': '2024-01-15', - 'dateOfExpiration': '2025-01-15', - 'dateOfUpdate': '2024-01-15T10:30:00+00:00', - 'administratorSetStatus': 'active', - 'privilegeId': 'PRIV-KY-001', - 'status': 'active', - }, - } - ], - } - } - }, - } - - search_response = { - 'hits': { - 'total': {'value': 1, 'relation': 'eq'}, - 'hits': [hit], - } - } - self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) - - event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) - - response = search_api_handler(event, self.mock_context) - - self.assertEqual(200, response['statusCode']) - body = json.loads(response['body']) - - # Verify response contains fileUrl - self.assertIn('fileUrl', body) - - # Verify the CSV contains only the matched privilege - import boto3 - - s3_client = boto3.client('s3') - response = s3_client.list_objects_v2( - Bucket='test-export-results-bucket', Prefix='compact/aslp/privilegeSearch/caller/test-user-id' - ) - key = response['Contents'][0]['Key'] - csv_obj = s3_client.get_object(Bucket='test-export-results-bucket', Key=key) - csv_content = csv_obj['Body'].read().decode('utf-8') - - lines = csv_content.strip().split('\n') - self.assertEqual(2, len(lines)) # Header + 1 data row - self.assertIn('PRIV-KY-001', csv_content) - self.assertNotIn('PRIV-NE-001', csv_content) - self.assertNotIn('PRIV-CO-001', csv_content) + # Should have no objects + self.assertEqual(0, response.get('KeyCount', 0)) @patch('handlers.search.OpenSearchClient') def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, mock_opensearch_client): From 2ecead70aafd4ec67fdfc2d9f5aae92912c9d5dd Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 12:15:43 -0600 Subject: [PATCH 069/137] Search for all matches within a single query also removed 'dateOfUpdate' field to avoid confusion with other fields --- .../lambdas/python/search/handlers/search.py | 28 ++++++++----------- .../tests/function/test_search_privileges.py | 16 ++++++----- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index d7cb32685..73c58df81 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -6,7 +6,7 @@ from cc_common.data_model.schema.provider.api import ( ProviderGeneralResponseSchema, SearchProvidersRequestSchema, - StatePrivilegeGeneralResponseSchema, + StatePrivilegeGeneralResponseSchema, ExportPrivilegesRequestSchema, ) from cc_common.exceptions import ( CCInvalidRequestCustomResponseException, @@ -19,7 +19,6 @@ # Default and maximum page sizes for search results MAX_PROVIDER_PAGE_SIZE = 100 -PRIVILEGE_SEARCH_PAGE_SIZE = 2000 MAX_MATCH_TOTAL_ALLOWED = 10000 # Presigned URL expiration time in seconds (1 minute) @@ -38,7 +37,6 @@ 'dateOfExpiration', 'dateOfIssuance', 'dateOfRenewal', - 'dateOfUpdate', 'familyName', 'givenName', 'middleName', @@ -177,7 +175,7 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu # Parse and validate the request body using the schema body = _parse_and_validate_export_request_body(event) - # Build the OpenSearch search body (no pagination for export) + # Build the OpenSearch search body search_body = _build_export_search_body(body) # Build the index name for this compact @@ -308,16 +306,12 @@ def _parse_and_validate_export_request_body(event: dict) -> dict: :return: Validated request body with query :raises CCInvalidRequestException: If the request body is invalid """ - import json - try: - body = json.loads(event.get('body', '{}')) - if 'query' not in body: - raise CCInvalidRequestException('Request body must contain a query') - return body - except json.JSONDecodeError as e: - logger.warning('Invalid JSON in request body', error=str(e)) - raise CCInvalidRequestException('Invalid JSON in request body') from e + schema = ExportPrivilegesRequestSchema() + return schema.loads(event.get('body', '{}')) + except ValidationError as e: + logger.warning('Invalid request body', errors=e.messages) + raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e def _get_caller_user_id(event: dict) -> str: @@ -345,7 +339,7 @@ def _build_opensearch_search_body(body: dict, size_override: int) -> dict: :raises CCInvalidRequestException: If search_after is used without sort """ search_body = { - 'query': body.get('query', {'match_all': {}}), + 'query': body['query'], } # Add pagination parameters following OpenSearch DSL @@ -376,14 +370,16 @@ def _build_export_search_body(body: dict) -> dict: """ Build the OpenSearch search body for export requests. - Export requests do not support pagination - they return all results up to MAX_MATCH_TOTAL_ALLOWED. + Export requests retrieve all matching results in a single request, up to MAX_MATCH_TOTAL_ALLOWED. + OpenSearch's default index.max_result_window is 10,000, which aligns with our limit. + If there are more results than the limit, the export will fail with a 400 error. :param body: Validated request body :return: OpenSearch search body """ return { 'query': body.get('query', {'match_all': {}}), - 'size': PRIVILEGE_SEARCH_PAGE_SIZE, + 'size': MAX_MATCH_TOTAL_ALLOWED, } diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 5c71ebc30..3a7e13ddd 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -456,10 +456,10 @@ def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, moc csv_content = csv_obj['Body'].read().decode('utf-8') lines = csv_content.strip().split('\n') - self.assertEqual(3, len(lines)) # Header + 2 data rows - self.assertIn('PRIV-KY-001', csv_content) - self.assertIn('PRIV-NE-001', csv_content) - self.assertNotIn('PRIV-CO-001', csv_content) + self.assertEqual(3, len(lines))# Header + 2 data rows + self.assertEqual('type,providerId,compact,jurisdiction,licenseType,privilegeId,status,compactEligibility,dateOfExpiration,dateOfIssuance,dateOfRenewal,familyName,givenName,middleName,suffix,licenseJurisdiction,licenseStatus,licenseStatusName,licenseNumber,npi\r', lines[0]) + self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ky,audiologist,PRIV-KY-001,active,eligible,2025-01-15,2024-01-15,2024-01-15,Doe,John,,,oh,active,,AUD-12345,1234567890\r', lines[1]) + self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ne,audiologist,PRIV-NE-001,active,eligible,2025-02-01,2024-02-01,2024-02-01,Doe,John,,,oh,active,,AUD-12345,1234567890', lines[2]) @patch('handlers.search.OpenSearchClient') def test_privilege_export_without_inner_hits_exports_all_privileges(self, mock_opensearch_client): @@ -592,9 +592,11 @@ def test_privilege_export_without_inner_hits_exports_all_privileges(self, mock_o lines = csv_content.strip().split('\n') self.assertEqual(4, len(lines)) # Header + 3 data rows - self.assertIn('PRIV-KY-001', csv_content) - self.assertIn('PRIV-NE-001', csv_content) - self.assertIn('PRIV-CO-001', csv_content) + self.assertEqual('type,providerId,compact,jurisdiction,licenseType,privilegeId,status,compactEligibility,dateOfExpiration,dateOfIssuance,dateOfRenewal,familyName,givenName,middleName,suffix,licenseJurisdiction,licenseStatus,licenseStatusName,licenseNumber,npi\r', lines[0]) + self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ky,audiologist,PRIV-KY-001,active,eligible,2025-01-15,2024-01-15,2024-01-15,Doe,John,,,oh,active,,AUD-12345,1234567890\r', lines[1]) + self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ne,audiologist,PRIV-NE-001,active,eligible,2025-02-01,2024-02-01,2024-02-01,Doe,John,,,oh,active,,AUD-12345,1234567890\r', lines[2]) + self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,co,audiologist,PRIV-CO-001,inactive,eligible,2025-03-01,2024-03-01,2024-03-01,Doe,John,,,oh,active,,AUD-12345,1234567890', lines[3]) + def test_unsupported_route_returns_400(self): """Test that unsupported routes return a 400 error.""" From 87a323e28f43eb8f97498b72ea4e317047ad4c9d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 12:18:24 -0600 Subject: [PATCH 070/137] Add export request schema --- .../cc_common/data_model/schema/provider/api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index 043477ba3..316c40cb8 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -480,3 +480,18 @@ class SearchProvidersRequestSchema(CCRequestSchema): # This should be the 'sort' values from the last hit of the previous page # Example: ["provider-uuid-123", "2024-01-15T10:30:00Z"] search_after = Raw(required=False, allow_none=False) + + +class ExportPrivilegesRequestSchema(CCRequestSchema): + """ + Schema for Exporting list of privileges into CSV file. + + This schema is used to validate incoming requests to the advanced search providers API endpoint. + It accepts an OpenSearch DSL query body for flexible querying of the provider index. + + Serialization direction: + API -> load() -> Python + """ + + # The OpenSearch query body - we use Raw to allow the full flexibility of OpenSearch queries + query = Raw(required=True, allow_none=False) From dd8f133a2400397c758b17c95ccbdcf90f617d10 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 12:34:28 -0600 Subject: [PATCH 071/137] formatting --- .../lambdas/python/search/handlers/search.py | 4 +- .../tests/function/test_search_privileges.py | 38 ++++++++++++++----- .../search_persistent_stack/__init__.py | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 73c58df81..772ce3339 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -4,9 +4,10 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger from cc_common.data_model.schema.provider.api import ( + ExportPrivilegesRequestSchema, ProviderGeneralResponseSchema, SearchProvidersRequestSchema, - StatePrivilegeGeneralResponseSchema, ExportPrivilegesRequestSchema, + StatePrivilegeGeneralResponseSchema, ) from cc_common.exceptions import ( CCInvalidRequestCustomResponseException, @@ -48,6 +49,7 @@ 'npi', ] + # TODO - add auth wrapper to check for readGeneral scope after testing @api_handler def search_api_handler(event: dict, context: LambdaContext): diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 3a7e13ddd..99aacb2b3 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -456,10 +456,19 @@ def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, moc csv_content = csv_obj['Body'].read().decode('utf-8') lines = csv_content.strip().split('\n') - self.assertEqual(3, len(lines))# Header + 2 data rows - self.assertEqual('type,providerId,compact,jurisdiction,licenseType,privilegeId,status,compactEligibility,dateOfExpiration,dateOfIssuance,dateOfRenewal,familyName,givenName,middleName,suffix,licenseJurisdiction,licenseStatus,licenseStatusName,licenseNumber,npi\r', lines[0]) - self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ky,audiologist,PRIV-KY-001,active,eligible,2025-01-15,2024-01-15,2024-01-15,Doe,John,,,oh,active,,AUD-12345,1234567890\r', lines[1]) - self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ne,audiologist,PRIV-NE-001,active,eligible,2025-02-01,2024-02-01,2024-02-01,Doe,John,,,oh,active,,AUD-12345,1234567890', lines[2]) + self.assertEqual(3, len(lines)) # Header + 2 data rows + self.assertEqual( + 'type,providerId,compact,jurisdiction,licenseType,privilegeId,status,compactEligibility,dateOfExpiration,dateOfIssuance,dateOfRenewal,familyName,givenName,middleName,suffix,licenseJurisdiction,licenseStatus,licenseStatusName,licenseNumber,npi\r', + lines[0], + ) + self.assertEqual( + 'statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ky,audiologist,PRIV-KY-001,active,eligible,2025-01-15,2024-01-15,2024-01-15,Doe,John,,,oh,active,,AUD-12345,1234567890\r', + lines[1], + ) + self.assertEqual( + 'statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ne,audiologist,PRIV-NE-001,active,eligible,2025-02-01,2024-02-01,2024-02-01,Doe,John,,,oh,active,,AUD-12345,1234567890', + lines[2], + ) @patch('handlers.search.OpenSearchClient') def test_privilege_export_without_inner_hits_exports_all_privileges(self, mock_opensearch_client): @@ -592,11 +601,22 @@ def test_privilege_export_without_inner_hits_exports_all_privileges(self, mock_o lines = csv_content.strip().split('\n') self.assertEqual(4, len(lines)) # Header + 3 data rows - self.assertEqual('type,providerId,compact,jurisdiction,licenseType,privilegeId,status,compactEligibility,dateOfExpiration,dateOfIssuance,dateOfRenewal,familyName,givenName,middleName,suffix,licenseJurisdiction,licenseStatus,licenseStatusName,licenseNumber,npi\r', lines[0]) - self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ky,audiologist,PRIV-KY-001,active,eligible,2025-01-15,2024-01-15,2024-01-15,Doe,John,,,oh,active,,AUD-12345,1234567890\r', lines[1]) - self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ne,audiologist,PRIV-NE-001,active,eligible,2025-02-01,2024-02-01,2024-02-01,Doe,John,,,oh,active,,AUD-12345,1234567890\r', lines[2]) - self.assertEqual('statePrivilege,00000000-0000-0000-0000-000000000001,aslp,co,audiologist,PRIV-CO-001,inactive,eligible,2025-03-01,2024-03-01,2024-03-01,Doe,John,,,oh,active,,AUD-12345,1234567890', lines[3]) - + self.assertEqual( + 'type,providerId,compact,jurisdiction,licenseType,privilegeId,status,compactEligibility,dateOfExpiration,dateOfIssuance,dateOfRenewal,familyName,givenName,middleName,suffix,licenseJurisdiction,licenseStatus,licenseStatusName,licenseNumber,npi\r', + lines[0], + ) + self.assertEqual( + 'statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ky,audiologist,PRIV-KY-001,active,eligible,2025-01-15,2024-01-15,2024-01-15,Doe,John,,,oh,active,,AUD-12345,1234567890\r', + lines[1], + ) + self.assertEqual( + 'statePrivilege,00000000-0000-0000-0000-000000000001,aslp,ne,audiologist,PRIV-NE-001,active,eligible,2025-02-01,2024-02-01,2024-02-01,Doe,John,,,oh,active,,AUD-12345,1234567890\r', + lines[2], + ) + self.assertEqual( + 'statePrivilege,00000000-0000-0000-0000-000000000001,aslp,co,audiologist,PRIV-CO-001,inactive,eligible,2025-03-01,2024-03-01,2024-03-01,Doe,John,,,oh,active,,AUD-12345,1234567890', + lines[3], + ) def test_unsupported_route_returns_400(self): """Test that unsupported routes return a 400 error.""" diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 62be3fc05..efc9118e4 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -29,7 +29,7 @@ class SearchPersistentStack(AppStack): on occasion requiring AWS support intervention (every time we attempted to update the engine version during development, the deployment never completed). If you intend to update any field that will require a blue/green deployment as described here: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html - Note that worse case scenario, you may have to delete the entire stack, re-deploy it, and re-index all the data from + Note that worst case scenario, you may have to delete the entire stack, re-deploy it, and re-index all the data from the provider table. In light of this, DO NOT place any resources in this stack that should never be deleted. """ From 687a825944d524eb8f016d7e4e154116ae1df9ef Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 12:50:03 -0600 Subject: [PATCH 072/137] Add runtime auth check for search endpoints --- .../lambdas/python/search/handlers/search.py | 13 +++-- .../tests/function/test_search_privileges.py | 49 +++++++++---------- .../tests/function/test_search_providers.py | 21 +++++++- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 772ce3339..8c2191e2f 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -3,6 +3,7 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger +from cc_common.data_model.schema.common import CCPermissionsAction from cc_common.data_model.schema.provider.api import ( ExportPrivilegesRequestSchema, ProviderGeneralResponseSchema, @@ -10,11 +11,12 @@ StatePrivilegeGeneralResponseSchema, ) from cc_common.exceptions import ( + CCInternalException, CCInvalidRequestCustomResponseException, CCInvalidRequestException, CCNotFoundException, ) -from cc_common.utils import api_handler +from cc_common.utils import api_handler, authorize_compact_level_only_action from marshmallow import ValidationError from opensearch_client import OpenSearchClient @@ -50,8 +52,8 @@ ] -# TODO - add auth wrapper to check for readGeneral scope after testing @api_handler +@authorize_compact_level_only_action(action=CCPermissionsAction.READ_GENERAL) def search_api_handler(event: dict, context: LambdaContext): """ Main entry point for search API. @@ -327,9 +329,10 @@ def _get_caller_user_id(event: dict) -> str: try: return event['requestContext']['authorizer']['claims']['sub'] except (KeyError, TypeError) as e: - logger.warning('Could not extract user id from event', error=str(e)) - # TODO - remove this after testing and raise errors - return 'anonymous' + # the api auth wrapper should have detected this earlier, so if get here there is an issue with the + # setup. Raise an internal exception + logger.error('Could not extract user id from event', error=str(e)) + raise CCInternalException('Could not determine caller id for privilege report export') from e def _build_opensearch_search_body(body: dict, size_override: int) -> dict: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 99aacb2b3..d8d3e2769 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -13,10 +13,16 @@ class TestExportPrivileges(TstFunction): def setUp(self): super().setUp() - def _create_api_event(self, compact: str, body: dict = None) -> dict: + def _create_api_event( + self, + compact: str, + body: dict = None, + resource_override: str = None, + scopes_override: str = None, + ) -> dict: """Create a standard API Gateway event for export_privileges.""" return { - 'resource': '/v1/compacts/{compact}/privileges/export', + 'resource': '/v1/compacts/{compact}/privileges/export' if not resource_override else resource_override, 'path': f'/v1/compacts/{compact}/privileges/export', 'httpMethod': 'POST', 'headers': { @@ -36,6 +42,7 @@ def _create_api_event(self, compact: str, body: dict = None) -> dict: 'claims': { 'sub': 'test-user-id', 'cognito:username': 'test-user', + 'scope': f'openid email {compact}/readGeneral' if not scopes_override else scopes_override, } }, }, @@ -623,33 +630,23 @@ def test_unsupported_route_returns_400(self): from handlers.search import search_api_handler # Create event with unsupported route - event = { - 'resource': '/v1/compacts/{compact}/unknown/search', - 'path': '/v1/compacts/aslp/unknown/search', - 'httpMethod': 'POST', - 'headers': { - 'Content-Type': 'application/json', - 'origin': 'https://example.org', - }, - 'multiValueHeaders': {}, - 'queryStringParameters': None, - 'pathParameters': {'compact': 'aslp'}, - 'requestContext': { - 'resourcePath': '/v1/compacts/{compact}/unknown/search', - 'httpMethod': 'POST', - 'authorizer': { - 'claims': { - 'sub': 'test-user-id', - 'cognito:username': 'test-user', - } - }, - }, - 'body': json.dumps({'query': {'match_all': {}}}), - 'isBase64Encoded': False, - } + event = self._create_api_event(compact='aslp', resource_override='/v1/compacts/aslp/unknown/search') response = search_api_handler(event, self.mock_context) self.assertEqual(400, response['statusCode']) body = json.loads(response['body']) self.assertIn('Unsupported method or resource', body['message']) + + def test_missing_scopes_returns_403(self): + """Test that unsupported routes return a 400 error.""" + from handlers.search import search_api_handler + + # Create event with unsupported route + event = self._create_api_event(compact='aslp', scopes_override='openid email') + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(403, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Access denied', body['message']) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 8b75c2ba7..ddea620f0 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -13,7 +13,12 @@ class TestSearchProviders(TstFunction): def setUp(self): super().setUp() - def _create_api_event(self, compact: str, body: dict = None) -> dict: + def _create_api_event( + self, + compact: str, + body: dict = None, + scopes_override: str = None, + ) -> dict: """Create a standard API Gateway event for search_providers.""" return { 'resource': '/v1/compacts/{compact}/providers/search', @@ -36,6 +41,7 @@ def _create_api_event(self, compact: str, body: dict = None) -> dict: 'claims': { 'sub': 'test-user-id', 'cognito:username': 'test-user', + 'scope': f'openid email {compact}/readGeneral' if not scopes_override else scopes_override, } }, }, @@ -326,3 +332,16 @@ def test_search_uses_correct_index_for_compact(self, mock_opensearch_client): call_args = mock_client_instance.search.call_args self.assertEqual(f'compact_{compact}_providers', call_args.kwargs['index_name']) + + def test_missing_scopes_returns_403(self): + """Test that unsupported routes return a 400 error.""" + from handlers.search import search_api_handler + + # Create event with unsupported route + event = self._create_api_event(compact='aslp', scopes_override='openid email') + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(403, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Access denied', body['message']) From 02a1acec358010db36b9fe5f240e294c6258c4d2 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 12:54:15 -0600 Subject: [PATCH 073/137] set endpoints to member variables --- .../stacks/search_api_stack/v1_api/privilege_search.py | 2 +- .../stacks/search_api_stack/v1_api/provider_search.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py index 31f120edb..258cf2396 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py @@ -43,7 +43,7 @@ def _add_export_privileges( # Get the search handler from the search persistent stack (same handler as provider search) handler = search_persistent_stack.search_handler.handler - privilege_search = export_resource.add_method( + self.privilege_search_export_endpoint = export_resource.add_method( 'POST', request_validator=self.api.parameter_body_validator, request_models={'application/json': self.api_model.search_privileges_request_model}, diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py index 7e08dcb43..adba15aaf 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py @@ -46,7 +46,7 @@ def _add_search_providers( # Get the search providers handler from the search persistent stack handler = search_persistent_stack.search_handler.handler - search_resource.add_method( + self.provider_search_endpoint = search_resource.add_method( 'POST', request_validator=self.api.parameter_body_validator, request_models={'application/json': self.api_model.search_providers_request_model}, From 05cd63207d1260ca47d68b29122fbf0024c6fa2d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 12:59:38 -0600 Subject: [PATCH 074/137] fix test comments --- .../python/search/tests/function/test_search_privileges.py | 2 +- .../python/search/tests/function/test_search_providers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index d8d3e2769..4a7f9e503 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -639,7 +639,7 @@ def test_unsupported_route_returns_400(self): self.assertIn('Unsupported method or resource', body['message']) def test_missing_scopes_returns_403(self): - """Test that unsupported routes return a 400 error.""" + """Test that missing auth scope returns a 403 error.""" from handlers.search import search_api_handler # Create event with unsupported route diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index ddea620f0..a854e98d4 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -334,7 +334,7 @@ def test_search_uses_correct_index_for_compact(self, mock_opensearch_client): self.assertEqual(f'compact_{compact}_providers', call_args.kwargs['index_name']) def test_missing_scopes_returns_403(self): - """Test that unsupported routes return a 400 error.""" + """Test that missing auth scope returns a 403 error.""" from handlers.search import search_api_handler # Create event with unsupported route From 9e300b5c02bab15154fb798a4e3197fa2f49d409 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 9 Dec 2025 13:08:10 -0600 Subject: [PATCH 075/137] Increase memory size of search handler --- .../stacks/search_persistent_stack/search_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_handler.py b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py index 0a03c6f8e..d2872e553 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/search_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py @@ -65,7 +65,9 @@ def __init__( **stack.common_env_vars, }, timeout=Duration.seconds(29), - memory_size=256, + # memory slightly larger to manage pulling down privilege reports for CSV export + # and to improve performance of search in general + memory_size=2048, vpc=vpc_stack.vpc, vpc_subnets=vpc_subnets, security_groups=[vpc_stack.lambda_security_group], From da5c5236581317211f7dc85ea23e03c36b67b705 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 5 Dec 2025 22:50:25 -0600 Subject: [PATCH 076/137] Add DynamoDB stream ingest handler --- .../handlers/index_provider_documents.py | 0 .../handlers/populate_provider_documents.py | 41 +- .../search/handlers/provider_update_ingest.py | 174 +++++ .../function/test_provider_update_ingest.py | 609 ++++++++++++++++++ .../lambdas/python/search/utils.py | 41 ++ .../stacks/persistent_stack/provider_table.py | 2 + .../search_persistent_stack/__init__.py | 15 + .../provider_update_ingest_handler.py | 170 +++++ 8 files changed, 1022 insertions(+), 30 deletions(-) delete mode 100644 backend/compact-connect/lambdas/python/search/handlers/index_provider_documents.py create mode 100644 backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py create mode 100644 backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py create mode 100644 backend/compact-connect/lambdas/python/search/utils.py create mode 100644 backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py diff --git a/backend/compact-connect/lambdas/python/search/handlers/index_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/index_provider_documents.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index e74cb236a..6208faca1 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -20,15 +20,12 @@ } """ -import json - from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger -from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema -from cc_common.exceptions import CCInternalException, CCNotFoundException -from cc_common.utils import ResponseEncoder +from cc_common.exceptions import CCInternalException from marshmallow import ValidationError from opensearch_client import OpenSearchClient +from utils import generate_provider_opensearch_document # Batch size for DynamoDB pagination DYNAMODB_PAGE_SIZE = 1000 @@ -206,32 +203,16 @@ def populate_provider_documents(event: dict, context: LambdaContext): continue try: - # Get complete provider records - provider_user_records = data_client.get_provider_user_records( - compact=compact, - provider_id=provider_id, - consistent_read=True, - ) - - # Generate API response object with all nested records - api_response = provider_user_records.generate_api_response_object() - - # Sanitize using ProviderGeneralResponseSchema - schema = ProviderGeneralResponseSchema() - sanitized_document = schema.load(api_response) - - # run the full provider document through our ResponseEncoder to convert sets - # to lists (e.g., privilegeJurisdictions) and datetime objects to strings for JSON serialization - serializable_document = json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) - documents_to_index.append(serializable_document) - - except CCNotFoundException: - logger.warning('Provider not found when fetching records', provider_id=provider_id, compact=compact) - compact_stats['providers_failed'] += 1 - continue - except ValidationError as e: + # Use the shared utility to process the provider + serializable_document = generate_provider_opensearch_document(data_client, compact, provider_id) + if serializable_document: + documents_to_index.append(serializable_document) + else: + compact_stats['providers_failed'] += 1 + + except ValidationError as e: # noqa: BLE001 logger.warning( - 'Failed to sanitize provider record', + 'Failed to process provider record', provider_id=provider_id, compact=compact, errors=e.messages, diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py new file mode 100644 index 000000000..740fefdb3 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -0,0 +1,174 @@ +""" +Lambda handler to process DynamoDB stream events and index provider documents into OpenSearch. + +This Lambda is triggered by DynamoDB streams from the provider table. It processes +events in batches, deduplicates provider IDs by compact, and bulk indexes the +sanitized provider documents into the appropriate OpenSearch indices. + +The handler supports partial batch failures using the reportBatchItemFailures +response type, allowing successful records to be processed while failed records +are sent to the dead letter queue. +""" + +from aws_lambda_powertools.utilities.typing import LambdaContext +from cc_common.config import config, logger +from cc_common.exceptions import CCInternalException, CCNotFoundException +from marshmallow import ValidationError +from opensearch_client import OpenSearchClient +from utils import generate_provider_opensearch_document + + +def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # noqa: ARG001 + """ + Process DynamoDB stream events and index provider documents into OpenSearch. + + This function: + 1. Creates a set for each compact to deduplicate provider IDs + 2. Extracts compact and providerId from each stream record (old or new image) + 3. Processes each unique provider by compact using the shared utility + 4. Bulk indexes the documents into the appropriate OpenSearch index + + :param event: DynamoDB stream event containing records + :param context: Lambda context + :return: Response with batch item failures for partial success handling + """ + records = event.get('Records', []) + + if not records: + logger.info('No records to process') + return {'batchItemFailures': []} + + logger.info('Processing DynamoDB stream batch', record_count=len(records)) + + # Create a set for each compact to deduplicate provider IDs + providers_by_compact: dict[str, set[str]] = {compact: set() for compact in config.compacts} + + # Track which sequence numbers correspond to which compact/provider for failure reporting + record_mapping: dict[str, tuple[str, str]] = {} # sequence_number -> (compact, provider_id) + + # Extract compact and providerId from each record + for record in records: + sequence_number = record.get('dynamodb', {}).get('SequenceNumber') + + # Try to get the data from NewImage first, fall back to OldImage for deletes + image = record.get('dynamodb', {}).get('NewImage') or record.get('dynamodb', {}).get('OldImage') + + if not image: + logger.warning('Record has no image data', record=record) + continue + + # Extract compact and providerId from the DynamoDB image + # The format is {'S': 'value'} for string attributes + compact = _extract_string_value(image.get('compact')) + provider_id = _extract_string_value(image.get('providerId')) + record_type = _extract_string_value(image.get('type')) + + if not compact or not provider_id: + logger.error( + 'Record missing required fields', + record_type=record_type, + sequence_number=sequence_number, + ) + continue + + # Add to the appropriate compact's set (deduplication happens automatically) + if compact in providers_by_compact: + providers_by_compact[compact].add(provider_id) + record_mapping[sequence_number] = (compact, provider_id) + else: + logger.warning('Unknown compact in record', compact=compact, provider_id=provider_id) + + # Process providers and bulk index by compact + opensearch_client = OpenSearchClient() + batch_item_failures = [] + failed_providers: set[tuple[str, str]] = set() # (compact, provider_id) pairs that failed + + for compact, provider_ids in providers_by_compact.items(): + index_name = f'compact_{compact}_providers' + logger.info('Processing providers for compact', compact=compact, provider_count=len(provider_ids)) + + # Use the shared utility to process providers + data_client = config.data_client + documents = [] + + for provider_id in provider_ids: + try: + document = generate_provider_opensearch_document(data_client, compact, provider_id) + documents.append(document) + except (CCNotFoundException, ValidationError) as e: + logger.warning( + 'Failed to process provider for indexing', + provider_id=provider_id, + compact=compact, + error=str(e), + ) + failed_providers.add((compact, provider_id)) + + if failed_providers: + logger.warning( + 'Some providers failed processing', + compact=compact, + failed_count=len(failed_providers), + successful_count=len(documents), + ) + + # Bulk index the documents + if documents: + try: + response = opensearch_client.bulk_index(index_name=index_name, documents=documents) + + # Check for individual document failures + if response.get('errors'): + for item in response.get('items', []): + index_result = item.get('index', {}) + if index_result.get('error'): + doc_id = index_result.get('_id') + logger.error( + 'Document indexing failed', + document_id=doc_id, + error=index_result.get('error'), + ) + failed_providers.add((compact, doc_id)) + + logger.info( + 'Bulk indexed documents', + index_name=index_name, + document_count=len(documents), + had_errors=response.get('errors', False), + ) + except CCInternalException as e: + # All documents for this compact failed to index + logger.error( + 'Failed to bulk index documents after retries', + index_name=index_name, + document_count=len(documents), + error=str(e), + ) + # Mark all providers in this compact as failed + for provider_id in provider_ids: + failed_providers.add((compact, provider_id)) + + # Build batch item failures response for failed providers + # Map back from failed providers to their sequence numbers + for sequence_number, (compact, provider_id) in record_mapping.items(): + if (compact, provider_id) in failed_providers: + batch_item_failures.append({'itemIdentifier': sequence_number}) + + if batch_item_failures: + logger.warning('Reporting batch item failures', failure_count=len(batch_item_failures)) + + return {'batchItemFailures': batch_item_failures} + + +def _extract_string_value(dynamo_attribute: dict | None) -> str | None: + """ + Extract a string value from a DynamoDB attribute. + + DynamoDB stream records use the format {'S': 'value'} for string attributes. + + :param dynamo_attribute: The DynamoDB attribute dict + :return: The string value, or None if not present + """ + if dynamo_attribute is None: + return None + return dynamo_attribute.get('S') diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py new file mode 100644 index 000000000..41285de7b --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -0,0 +1,609 @@ +from unittest.mock import MagicMock, Mock, patch + +from common_test.test_constants import ( + DEFAULT_LICENSE_EXPIRATION_DATE, + DEFAULT_LICENSE_ISSUANCE_DATE, + DEFAULT_LICENSE_RENEWAL_DATE, + DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, + DEFAULT_PROVIDER_UPDATE_DATETIME, + DEFAULT_REGISTERED_EMAIL_ADDRESS, +) +from moto import mock_aws + +from . import TstFunction + +MOCK_ASLP_PROVIDER_ID = '00000000-0000-0000-0000-000000000001' +MOCK_OCTP_PROVIDER_ID = '00000000-0000-0000-0000-000000000002' + +TEST_LICENSE_TYPE_MAPPING = { + 'aslp': 'audiologist', + 'octp': 'occupational therapist', +} +TEST_PROVIDER_ID_MAPPING = { + 'aslp': MOCK_ASLP_PROVIDER_ID, + 'octp': MOCK_OCTP_PROVIDER_ID, +} + + +@mock_aws +class TestProviderUpdateIngest(TstFunction): + """Test suite for provider update ingest handler.""" + + def setUp(self): + super().setUp() + + def _put_test_provider_and_license_record_in_dynamodb_table(self, compact: str, provider_id: str = None): + """Helper to create test provider and license records in DynamoDB.""" + if provider_id is None: + provider_id = TEST_PROVIDER_ID_MAPPING[compact] + + self.test_data_generator.put_default_provider_record_in_provider_table( + value_overrides={ + 'compact': compact, + 'providerId': provider_id, + 'givenName': f'test{compact}GivenName', + 'familyName': f'test{compact}FamilyName', + }, + date_of_update_override=DEFAULT_PROVIDER_UPDATE_DATETIME, + ) + self.test_data_generator.put_default_license_record_in_provider_table( + value_overrides={ + 'compact': compact, + 'providerId': provider_id, + 'givenName': f'test{compact}GivenName', + 'familyName': f'test{compact}FamilyName', + 'licenseType': TEST_LICENSE_TYPE_MAPPING[compact], + }, + date_of_update_override=DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, + ) + + def _create_dynamodb_stream_record( + self, + compact: str, + provider_id: str, + sequence_number: str, + event_name: str = 'MODIFY', + include_old_image: bool = True, + ) -> dict: + """ + Create a DynamoDB stream record in the format received by Lambda. + + DynamoDB stream records contain the image data in a specific format where + each attribute is wrapped with its type indicator (e.g., {'S': 'value'} for strings). + + :param compact: The compact abbreviation + :param provider_id: The provider ID + :param sequence_number: The stream sequence number + :param event_name: The event type (INSERT, MODIFY, REMOVE) + :param include_old_image: Whether to include OldImage (False for INSERT events) + """ + image_data = { + 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, + 'sk': {'S': f'{compact}#PROVIDER'}, + 'compact': {'S': compact}, + 'providerId': {'S': provider_id}, + 'type': {'S': 'provider'}, + 'givenName': {'S': f'test{compact}GivenName'}, + 'familyName': {'S': f'test{compact}FamilyName'}, + } + + dynamodb_data = { + 'ApproximateCreationDateTime': 1234567890, + 'Keys': { + 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, + 'sk': {'S': f'{compact}#PROVIDER'}, + }, + 'NewImage': image_data, + 'SequenceNumber': sequence_number, + 'SizeBytes': 256, + 'StreamViewType': 'NEW_AND_OLD_IMAGES', + } + + # Include OldImage only if requested (MODIFY events have both, INSERT events only have NewImage) + if include_old_image: + dynamodb_data['OldImage'] = image_data + + return { + 'eventID': f'event-{sequence_number}', + 'eventName': event_name, + 'eventVersion': '1.1', + 'eventSource': 'aws:dynamodb', + 'awsRegion': 'us-east-1', + 'dynamodb': dynamodb_data, + 'eventSourceARN': 'arn:aws:dynamodb:us-east-1:123456789012:table/provider-table/stream/1234', + } + + def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_index_response: dict = None): + """Helper to configure the mock OpenSearch client.""" + if not bulk_index_response: + bulk_index_response = {'items': [], 'errors': False} + + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_index.return_value = bulk_index_response + return mock_client_instance + + def _generate_expected_document(self, compact: str, provider_id: str = None) -> dict: + """Generate the expected document that should be indexed into OpenSearch.""" + if provider_id is None: + provider_id = TEST_PROVIDER_ID_MAPPING[compact] + + return { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': DEFAULT_PROVIDER_UPDATE_DATETIME, + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'currentHomeJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'compactEligibility': 'ineligible', + 'npi': '0608337260', + 'givenName': f'test{compact}GivenName', + 'middleName': 'Gunnar', + 'familyName': f'test{compact}FamilyName', + 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, + 'compactConnectRegisteredEmailAddress': DEFAULT_REGISTERED_EMAIL_ADDRESS, + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'privilegeJurisdictions': ['ne'], + 'birthMonthDay': '06-06', + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license', + 'dateOfUpdate': DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': TEST_LICENSE_TYPE_MAPPING[compact], + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseStatus': 'inactive', + 'jurisdictionUploadedLicenseStatus': 'active', + 'compactEligibility': 'ineligible', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'npi': '0608337260', + 'licenseNumber': 'A0608337260', + 'givenName': f'test{compact}GivenName', + 'middleName': 'Gunnar', + 'familyName': f'test{compact}FamilyName', + 'dateOfIssuance': DEFAULT_LICENSE_ISSUANCE_DATE, + 'dateOfRenewal': DEFAULT_LICENSE_RENEWAL_DATE, + 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'homeAddressCity': 'Columbus', + 'homeAddressState': 'oh', + 'homeAddressPostalCode': '43004', + 'emailAddress': 'björk@example.com', + 'phoneNumber': '+13213214321', + 'adverseActions': [], + 'investigations': [], + } + ], + 'privileges': [], + 'militaryAffiliations': [], + } + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_opensearch_client_called_with_expected_parameters(self, mock_opensearch_client): + """Test that OpenSearch client is called with expected parameters when indexing a record.""" + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create provider and license records in DynamoDB + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + + # Create a DynamoDB stream event + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk_index was called once with expected parameters + self.assertEqual(1, mock_client_instance.bulk_index.call_count) + + # Verify the call arguments + call_args = mock_client_instance.bulk_index.call_args + self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) + self.assertEqual([self._generate_expected_document('aslp')], call_args.kwargs['documents']) + + # Verify no batch item failures + self.assertEqual({'batchItemFailures': []}, result) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_provider_ids_are_deduped_only_one_document_indexed(self, mock_opensearch_client): + """Test that duplicate provider IDs in the batch are deduplicated.""" + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create provider and license records in DynamoDB + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + + # Create multiple DynamoDB stream events for the SAME provider (simulating multiple updates) + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + event_name='INSERT', + ), + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12346', + event_name='MODIFY', + ), + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12347', + event_name='MODIFY', + ), + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that bulk_index was called only once despite 3 records + self.assertEqual(1, mock_client_instance.bulk_index.call_count) + + # Verify only ONE document was indexed (deduplication worked) + call_args = mock_client_instance.bulk_index.call_args + self.assertEqual(1, len(call_args.kwargs['documents'])) + self.assertEqual(MOCK_ASLP_PROVIDER_ID, call_args.kwargs['documents'][0]['providerId']) + + # Verify no batch item failures + self.assertEqual({'batchItemFailures': []}, result) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_client): + """Test that a record that fails validation is returned in batchItemFailures.""" + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + self._when_testing_mock_opensearch_client(mock_opensearch_client) + + provider = self.test_data_generator.generate_default_provider( + value_overrides={ + 'compact': 'aslp', + 'providerId': MOCK_ASLP_PROVIDER_ID, + 'givenName': 'testGivenName', + 'familyName': 'testFamilyName', + } + ) + serialized_provider = provider.serialize_to_database_record() + # put invalid compact to fail validation + serialized_provider['compact'] = 'foo' + self.config.provider_table.put_item(Item=serialized_provider) + + # Create DynamoDB stream event for the provider without complete records + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Verify that the batch item failure is returned with the sequence number + self.assertEqual(1, len(result['batchItemFailures'])) + self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opensearch_client): + """Test that a record which fails to be indexed by OpenSearch is in batchItemFailures.""" + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client to return errors for specific documents + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + + # Simulate OpenSearch returning an error for one document + mock_client_instance.bulk_index.return_value = { + 'errors': True, + 'items': [ + { + 'index': { + '_id': MOCK_ASLP_PROVIDER_ID, + '_index': 'compact_aslp_providers', + 'status': 201, + 'result': 'created', + } + }, + { + 'index': { + '_id': MOCK_OCTP_PROVIDER_ID, + '_index': 'compact_octp_providers', + 'status': 400, + 'error': { + 'type': 'mapper_parsing_exception', + 'reason': 'failed to parse field', + }, + } + }, + ], + } + + # Create provider and license records in DynamoDB for both compacts + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + self._put_test_provider_and_license_record_in_dynamodb_table('octp') + + # Create DynamoDB stream events for both providers + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ), + self._create_dynamodb_stream_record( + compact='octp', + provider_id=MOCK_OCTP_PROVIDER_ID, + sequence_number='12346', + ), + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Verify that only the failed document's sequence number is in batchItemFailures + self.assertEqual(1, len(result['batchItemFailures'])) + self.assertEqual('12346', result['batchItemFailures'][0]['itemIdentifier']) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_bulk_index_exception_returns_all_batch_item_failures(self, mock_opensearch_client): + """Test that when bulk_index raises an exception, all providers are marked as failed.""" + from cc_common.exceptions import CCInternalException + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client to raise an exception + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_index.side_effect = CCInternalException('Connection timeout after 5 retries') + + # Create provider and license records in DynamoDB for both compacts + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + self._put_test_provider_and_license_record_in_dynamodb_table('octp') + + # Create DynamoDB stream events for both providers + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ), + self._create_dynamodb_stream_record( + compact='octp', + provider_id=MOCK_OCTP_PROVIDER_ID, + sequence_number='12346', + ), + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Verify that both records were returned in batch failures + self.assertEqual(2, len(result['batchItemFailures'])) + self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) + self.assertEqual('12346', result['batchItemFailures'][1]['itemIdentifier']) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_multiple_compacts_indexed_separately(self, mock_opensearch_client): + """Test that providers from different compacts are indexed in their respective indices.""" + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create provider and license records for two different compacts + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + self._put_test_provider_and_license_record_in_dynamodb_table('octp') + + # Create DynamoDB stream events for both compacts + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ), + self._create_dynamodb_stream_record( + compact='octp', + provider_id=MOCK_OCTP_PROVIDER_ID, + sequence_number='12346', + ), + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that bulk_index was called for each compact that had providers + # Note: The handler iterates over all compacts, but only calls bulk_index if there are documents + call_args_list = mock_client_instance.bulk_index.call_args_list + + # Find the calls for aslp and octp + aslp_calls = [c for c in call_args_list if c.kwargs['index_name'] == 'compact_aslp_providers'] + octp_calls = [c for c in call_args_list if c.kwargs['index_name'] == 'compact_octp_providers'] + + self.assertEqual(1, len(aslp_calls)) + self.assertEqual(1, len(octp_calls)) + + # Verify each call has the correct document + self.assertEqual([self._generate_expected_document('aslp')], aslp_calls[0].kwargs['documents']) + self.assertEqual([self._generate_expected_document('octp')], octp_calls[0].kwargs['documents']) + + # Verify no batch item failures + self.assertEqual({'batchItemFailures': []}, result) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_empty_records_returns_empty_batch_failures(self, mock_opensearch_client): + """Test that an empty Records list returns empty batchItemFailures.""" + from handlers.provider_update_ingest import provider_update_ingest_handler + + event = {'Records': []} + + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Verify empty response + self.assertEqual({'batchItemFailures': []}, result) + + # Verify OpenSearch client was never called + mock_opensearch_client.assert_not_called() + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_insert_event_without_old_image_indexes_successfully(self, mock_opensearch_client): + """Test that INSERT events (newly created records) without OldImage are processed correctly. + + When a new record is created in DynamoDB, the stream event contains only NewImage + and no OldImage. The handler should extract the compact and providerId from NewImage + and successfully index the document. + """ + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create provider and license records in DynamoDB + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + + # Create a DynamoDB stream event for INSERT (no OldImage) + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + event_name='INSERT', + include_old_image=False, # INSERT events don't have OldImage + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk_index was called with the correct parameters + self.assertEqual(1, mock_client_instance.bulk_index.call_count) + + # Verify the call arguments + call_args = mock_client_instance.bulk_index.call_args + self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) + self.assertEqual([self._generate_expected_document('aslp')], call_args.kwargs['documents']) + + # Verify no batch item failures for INSERT event + self.assertEqual({'batchItemFailures': []}, result) + + def _create_dynamodb_stream_record_with_old_image_only( + self, compact: str, provider_id: str, sequence_number: str + ) -> dict: + """Create a DynamoDB stream record for REMOVE events (only OldImage, no NewImage).""" + image_data = { + 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, + 'sk': {'S': f'{compact}#PROVIDER'}, + 'compact': {'S': compact}, + 'providerId': {'S': provider_id}, + 'type': {'S': 'provider'}, + 'givenName': {'S': f'test{compact}GivenName'}, + 'familyName': {'S': f'test{compact}FamilyName'}, + } + + return { + 'eventID': f'event-{sequence_number}', + 'eventName': 'REMOVE', + 'eventVersion': '1.1', + 'eventSource': 'aws:dynamodb', + 'awsRegion': 'us-east-1', + 'dynamodb': { + 'ApproximateCreationDateTime': 1234567890, + 'Keys': { + 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, + 'sk': {'S': f'{compact}#PROVIDER'}, + }, + 'OldImage': image_data, # REMOVE events only have OldImage + 'SequenceNumber': sequence_number, + 'SizeBytes': 256, + 'StreamViewType': 'NEW_AND_OLD_IMAGES', + }, + 'eventSourceARN': 'arn:aws:dynamodb:us-east-1:123456789012:table/provider-table/stream/1234', + } + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opensearch_client): + """Test that REMOVE events (deleted records) with only OldImage are processed correctly. + + When a record is deleted from DynamoDB, the stream event contains only OldImage + and no NewImage. The handler should extract the compact and providerId from OldImage + and still index/update the document (to reflect the latest state of the provider). + """ + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + # Create provider and license records in DynamoDB + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + + # Create a DynamoDB stream event for REMOVE (only OldImage, no NewImage) + event = { + 'Records': [ + self._create_dynamodb_stream_record_with_old_image_only( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk_index was called with the correct parameters + self.assertEqual(1, mock_client_instance.bulk_index.call_count) + + # Verify the call arguments + call_args = mock_client_instance.bulk_index.call_args + self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) + self.assertEqual([self._generate_expected_document('aslp')], call_args.kwargs['documents']) + + # Verify no batch item failures for REMOVE event + self.assertEqual({'batchItemFailures': []}, result) diff --git a/backend/compact-connect/lambdas/python/search/utils.py b/backend/compact-connect/lambdas/python/search/utils.py new file mode 100644 index 000000000..dbf145edc --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/utils.py @@ -0,0 +1,41 @@ +""" +Utility functions for provider document processing and OpenSearch indexing. + +This module contains shared logic for processing provider records and preparing +them for OpenSearch indexing. It is used by both the populate_provider_documents +and provider_update_ingest handlers. +""" + +import json + +from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema +from cc_common.utils import ResponseEncoder + + +def generate_provider_opensearch_document(data_client, compact: str, provider_id: str) -> dict: + """ + Process a single provider and return the sanitized document ready for indexing. + + :param data_client: The data client for accessing DynamoDB + :param compact: The compact abbreviation + :param provider_id: The provider ID to process + :return: Sanitized document ready for indexing, or None if processing failed + :raises CCNotFoundException: If the provider is not found + :raises ValidationError: If the provider data fails schema validation + """ + # Get complete provider records + provider_user_records = data_client.get_provider_user_records( + compact=compact, + provider_id=provider_id, + consistent_read=True, + ) + + # Generate API response object with all nested records + api_response = provider_user_records.generate_api_response_object() + + # Sanitize using ProviderGeneralResponseSchema + schema = ProviderGeneralResponseSchema() + sanitized_document = schema.load(api_response) + + # Serialize using ResponseEncoder to convert sets to lists and datetime objects to strings + return json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) diff --git a/backend/compact-connect/stacks/persistent_stack/provider_table.py b/backend/compact-connect/stacks/persistent_stack/provider_table.py index 9ca0f49e4..2b59c458d 100644 --- a/backend/compact-connect/stacks/persistent_stack/provider_table.py +++ b/backend/compact-connect/stacks/persistent_stack/provider_table.py @@ -6,6 +6,7 @@ BillingMode, PointInTimeRecoverySpecification, ProjectionType, + StreamViewType, Table, TableEncryption, ) @@ -42,6 +43,7 @@ def __init__( deletion_protection=True if removal_policy == RemovalPolicy.RETAIN else False, partition_key=Attribute(name='pk', type=AttributeType.STRING), sort_key=Attribute(name='sk', type=AttributeType.STRING), + stream=StreamViewType.NEW_AND_OLD_IMAGES, **kwargs, ) self.provider_fam_giv_mid_index_name = 'providerFamGivMid' diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index efc9118e4..ee7365fdf 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -7,6 +7,7 @@ from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain +from stacks.search_persistent_stack.provider_update_ingest_handler import ProviderUpdateIngestHandler from stacks.search_persistent_stack.search_handler import SearchHandler from stacks.vpc_stack import VpcStack @@ -132,3 +133,17 @@ def __init__( provider_table=persistent_stack.provider_table, alarm_topic=persistent_stack.alarm_topic, ) + + # Create the provider update ingest handler for DynamoDB stream processing + # This handler processes real-time updates from the provider table stream + self.provider_update_ingest_handler = ProviderUpdateIngestHandler( + self, + construct_id='providerUpdateIngestHandler', + opensearch_domain=self.domain, + vpc_stack=vpc_stack, + vpc_subnets=self.provider_search_domain.vpc_subnets, + lambda_role=self.opensearch_ingest_lambda_role, + provider_table=persistent_stack.provider_table, + encryption_key=self.opensearch_encryption_key, + alarm_topic=persistent_stack.alarm_topic, + ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py new file mode 100644 index 000000000..118e32b08 --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -0,0 +1,170 @@ +import os + +from aws_cdk import Duration +from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Stats, TreatMissingData +from aws_cdk.aws_cloudwatch_actions import SnsAction +from aws_cdk.aws_dynamodb import ITable +from aws_cdk.aws_ec2 import SubnetSelection +from aws_cdk.aws_iam import IRole +from aws_cdk.aws_kms import IKey +from aws_cdk.aws_lambda import StartingPosition +from aws_cdk.aws_lambda_event_sources import DynamoEventSource, SqsDlq +from aws_cdk.aws_logs import RetentionDays +from aws_cdk.aws_opensearchservice import Domain +from aws_cdk.aws_sns import ITopic +from aws_cdk.aws_sqs import Queue, QueueEncryption +from cdk_nag import NagSuppressions +from common_constructs.stack import Stack +from constructs import Construct + +from common_constructs.python_function import PythonFunction +from stacks.vpc_stack import VpcStack + + +class ProviderUpdateIngestHandler(Construct): + """ + Construct for the Provider Update Ingest Lambda function. + + This construct creates the Lambda function that processes DynamoDB stream events + from the provider table and indexes the updated provider documents into OpenSearch. + + The Lambda is triggered by DynamoDB streams and processes events in batches, + deduplicating provider IDs by compact before bulk indexing into OpenSearch. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + opensearch_domain: Domain, + vpc_stack: VpcStack, + vpc_subnets: SubnetSelection, + lambda_role: IRole, + provider_table: ITable, + encryption_key: IKey, + alarm_topic: ITopic, + ): + """ + Initialize the ProviderUpdateIngestHandler construct. + + :param scope: The scope of the construct + :param construct_id: The id of the construct + :param opensearch_domain: The reference to the OpenSearch domain resource + :param vpc_stack: The VPC stack + :param vpc_subnets: The VPC subnets for Lambda deployment + :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) + :param provider_table: The DynamoDB provider table with stream enabled + :param encryption_key: The KMS encryption key for the SQS queue + :param alarm_topic: The SNS topic for alarms + """ + super().__init__(scope, construct_id) + stack = Stack.of(scope) + + # Create the dead letter queue for failed stream events + self.dlq = Queue( + self, + 'ProviderUpdateIngestDLQ', + encryption=QueueEncryption.KMS, + encryption_master_key=encryption_key, + enforce_ssl=True, + ) + + # Create Lambda function for processing provider updates from DynamoDB streams + self.handler = PythonFunction( + self, + 'ProviderUpdateIngestFunction', + description='Processes DynamoDB stream events and indexes provider documents into OpenSearch', + index=os.path.join('handlers', 'provider_update_ingest.py'), + lambda_dir='search', + handler='provider_update_ingest_handler', + role=lambda_role, + log_retention=RetentionDays.ONE_MONTH, + environment={ + 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, + 'PROVIDER_TABLE_NAME': provider_table.table_name, + **stack.common_env_vars, + }, + # Allow enough time for processing large batches + timeout=Duration.minutes(5), + memory_size=512, + vpc=vpc_stack.vpc, + vpc_subnets=vpc_subnets, + security_groups=[vpc_stack.lambda_security_group], + alarm_topic=alarm_topic, + ) + + # Add DynamoDB stream as event source + self.handler.add_event_source( + DynamoEventSource( + provider_table, + starting_position=StartingPosition.TRIM_HORIZON, + batch_size=1000, + # Setting this to 15 seconds to give downstream updates time to be batched with initial + # updates to reduce the number of provider update calls. This can be adjusted as needed + max_batching_window=Duration.seconds(15), + bisect_batch_on_error=True, + retry_attempts=3, + on_failure=SqsDlq(self.dlq), + report_batch_item_failures=True, + ) + ) + + # Grant the handler write access to the OpenSearch domain + opensearch_domain.grant_write(self.handler) + + # Grant the handler read access to the provider table for fetching full provider records + provider_table.grant_read_data(self.handler) + + # Grant the DLQ permission to use the encryption key + encryption_key.grant_encrypt_decrypt(self.handler) + + # Add alarm for Lambda errors + Alarm( + self, + 'ProviderUpdateIngestErrorAlarm', + metric=self.handler.metric_errors(statistic=Stats.SUM), + evaluation_periods=1, + threshold=1, + actions_enabled=True, + alarm_description=f'{self.handler.node.path} failed to process a DynamoDB stream batch', + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + ).add_alarm_action(SnsAction(alarm_topic)) + + # Add alarm for DLQ messages + Alarm( + self, + 'ProviderUpdateIngestDLQAlarm', + metric=self.dlq.metric_approximate_number_of_messages_visible(), + evaluation_periods=1, + threshold=1, + actions_enabled=True, + alarm_description=f'{self.dlq.node.path} has messages - provider update ingest failures', + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + ).add_alarm_action(SnsAction(alarm_topic)) + + # Add CDK Nag suppressions for the Lambda function's IAM role + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{self.handler.role.node.path}/DefaultPolicy/Resource', + [ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The grant_write method requires wildcard permissions on the OpenSearch domain to ' + 'write to indices. This is appropriate for a function that needs to index ' + 'provider documents. The DynamoDB grant_read_data also requires index permissions. ' + 'The DynamoDB stream permissions require wildcard access to stream resources.', + }, + ], + ) + + NagSuppressions.add_resource_suppressions( + self.dlq, + [ + { + 'id': 'AwsSolutions-SQS3', + 'reason': 'This queue serves as a dead letter queue for the DynamoDB stream event source.', + }, + ], + ) From 4cd78977ea6f40d426e55c7441c47fd73f24dcb8 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 09:08:55 -0600 Subject: [PATCH 077/137] PR feedback --- .../search/handlers/populate_provider_documents.py | 7 ++----- .../python/search/handlers/provider_update_ingest.py | 12 ++++++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 6208faca1..e0d1cde82 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -205,12 +205,9 @@ def populate_provider_documents(event: dict, context: LambdaContext): try: # Use the shared utility to process the provider serializable_document = generate_provider_opensearch_document(data_client, compact, provider_id) - if serializable_document: - documents_to_index.append(serializable_document) - else: - compact_stats['providers_failed'] += 1 + documents_to_index.append(serializable_document) - except ValidationError as e: # noqa: BLE001 + except ValidationError as e: logger.warning( 'Failed to process provider record', provider_id=provider_id, diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 740fefdb3..4930e484b 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -81,7 +81,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # Process providers and bulk index by compact opensearch_client = OpenSearchClient() batch_item_failures = [] - failed_providers: set[tuple[str, str]] = set() # (compact, provider_id) pairs that failed + failed_providers: dict[str, set] = {compact: set() for compact in config.compacts} for compact, provider_ids in providers_by_compact.items(): index_name = f'compact_{compact}_providers' @@ -102,13 +102,13 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: compact=compact, error=str(e), ) - failed_providers.add((compact, provider_id)) + failed_providers[compact].add(provider_id) if failed_providers: logger.warning( 'Some providers failed processing', compact=compact, - failed_count=len(failed_providers), + failed_count=len(failed_providers[compact]), successful_count=len(documents), ) @@ -128,7 +128,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: document_id=doc_id, error=index_result.get('error'), ) - failed_providers.add((compact, doc_id)) + failed_providers[compact].add(doc_id) logger.info( 'Bulk indexed documents', @@ -146,12 +146,12 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: ) # Mark all providers in this compact as failed for provider_id in provider_ids: - failed_providers.add((compact, provider_id)) + failed_providers[compact].add(provider_id) # Build batch item failures response for failed providers # Map back from failed providers to their sequence numbers for sequence_number, (compact, provider_id) in record_mapping.items(): - if (compact, provider_id) in failed_providers: + if provider_id in failed_providers[compact]: batch_item_failures.append({'itemIdentifier': sequence_number}) if batch_item_failures: From afbac7021f695416b7d4f8913e04393b37bc3316 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 09:10:05 -0600 Subject: [PATCH 078/137] correct docstring --- backend/compact-connect/lambdas/python/search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/search/utils.py b/backend/compact-connect/lambdas/python/search/utils.py index dbf145edc..0cfa699ef 100644 --- a/backend/compact-connect/lambdas/python/search/utils.py +++ b/backend/compact-connect/lambdas/python/search/utils.py @@ -19,7 +19,7 @@ def generate_provider_opensearch_document(data_client, compact: str, provider_id :param data_client: The data client for accessing DynamoDB :param compact: The compact abbreviation :param provider_id: The provider ID to process - :return: Sanitized document ready for indexing, or None if processing failed + :return: Sanitized document ready for indexing :raises CCNotFoundException: If the provider is not found :raises ValidationError: If the provider data fails schema validation """ From 9900d88e4044b936a1205eec4c92e6f5d76a4d3d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 10:51:03 -0600 Subject: [PATCH 079/137] Handle deletion of documents if all provider records are removed from DynamoDB --- .../search/handlers/provider_update_ingest.py | 53 +++++++++++- .../python/search/opensearch_client.py | 57 +++++++++++-- .../function/test_provider_update_ingest.py | 84 +++++++++++++++++++ 3 files changed, 186 insertions(+), 8 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 4930e484b..a7afe9d87 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -90,12 +90,23 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # Use the shared utility to process providers data_client = config.data_client documents = [] + providers_to_delete = [] # Provider IDs that no longer exist and need to be deleted from the index for provider_id in provider_ids: try: document = generate_provider_opensearch_document(data_client, compact, provider_id) documents.append(document) - except (CCNotFoundException, ValidationError) as e: + except CCNotFoundException as e: + # if no provider records are found, the provider needs to be deleted from the index + logger.warning( + 'No provider records found. This may occur if a license upload rollback was performed or if records' + 'were manually deleted. Will delete provider document from index.', + provider_id=provider_id, + compact=compact, + error=str(e), + ) + providers_to_delete.append(provider_id) + except ValidationError as e: logger.warning( 'Failed to process provider for indexing', provider_id=provider_id, @@ -104,7 +115,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: ) failed_providers[compact].add(provider_id) - if failed_providers: + if failed_providers[compact]: logger.warning( 'Some providers failed processing', compact=compact, @@ -148,6 +159,44 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: for provider_id in provider_ids: failed_providers[compact].add(provider_id) + # Bulk delete providers that no longer exist + if providers_to_delete: + try: + response = opensearch_client.bulk_delete(index_name=index_name, document_ids=providers_to_delete) + + # Check for individual delete failures + if response.get('errors'): + for item in response.get('items', []): + delete_result = item.get('delete', {}) + if delete_result.get('error'): + doc_id = delete_result.get('_id') + # 404 (not_found) is not an error for delete - the document was already gone + if delete_result.get('status') != 404: + logger.error( + 'Document deletion failed', + document_id=doc_id, + error=delete_result.get('error'), + ) + failed_providers[compact].add(doc_id) + + logger.info( + 'Bulk deleted documents', + index_name=index_name, + document_count=len(providers_to_delete), + had_errors=response.get('errors', False), + ) + except CCInternalException as e: + # All deletes for this compact failed + logger.error( + 'Failed to bulk delete documents after retries', + index_name=index_name, + document_count=len(providers_to_delete), + error=str(e), + ) + # Mark all providers to delete as failed + for provider_id in providers_to_delete: + failed_providers[compact].add(provider_id) + # Build batch item failures response for failed providers # Map back from failed providers to their sequence numbers for sequence_number, (compact, provider_id) in record_mapping.items(): diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index dd0573e9c..e0d285313 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -88,6 +88,34 @@ def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'pr return self._bulk_index_with_retry(actions=actions, index_name=index_name, document_count=len(documents)) + def bulk_delete(self, index_name: str, document_ids: list[str]) -> dict: + """ + Bulk delete multiple documents from the specified index. + + This method implements retry logic with exponential backoff to handle transient + connection issues (e.g., ConnectionTimeout, TransportError). If all retry attempts + fail, a CCInternalException is raised to signal the caller to handle the failure. + + :param index_name: The name of the index to delete from + :param document_ids: List of document IDs to delete + :return: The bulk response from OpenSearch + :raises CCInternalException: If all retry attempts fail due to connection issues + """ + if not document_ids: + return {'items': [], 'errors': False} + + actions = [] + for doc_id in document_ids: + # Note: We specify the index via the `index` parameter in the bulk() call below, + # not in the action metadata. This is required because the OpenSearch domain has + # `rest.action.multi.allow_explicit_index: false` which prevents specifying + # indices in the request body for security purposes. + actions.append({'delete': {'_id': doc_id}}) + + return self._bulk_operation_with_retry( + actions=actions, index_name=index_name, operation_count=len(document_ids), operation_type='delete' + ) + def _bulk_index_with_retry(self, actions: list, index_name: str, document_count: int) -> dict: """ Execute bulk index with retry logic and exponential backoff. @@ -98,6 +126,23 @@ def _bulk_index_with_retry(self, actions: list, index_name: str, document_count: :return: The bulk response from OpenSearch :raises CCInternalException: If all retry attempts fail """ + return self._bulk_operation_with_retry( + actions=actions, index_name=index_name, operation_count=document_count, operation_type='index' + ) + + def _bulk_operation_with_retry( + self, actions: list, index_name: str, operation_count: int, operation_type: str + ) -> dict: + """ + Execute bulk operation with retry logic and exponential backoff. + + :param actions: The bulk actions to execute + :param index_name: The name of the index to operate on + :param operation_count: Number of operations being performed (for logging) + :param operation_type: Type of operation ('index' or 'delete') for logging + :return: The bulk response from OpenSearch + :raises CCInternalException: If all retry attempts fail + """ last_exception = None backoff_seconds = INITIAL_BACKOFF_SECONDS @@ -108,12 +153,12 @@ def _bulk_index_with_retry(self, actions: list, index_name: str, document_count: last_exception = e if attempt < MAX_RETRY_ATTEMPTS: logger.warning( - 'Bulk index attempt failed, retrying with backoff', + f'Bulk {operation_type} attempt failed, retrying with backoff', attempt=attempt, max_attempts=MAX_RETRY_ATTEMPTS, backoff_seconds=backoff_seconds, index_name=index_name, - document_count=document_count, + operation_count=operation_count, error=str(e), ) time.sleep(backoff_seconds) @@ -121,15 +166,15 @@ def _bulk_index_with_retry(self, actions: list, index_name: str, document_count: backoff_seconds = min(backoff_seconds * 2, MAX_BACKOFF_SECONDS) else: logger.error( - 'Bulk index failed after max retry attempts', + f'Bulk {operation_type} failed after max retry attempts', attempts=MAX_RETRY_ATTEMPTS, index_name=index_name, - document_count=document_count, + operation_count=operation_count, error=str(e), ) # All retry attempts failed raise CCInternalException( - f'Failed to bulk index {document_count} documents to {index_name} after {MAX_RETRY_ATTEMPTS} attempts. ' - f'Last error: {last_exception}' + f'Failed to bulk {operation_type} {operation_count} documents to {index_name} ' + f'after {MAX_RETRY_ATTEMPTS} attempts. Last error: {last_exception}' ) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index 41285de7b..ebde9e0fa 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -607,3 +607,87 @@ def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opense # Verify no batch item failures for REMOVE event self.assertEqual({'batchItemFailures': []}, result) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_provider_deleted_from_index_when_no_records_found(self, mock_opensearch_client): + """Test that when no provider records are found (CCNotFoundException), bulk_delete is called. + + This scenario occurs when a provider is completely removed from the system, + such as during a license upload rollback. The handler should call bulk_delete + to remove the provider document from the OpenSearch index. + """ + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_index.return_value = {'items': [], 'errors': False} + mock_client_instance.bulk_delete.return_value = {'items': [], 'errors': False} + + # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted + + # Create a DynamoDB stream event for a provider that no longer exists + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + event_name='REMOVE', + include_old_image=False, + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk_index was NOT called (no documents to index) + mock_client_instance.bulk_index.assert_not_called() + + # Assert that bulk_delete WAS called with the correct parameters + self.assertEqual(1, mock_client_instance.bulk_delete.call_count) + call_args = mock_client_instance.bulk_delete.call_args + self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) + self.assertEqual([MOCK_ASLP_PROVIDER_ID], call_args.kwargs['document_ids']) + + # Verify no batch item failures (deletion is expected behavior, not a failure) + self.assertEqual({'batchItemFailures': []}, result) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_client): + """Test that when bulk_delete fails, the provider is returned in batchItemFailures.""" + from cc_common.exceptions import CCInternalException + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client - bulk_delete raises exception + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_delete.side_effect = CCInternalException('Connection timeout after 5 retries') + + # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted + + # Create a DynamoDB stream event for a provider that no longer exists + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + event_name='REMOVE', + include_old_image=False, + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Verify that the batch item failure is returned + self.assertEqual(1, len(result['batchItemFailures'])) + self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) From 72588e2c93b39a8c1cc66bdafd79483c4568a2cb Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 10:56:16 -0600 Subject: [PATCH 080/137] add test case where document has already been deleted --- .../function/test_provider_update_ingest.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index ebde9e0fa..f0ba57a7e 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -691,3 +691,62 @@ def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_cl # Verify that the batch item failure is returned self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock_opensearch_client): + """Test that when bulk_delete returns 404 (document not found), it is NOT treated as a failure. + + This scenario occurs when a provider document has already been deleted from OpenSearch + (e.g., a previous delete succeeded, or the document never existed in the index). + The 404 response should be ignored since the desired end state (document not in index) + has been achieved. + """ + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client - bulk_delete returns 404 not_found response + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + + # Simulate OpenSearch bulk delete response when document doesn't exist + mock_client_instance.bulk_delete.return_value = { + 'errors': True, # OpenSearch reports this as an "error" even though it's just not found + 'items': [ + { + 'delete': { + '_index': 'compact_aslp_providers', + '_id': MOCK_ASLP_PROVIDER_ID, + 'status': 404, + 'result': 'not_found', + 'error': { + 'type': 'document_missing_exception', + 'reason': f'[_doc][{MOCK_ASLP_PROVIDER_ID}]: document missing', + }, + } + } + ], + } + + # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted + + # Create a DynamoDB stream event for a provider that no longer exists + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + event_name='REMOVE', + include_old_image=False, + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Assert that bulk_delete was called + self.assertEqual(1, mock_client_instance.bulk_delete.call_count) + + # Verify NO batch item failures - 404 is not treated as an error + self.assertEqual({'batchItemFailures': []}, result) From d1b6acc49891b1fd3e2fcc5ce106c6af2114e527 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 10:57:50 -0600 Subject: [PATCH 081/137] clarify log markers --- .../lambdas/python/search/handlers/provider_update_ingest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index a7afe9d87..d547d0451 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -136,7 +136,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: doc_id = index_result.get('_id') logger.error( 'Document indexing failed', - document_id=doc_id, + provider_id=doc_id, error=index_result.get('error'), ) failed_providers[compact].add(doc_id) @@ -174,7 +174,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: if delete_result.get('status') != 404: logger.error( 'Document deletion failed', - document_id=doc_id, + provider_id=doc_id, error=delete_result.get('error'), ) failed_providers[compact].add(doc_id) From cb2fbcd3bcb945cbd688cd12959c49013ae48b4d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 12:17:06 -0600 Subject: [PATCH 082/137] Track ingest failures for retries --- .../lambdas/python/common/cc_common/config.py | 14 ++++ .../handlers/populate_provider_documents.py | 2 +- .../search/handlers/provider_update_ingest.py | 29 ++++--- .../lambdas/python/search/tests/__init__.py | 1 + .../python/search/tests/function/__init__.py | 14 ++++ .../function/test_provider_update_ingest.py | 77 ++++++++++++++++++- .../lambdas/python/search/utils.py | 65 +++++++++++++++- .../search_persistent_stack/__init__.py | 18 +++++ .../provider_update_ingest_handler.py | 4 + .../search_event_state_table.py | 52 +++++++++++++ 10 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py diff --git a/backend/compact-connect/lambdas/python/common/cc_common/config.py b/backend/compact-connect/lambdas/python/common/cc_common/config.py index e952271fa..3bcb6392a 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/config.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/config.py @@ -322,6 +322,20 @@ def event_state_client(self): return EventStateClient(self) + @property + def search_event_state_table_name(self): + return os.environ['SEARCH_EVENT_STATE_TABLE_NAME'] + + @property + def search_event_state_table(self): + return boto3.resource('dynamodb').Table(self.search_event_state_table_name) + + @cached_property + def search_event_state_client(self): + from search.search_event_state_client import SearchEventStateClient + + return SearchEventStateClient(self) + @cached_property def allowed_origins(self): return json.loads(os.environ['ALLOWED_ORIGINS']) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index e0d1cde82..73ab9effd 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -204,7 +204,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): try: # Use the shared utility to process the provider - serializable_document = generate_provider_opensearch_document(data_client, compact, provider_id) + serializable_document = generate_provider_opensearch_document(compact, provider_id) documents_to_index.append(serializable_document) except ValidationError as e: diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index d547d0451..ae6452cf3 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -15,7 +15,7 @@ from cc_common.exceptions import CCInternalException, CCNotFoundException from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from utils import generate_provider_opensearch_document +from utils import generate_provider_opensearch_document, record_failed_indexing def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # noqa: ARG001 @@ -87,15 +87,13 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: index_name = f'compact_{compact}_providers' logger.info('Processing providers for compact', compact=compact, provider_count=len(provider_ids)) - # Use the shared utility to process providers - data_client = config.data_client - documents = [] + documents_to_index = [] providers_to_delete = [] # Provider IDs that no longer exist and need to be deleted from the index for provider_id in provider_ids: try: - document = generate_provider_opensearch_document(data_client, compact, provider_id) - documents.append(document) + document = generate_provider_opensearch_document(compact, provider_id) + documents_to_index.append(document) except CCNotFoundException as e: # if no provider records are found, the provider needs to be deleted from the index logger.warning( @@ -117,16 +115,16 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: if failed_providers[compact]: logger.warning( - 'Some providers failed processing', + 'Some providers failed serialization', compact=compact, failed_count=len(failed_providers[compact]), - successful_count=len(documents), + successful_count=len(documents_to_index), ) # Bulk index the documents - if documents: + if documents_to_index: try: - response = opensearch_client.bulk_index(index_name=index_name, documents=documents) + response = opensearch_client.bulk_index(index_name=index_name, documents=documents_to_index) # Check for individual document failures if response.get('errors'): @@ -144,7 +142,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: logger.info( 'Bulk indexed documents', index_name=index_name, - document_count=len(documents), + document_count=len(documents_to_index), had_errors=response.get('errors', False), ) except CCInternalException as e: @@ -152,7 +150,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: logger.error( 'Failed to bulk index documents after retries', index_name=index_name, - document_count=len(documents), + document_count=len(documents_to_index), error=str(e), ) # Mark all providers in this compact as failed @@ -199,9 +197,16 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # Build batch item failures response for failed providers # Map back from failed providers to their sequence numbers + # Also record failures in the event state table for retry purposes for sequence_number, (compact, provider_id) in record_mapping.items(): if provider_id in failed_providers[compact]: batch_item_failures.append({'itemIdentifier': sequence_number}) + # Record the failure in the event state table for retry + record_failed_indexing( + compact=compact, + provider_id=provider_id, + sequence_number=sequence_number, + ) if batch_item_failures: logger.warning('Reporting batch item failures', failure_count=len(batch_item_failures)) diff --git a/backend/compact-connect/lambdas/python/search/tests/__init__.py b/backend/compact-connect/lambdas/python/search/tests/__init__.py index 53d1a14ab..dcc0127b6 100644 --- a/backend/compact-connect/lambdas/python/search/tests/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/__init__.py @@ -24,6 +24,7 @@ def setUpClass(cls): 'LICENSE_GSI_NAME': 'licenseGSI', 'OPENSEARCH_HOST_ENDPOINT': 'vpc-providersearchd-5bzuqxhpxffk-w6dkpddu.us-east-1.es.amazonaws.com', 'EXPORT_RESULTS_BUCKET_NAME': 'test-export-results-bucket', + 'SEARCH_EVENT_STATE_TABLE_NAME': 'search-event-state-table', 'JURISDICTIONS': json.dumps( [ 'al', diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py index 9044cc9c8..414868cb4 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -27,9 +27,11 @@ def setUp(self): # noqa: N801 invalid-name def build_resources(self): self.create_provider_table() self.create_export_results_bucket() + self.create_search_event_state_table() def delete_resources(self): self._provider_table.delete() + self._search_event_state_table.delete() # must delete all objects in the bucket before deleting the bucket self._bucket.objects.delete() self._bucket.delete() @@ -91,3 +93,15 @@ def create_provider_table(self): }, ], ) + + def create_search_event_state_table(self): + """Create the mock DynamoDB table for search event state tracking""" + self._search_event_state_table = boto3.resource('dynamodb').create_table( + AttributeDefinitions=[ + {'AttributeName': 'pk', 'AttributeType': 'S'}, + {'AttributeName': 'sk', 'AttributeType': 'S'}, + ], + TableName=os.environ['SEARCH_EVENT_STATE_TABLE_NAME'], + KeySchema=[{'AttributeName': 'pk', 'KeyType': 'HASH'}, {'AttributeName': 'sk', 'KeyType': 'RANGE'}], + BillingMode='PAY_PER_REQUEST', + ) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index f0ba57a7e..b8921cd5d 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import ANY, MagicMock, Mock, patch from common_test.test_constants import ( DEFAULT_LICENSE_EXPIRATION_DATE, @@ -750,3 +750,78 @@ def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock # Verify NO batch item failures - 404 is not treated as an error self.assertEqual({'batchItemFailures': []}, result) + + @patch('handlers.provider_update_ingest.OpenSearchClient') + def test_failed_indexing_recorded_in_event_state_table(self, mock_opensearch_client): + """Test that failed indexing operations are recorded in the search event state table. + + When a provider fails to index (e.g., validation error or OpenSearch error), + the handler should record the failure in the search event state table with the compact, + provider ID, and sequence number for retry purposes. + """ + from handlers.provider_update_ingest import provider_update_ingest_handler + + # Set up mock OpenSearch client - bulk_index returns error for one document + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + + # Simulate OpenSearch returning an error for one document + mock_client_instance.bulk_index.return_value = { + 'errors': True, + 'items': [ + { + 'index': { + '_id': MOCK_ASLP_PROVIDER_ID, + '_index': 'compact_aslp_providers', + 'status': 400, + 'error': { + 'type': 'mapper_parsing_exception', + 'reason': 'failed to parse field', + }, + } + } + ], + } + + # Create provider and license records in DynamoDB + self._put_test_provider_and_license_record_in_dynamodb_table('aslp') + + # Create DynamoDB stream event + event = { + 'Records': [ + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='12345', + ) + ] + } + + # Run the handler + mock_context = MagicMock() + result = provider_update_ingest_handler(event, mock_context) + + # Verify that the batch item failure is returned + self.assertEqual(1, len(result['batchItemFailures'])) + self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) + + # Verify that a record was written to the search event state table + response = self.config.search_event_state_table.get_item( + Key={ + 'pk': f'COMPACT#aslp#PROVIDER#{MOCK_ASLP_PROVIDER_ID}', + 'sk': 'SEQUENCE#12345', + } + ) + + self.assertEqual( + { + 'compact': 'aslp', + 'pk': f'COMPACT#aslp#PROVIDER#{MOCK_ASLP_PROVIDER_ID}', + 'sk': 'SEQUENCE#12345', + 'providerId': MOCK_ASLP_PROVIDER_ID, + 'sequenceNumber': '12345', + # verify that TTL is set + 'ttl': ANY, + }, + response['Item'], + ) diff --git a/backend/compact-connect/lambdas/python/search/utils.py b/backend/compact-connect/lambdas/python/search/utils.py index 0cfa699ef..49fdc4413 100644 --- a/backend/compact-connect/lambdas/python/search/utils.py +++ b/backend/compact-connect/lambdas/python/search/utils.py @@ -7,12 +7,15 @@ """ import json +import time +from datetime import timedelta +from cc_common.config import config, logger from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema from cc_common.utils import ResponseEncoder -def generate_provider_opensearch_document(data_client, compact: str, provider_id: str) -> dict: +def generate_provider_opensearch_document(compact: str, provider_id: str) -> dict: """ Process a single provider and return the sanitized document ready for indexing. @@ -24,7 +27,7 @@ def generate_provider_opensearch_document(data_client, compact: str, provider_id :raises ValidationError: If the provider data fails schema validation """ # Get complete provider records - provider_user_records = data_client.get_provider_user_records( + provider_user_records = config.data_client.get_provider_user_records( compact=compact, provider_id=provider_id, consistent_read=True, @@ -39,3 +42,61 @@ def generate_provider_opensearch_document(data_client, compact: str, provider_id # Serialize using ResponseEncoder to convert sets to lists and datetime objects to strings return json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) + + +def record_failed_indexing( + *, + compact: str, + provider_id: str, + sequence_number: str, + ttl_days: int = 7, +) -> None: + """ + Record a failed indexing operation to the search event state table. + + This method stores the compact, provider ID, and sequence number so that + developers can replay failed indexing operations. + + :param compact: The compact abbreviation (e.g., 'aslp') + :param provider_id: The provider ID that failed to index + :param sequence_number: The DynamoDB stream sequence number of the failed record + :param ttl_days: TTL in days (default 7 days) + """ + # Build partition and sort keys + # PK: COMPACT#{compact}#PROVIDER#{provider_id} - allows querying all failures for a provider + # SK: SEQUENCE#{sequence_number} - allows identifying the specific stream record + pk = f'COMPACT#{compact}#PROVIDER#{provider_id}' + sk = f'SEQUENCE#{sequence_number}' + + # Calculate TTL (Unix timestamp in seconds) + ttl = int(time.time()) + int(timedelta(days=ttl_days).total_seconds()) + + # Build item + item = { + 'pk': pk, + 'sk': sk, + 'compact': compact, + 'providerId': provider_id, + 'sequenceNumber': sequence_number, + 'ttl': ttl, + } + + # Write to table + try: + config.search_event_state_table.put_item(Item=item) + logger.info( + 'Recorded failed indexing operation', + compact=compact, + provider_id=provider_id, + sequence_number=sequence_number, + ttl_days=ttl_days, + ) + except Exception as e: # noqa: BLE001 + # Log error but don't fail the handler - this is tracking data, not critical path + logger.error( + 'Failed to record indexing failure in event state table', + compact=compact, + provider_id=provider_id, + sequence_number=sequence_number, + error=str(e), + ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index ee7365fdf..beb5cf264 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,13 +1,16 @@ +from aws_cdk import RemovalPolicy from aws_cdk.aws_iam import Role, ServicePrincipal from common_constructs.stack import AppStack from constructs import Construct +from common_constructs.constants import PROD_ENV_NAME from stacks.persistent_stack import PersistentStack from stacks.search_persistent_stack.export_results_bucket import ExportResultsBucket from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain from stacks.search_persistent_stack.provider_update_ingest_handler import ProviderUpdateIngestHandler +from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.search_persistent_stack.search_handler import SearchHandler from stacks.vpc_stack import VpcStack @@ -90,6 +93,20 @@ def __init__( self.domain = self.provider_search_domain.domain self.opensearch_encryption_key = self.provider_search_domain.encryption_key + # Determine removal policy based on environment + removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY + + # Create the search event state table for tracking failed indexing operations + self.search_event_state_table = SearchEventStateTable( + self, + 'SearchEventStateTable', + encryption_key=self.opensearch_encryption_key, + removal_policy=removal_policy, + ) + + # Grant the ingest role access to the search event state table for tracking failures + self.search_event_state_table.grant_write_data(self.opensearch_ingest_lambda_role) + # Create the export results bucket for temporary CSV files self.export_results_bucket = ExportResultsBucket( self, @@ -144,6 +161,7 @@ def __init__( vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, + search_event_state_table=self.search_event_state_table, encryption_key=self.opensearch_encryption_key, alarm_topic=persistent_stack.alarm_topic, ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 118e32b08..0c8f1f61b 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -18,6 +18,7 @@ from constructs import Construct from common_constructs.python_function import PythonFunction +from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.vpc_stack import VpcStack @@ -41,6 +42,7 @@ def __init__( vpc_subnets: SubnetSelection, lambda_role: IRole, provider_table: ITable, + search_event_state_table: SearchEventStateTable, encryption_key: IKey, alarm_topic: ITopic, ): @@ -54,6 +56,7 @@ def __init__( :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) :param provider_table: The DynamoDB provider table with stream enabled + :param search_event_state_table: The DynamoDB table for tracking failed indexing operations :param encryption_key: The KMS encryption key for the SQS queue :param alarm_topic: The SNS topic for alarms """ @@ -82,6 +85,7 @@ def __init__( environment={ 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, 'PROVIDER_TABLE_NAME': provider_table.table_name, + 'SEARCH_EVENT_STATE_TABLE_NAME': search_event_state_table.table_name, **stack.common_env_vars, }, # Allow enough time for processing large batches diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py b/backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py new file mode 100644 index 000000000..bf6b2cda6 --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py @@ -0,0 +1,52 @@ +from aws_cdk import RemovalPolicy +from aws_cdk.aws_dynamodb import ( + AttributeType, + BillingMode, + PointInTimeRecoverySpecification, + Table, + TableEncryption, +) +from aws_cdk.aws_kms import IKey +from cdk_nag import NagSuppressions +from constructs import Construct + + +class SearchEventStateTable(Table): + """ + DynamoDB table for tracking state of search update events for failure retries. + + This table is used to maintain idempotency and track failures of various indexing operations + performed when provider records are updated. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + encryption_key: IKey, + removal_policy: RemovalPolicy, + ) -> None: + super().__init__( + scope, + construct_id, + billing_mode=BillingMode.PAY_PER_REQUEST, + encryption=TableEncryption.CUSTOMER_MANAGED, + encryption_key=encryption_key, + partition_key={'name': 'pk', 'type': AttributeType.STRING}, + sort_key={'name': 'sk', 'type': AttributeType.STRING}, + point_in_time_recovery_specification=PointInTimeRecoverySpecification(point_in_time_recovery_enabled=True), + removal_policy=removal_policy, + deletion_protection=True if removal_policy == RemovalPolicy.RETAIN else False, + time_to_live_attribute='ttl', + ) + + NagSuppressions.add_resource_suppressions( + self, + suppressions=[ + { + 'id': 'HIPAA.Security-DynamoDBInBackupPlan', + 'reason': 'These records are not intended to be backed up. This table is only for temporary event ' + 'state tracking for retries and all records expire after several weeks.', + }, + ], + ) From 59428bcd5be0eb5bcf164590d2df6a8239038e2d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 12:36:38 -0600 Subject: [PATCH 083/137] write failures in batch --- .../search/handlers/provider_update_ingest.py | 21 ++++-- .../function/test_provider_update_ingest.py | 8 +- .../lambdas/python/search/utils.py | 74 ++++++++++--------- 3 files changed, 55 insertions(+), 48 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index ae6452cf3..741591223 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -15,7 +15,7 @@ from cc_common.exceptions import CCInternalException, CCNotFoundException from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from utils import generate_provider_opensearch_document, record_failed_indexing +from utils import generate_provider_opensearch_document, record_failed_indexing_batch def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # noqa: ARG001 @@ -197,16 +197,21 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # Build batch item failures response for failed providers # Map back from failed providers to their sequence numbers - # Also record failures in the event state table for retry purposes + # Also collect failures to record in the event state table for retry purposes + failures_to_record = [] for sequence_number, (compact, provider_id) in record_mapping.items(): if provider_id in failed_providers[compact]: batch_item_failures.append({'itemIdentifier': sequence_number}) - # Record the failure in the event state table for retry - record_failed_indexing( - compact=compact, - provider_id=provider_id, - sequence_number=sequence_number, - ) + # Collect failure information for batch recording + failures_to_record.append({ + 'compact': compact, + 'provider_id': provider_id, + 'sequence_number': sequence_number, + }) + + # Record all failures in the event state table using batch writer (for replays) + if failures_to_record: + record_failed_indexing_batch(failures_to_record) if batch_item_failures: logger.warning('Reporting batch item failures', failure_count=len(batch_item_failures)) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index b8921cd5d..f5e504d41 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -808,16 +808,16 @@ def test_failed_indexing_recorded_in_event_state_table(self, mock_opensearch_cli # Verify that a record was written to the search event state table response = self.config.search_event_state_table.get_item( Key={ - 'pk': f'COMPACT#aslp#PROVIDER#{MOCK_ASLP_PROVIDER_ID}', - 'sk': 'SEQUENCE#12345', + 'pk': 'COMPACT#aslp#FAILED_INGEST', + 'sk': f'PROVIDER#{MOCK_ASLP_PROVIDER_ID}#SEQUENCE#12345', } ) self.assertEqual( { 'compact': 'aslp', - 'pk': f'COMPACT#aslp#PROVIDER#{MOCK_ASLP_PROVIDER_ID}', - 'sk': 'SEQUENCE#12345', + 'pk': 'COMPACT#aslp#FAILED_INGEST', + 'sk': f'PROVIDER#{MOCK_ASLP_PROVIDER_ID}#SEQUENCE#12345', 'providerId': MOCK_ASLP_PROVIDER_ID, 'sequenceNumber': '12345', # verify that TTL is set diff --git a/backend/compact-connect/lambdas/python/search/utils.py b/backend/compact-connect/lambdas/python/search/utils.py index 49fdc4413..6fdd95541 100644 --- a/backend/compact-connect/lambdas/python/search/utils.py +++ b/backend/compact-connect/lambdas/python/search/utils.py @@ -19,7 +19,6 @@ def generate_provider_opensearch_document(compact: str, provider_id: str) -> dic """ Process a single provider and return the sanitized document ready for indexing. - :param data_client: The data client for accessing DynamoDB :param compact: The compact abbreviation :param provider_id: The provider ID to process :return: Sanitized document ready for indexing @@ -44,59 +43,62 @@ def generate_provider_opensearch_document(compact: str, provider_id: str) -> dic return json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) -def record_failed_indexing( +def record_failed_indexing_batch( + failures: list[dict[str, str]], *, - compact: str, - provider_id: str, - sequence_number: str, ttl_days: int = 7, ) -> None: """ - Record a failed indexing operation to the search event state table. + Record multiple failed indexing operations to the search event state table using batch writes. - This method stores the compact, provider ID, and sequence number so that - developers can replay failed indexing operations. + This method stores the compact, provider ID, and sequence number for each failure so that + developers can replay failed indexing operations. Uses DynamoDB batch writer for efficient + bulk writes. - :param compact: The compact abbreviation (e.g., 'aslp') - :param provider_id: The provider ID that failed to index - :param sequence_number: The DynamoDB stream sequence number of the failed record + :param failures: List of failure records, each containing 'compact', 'provider_id', and 'sequence_number' :param ttl_days: TTL in days (default 7 days) """ - # Build partition and sort keys - # PK: COMPACT#{compact}#PROVIDER#{provider_id} - allows querying all failures for a provider - # SK: SEQUENCE#{sequence_number} - allows identifying the specific stream record - pk = f'COMPACT#{compact}#PROVIDER#{provider_id}' - sk = f'SEQUENCE#{sequence_number}' + if not failures: + return # Calculate TTL (Unix timestamp in seconds) ttl = int(time.time()) + int(timedelta(days=ttl_days).total_seconds()) - # Build item - item = { - 'pk': pk, - 'sk': sk, - 'compact': compact, - 'providerId': provider_id, - 'sequenceNumber': sequence_number, - 'ttl': ttl, - } - - # Write to table + # Use batch writer for efficient bulk writes try: - config.search_event_state_table.put_item(Item=item) + with config.search_event_state_table.batch_writer() as batch: + for failure in failures: + compact = failure['compact'] + provider_id = failure['provider_id'] + sequence_number = failure['sequence_number'] + + # Build partition and sort keys + # PK: COMPACT#{compact}#FAILED_INGEST - allows querying all failures for a provider + # SK: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - allows identifying the specific stream record + pk = f'COMPACT#{compact}#FAILED_INGEST' + sk = f'PROVIDER#{provider_id}#SEQUENCE#{sequence_number}' + + # Build item + item = { + 'pk': pk, + 'sk': sk, + 'compact': compact, + 'providerId': provider_id, + 'sequenceNumber': sequence_number, + 'ttl': ttl, + } + + batch.put_item(Item=item) + logger.info( - 'Recorded failed indexing operation', - compact=compact, - provider_id=provider_id, - sequence_number=sequence_number, + 'Recorded failed indexing operations in batch', + failure_count=len(failures), ttl_days=ttl_days, ) except Exception as e: # noqa: BLE001 # Log error but don't fail the handler - this is tracking data, not critical path logger.error( - 'Failed to record indexing failure in event state table', - compact=compact, - provider_id=provider_id, - sequence_number=sequence_number, + 'Failed to record indexing failures in event state table', + failure_count=len(failures), error=str(e), ) From ba5b86181950cca6a3ba2ef735f78ca3debeeb5c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 13:17:49 -0600 Subject: [PATCH 084/137] Retry failed ingest events --- .../handlers/populate_provider_documents.py | 131 ++++++++++++++++-- .../search/handlers/provider_update_ingest.py | 15 +- .../search/search_event_state_client.py | 131 ++++++++++++++++++ .../test_populate_provider_documents.py | 81 +++++++++++ .../lambdas/python/search/utils.py | 65 +-------- .../search_persistent_stack/__init__.py | 5 +- 6 files changed, 348 insertions(+), 80 deletions(-) create mode 100644 backend/compact-connect/lambdas/python/search/search_event_state_client.py diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 73ab9effd..72c8828dc 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -22,9 +22,10 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger -from cc_common.exceptions import CCInternalException +from cc_common.exceptions import CCInternalException, CCNotFoundException from marshmallow import ValidationError from opensearch_client import OpenSearchClient +from search_event_state_client import get_failed_ingest_provider_ids from utils import generate_provider_opensearch_document # Batch size for DynamoDB pagination @@ -40,23 +41,32 @@ def populate_provider_documents(event: dict, context: LambdaContext): """ Populate OpenSearch indices with provider documents. - This function scans all providers in the provider table using the providerDateOfUpdate GSI, - retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, - and bulk indexes them into the appropriate compact-specific OpenSearch index. + This function can operate in two modes: + 1. Normal mode: Scans all providers in the provider table using the providerDateOfUpdate GSI + 2. Retry mode: Re-indexes providers that previously failed ingestion (when 'retry_ingest_failures_for_compact' is provided) + + Retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, + and bulk indexes them into the appropriate OpenSearch indices. If processing cannot complete within 12 minutes, the function returns pagination information that can be passed as input to continue processing. - :param event: Lambda event with optional pagination parameters: - - startingCompact: The compact to start/resume processing from - - startingLastKey: The DynamoDB pagination key to resume from + :param event: Lambda event with optional parameters: + - retry_ingest_failures_for_compact: If present, retry indexing for all failed providers for this compact + - startingCompact: The compact to start/resume processing from (normal mode only) + - startingLastKey: The DynamoDB pagination key to resume from (normal mode only) :param context: Lambda context :return: Summary of indexing operation, including pagination info if incomplete """ data_client = config.data_client opensearch_client = OpenSearchClient() - # Get optional pagination parameters from event for resumption + # Check if this is a retry operation for failed ingestions + retry_compact = event.get('retry_ingest_failures_for_compact') + if retry_compact: + return _retry_failed_ingest_events(retry_compact, opensearch_client) + + # Get optional pagination parameters from event for resumption (normal mode) starting_compact = event.get('startingCompact') starting_last_key = event.get('startingLastKey') @@ -367,3 +377,108 @@ def _bulk_index_documents(opensearch_client: OpenSearchClient, index_name: str, document_count=len(documents), ) return len(documents) + + +def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClient) -> dict: + """ + Retry indexing for all providers that previously failed ingestion for a specific compact. + + This function queries the search event state table for all failed ingest records, + then processes and indexes those providers. + + :param compact: The compact abbreviation to retry failed ingestions for + :param opensearch_client: The OpenSearch client + :return: Summary of indexing operation + """ + logger.info('Retrying failed ingestions for compact', compact=compact) + + # Get list of provider IDs that failed ingestion + failed_provider_ids = get_failed_ingest_provider_ids(compact) + + if not failed_provider_ids: + logger.info('No failed ingestions found for compact', compact=compact) + return { + 'total_providers_processed': 0, + 'total_providers_indexed': 0, + 'total_providers_failed': 0, + 'compacts_processed': [ + { + 'compact': compact, + 'providers_processed': 0, + 'providers_indexed': 0, + 'providers_failed': 0, + } + ], + 'completed': True, + } + + index_name = f'compact_{compact}_providers' + documents_to_index = [] + compact_stats = { + 'providers_processed': 0, + 'providers_indexed': 0, + 'providers_failed': 0, + } + + # Process each failed provider + for provider_id in failed_provider_ids: + compact_stats['providers_processed'] += 1 + + try: + # Use the shared utility to process the provider + serializable_document = generate_provider_opensearch_document(compact, provider_id) + documents_to_index.append(serializable_document) + except (CCNotFoundException, ValidationError) as e: + logger.warning( + 'Failed to process provider during retry', + provider_id=provider_id, + compact=compact, + error=str(e), + ) + compact_stats['providers_failed'] += 1 + continue + + # Bulk index when batch is full + if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: + try: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) + compact_stats['providers_indexed'] += indexed_count + documents_to_index = [] + except CCInternalException as e: + logger.error( + 'Failed to bulk index during retry', + compact=compact, + error=str(e), + ) + compact_stats['providers_failed'] += len(documents_to_index) + documents_to_index = [] + + # Index any remaining documents + if documents_to_index: + try: + indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) + compact_stats['providers_indexed'] += indexed_count + except CCInternalException as e: + logger.error('Failed to index remaining documents during retry', error=str(e)) + compact_stats['providers_failed'] += len(documents_to_index) + + logger.info( + 'Completed retrying failed ingestions', + compact=compact, + providers_processed=compact_stats['providers_processed'], + providers_indexed=compact_stats['providers_indexed'], + providers_failed=compact_stats['providers_failed'], + ) + + return { + 'total_providers_processed': compact_stats['providers_processed'], + 'total_providers_indexed': compact_stats['providers_indexed'], + 'total_providers_failed': compact_stats['providers_failed'], + 'compacts_processed': [ + { + 'compact': compact, + **compact_stats, + } + ], + 'completed': True, + } diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 741591223..9093eba0c 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -15,7 +15,8 @@ from cc_common.exceptions import CCInternalException, CCNotFoundException from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from utils import generate_provider_opensearch_document, record_failed_indexing_batch +from search_event_state_client import record_failed_indexing_batch +from utils import generate_provider_opensearch_document def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # noqa: ARG001 @@ -203,11 +204,13 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: if provider_id in failed_providers[compact]: batch_item_failures.append({'itemIdentifier': sequence_number}) # Collect failure information for batch recording - failures_to_record.append({ - 'compact': compact, - 'provider_id': provider_id, - 'sequence_number': sequence_number, - }) + failures_to_record.append( + { + 'compact': compact, + 'provider_id': provider_id, + 'sequence_number': sequence_number, + } + ) # Record all failures in the event state table using batch writer (for replays) if failures_to_record: diff --git a/backend/compact-connect/lambdas/python/search/search_event_state_client.py b/backend/compact-connect/lambdas/python/search/search_event_state_client.py new file mode 100644 index 000000000..faeda02b0 --- /dev/null +++ b/backend/compact-connect/lambdas/python/search/search_event_state_client.py @@ -0,0 +1,131 @@ +import time +from datetime import timedelta + +from boto3.dynamodb.conditions import Key +from cc_common.config import config, logger + + +def record_failed_indexing_batch( + failures: list[dict[str, str]], + *, + ttl_days: int = 7, +) -> None: + """ + Record multiple failed indexing operations to the search event state table using batch writes. + + This method stores the compact, provider ID, and sequence number for each failure so that + developers can replay failed indexing operations. Uses DynamoDB batch writer for efficient + bulk writes. + + :param failures: List of failure records, each containing 'compact', 'provider_id', and 'sequence_number' + :param ttl_days: TTL in days (default 7 days) + """ + if not failures: + return + + # Calculate TTL (Unix timestamp in seconds) + ttl = int(time.time()) + int(timedelta(days=ttl_days).total_seconds()) + + # Use batch writer for efficient bulk writes + try: + with config.search_event_state_table.batch_writer() as batch: + for failure in failures: + compact = failure['compact'] + provider_id = failure['provider_id'] + sequence_number = failure['sequence_number'] + + # Build partition and sort keys + # PK: COMPACT#{compact}#FAILED_INGEST - allows querying all failures for a provider + # SK: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - allows identifying the specific stream record + pk = f'COMPACT#{compact}#FAILED_INGEST' + sk = f'PROVIDER#{provider_id}#SEQUENCE#{sequence_number}' + + # Build item + item = { + 'pk': pk, + 'sk': sk, + 'compact': compact, + 'providerId': provider_id, + 'sequenceNumber': sequence_number, + 'ttl': ttl, + } + + batch.put_item(Item=item) + + logger.info( + 'Recorded failed indexing operations in batch', + failure_count=len(failures), + ttl_days=ttl_days, + ) + except Exception as e: # noqa: BLE001 + # Log error but don't fail the handler - this is tracking data, not critical path + logger.error( + 'Failed to record indexing failures in event state table', + failure_count=len(failures), + error=str(e), + ) + + +def get_failed_ingest_provider_ids(compact: str) -> list[str]: + """ + Query the search event state table for all failed ingest records for a compact. + + Returns a deduplicated list of provider IDs that have failed indexing operations. + This can be used to retry indexing for providers that previously failed. + + :param compact: The compact abbreviation (e.g., 'aslp') + :return: List of unique provider IDs that have failed indexing operations + """ + pk = f'COMPACT#{compact}#FAILED_INGEST' + provider_ids = set() + + try: + # Query all records with the partition key + # The SK pattern is PROVIDER#{provider_id}#SEQUENCE#{sequence_number} + # so we can extract provider IDs from the SK + response = config.search_event_state_table.query( + KeyConditionExpression=Key('pk').eq(pk), + ) + + # Extract provider IDs from the sort keys + for item in response.get('Items', []): + sk = item.get('sk', '') + # SK format: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} + if sk.startswith('PROVIDER#'): + # Extract provider ID from SK + # Format: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} + parts = sk.split('#') + if len(parts) >= 2: + provider_id = parts[1] # The provider ID is the second part + provider_ids.add(provider_id) + + # Handle pagination + while 'LastEvaluatedKey' in response: + response = config.search_event_state_table.query( + KeyConditionExpression=Key('pk').eq(pk), + ExclusiveStartKey=response['LastEvaluatedKey'], + ) + + for item in response.get('Items', []): + sk = item.get('sk', '') + if sk.startswith('PROVIDER#'): + parts = sk.split('#') + if len(parts) >= 2: + provider_id = parts[1] + provider_ids.add(provider_id) + + provider_ids_list = sorted(list(provider_ids)) + logger.info( + 'Retrieved failed ingest provider IDs', + compact=compact, + provider_count=len(provider_ids_list), + ) + return provider_ids_list + + except Exception as e: # noqa: BLE001 + logger.error( + 'Failed to query failed ingest records', + compact=compact, + error=str(e), + ) + return [] diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 29e9f61e0..87598c7a9 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -56,6 +56,28 @@ def _put_test_provider_and_license_record_in_dynamodb_table(self, compact): date_of_update_override=DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, ) + def _put_failed_ingest_record_in_search_event_state_table( + self, compact: str, provider_id: str, sequence_number: str + ): + """Put a failed ingest record in the search event state table for testing.""" + import time + from datetime import timedelta + + pk = f'COMPACT#{compact}#FAILED_INGEST' + sk = f'PROVIDER#{provider_id}#SEQUENCE#{sequence_number}' + ttl = int(time.time()) + int(timedelta(days=7).total_seconds()) + + self.config.search_event_state_table.put_item( + Item={ + 'pk': pk, + 'sk': sk, + 'compact': compact, + 'providerId': provider_id, + 'sequenceNumber': sequence_number, + 'ttl': ttl, + } + ) + def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_index_response: dict = None): if not bulk_index_response: bulk_index_response = {'items': [], 'errors': False} @@ -325,3 +347,62 @@ def test_returns_pagination_info_when_bulk_indexing_fails_after_retries(self, mo # Verify octp and coun were indexed self.assertEqual(2, second_result['total_providers_indexed']) self.assertEqual(2, mock_client_instance.bulk_index.call_count) + + @patch('handlers.populate_provider_documents.OpenSearchClient') + def test_retry_ingest_failures_for_compact_indexes_only_failed_providers(self, mock_opensearch_client): + """Test that retry_ingest_failures_for_compact only indexes providers from the search event state table. + + This test verifies: + 1. A failed ingest record is put in the search event state table + 2. A provider record exists in the provider table + 3. When 'retry_ingest_failures_for_compact' is passed, only that provider is indexed + 4. The opensearch_client is called with the correct provider document + """ + from handlers.populate_provider_documents import populate_provider_documents + + # Set up the mock opensearch client + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + + compact = 'aslp' + provider_id = MOCK_ASLP_PROVIDER_ID + sequence_number = '12345' + + # Put a failed ingest record in the search event state table + self._put_failed_ingest_record_in_search_event_state_table(compact, provider_id, sequence_number) + + # Put provider and license records in the provider table + self._put_test_provider_and_license_record_in_dynamodb_table(compact) + + # Create event with retry_ingest_failures_for_compact + event = {'retry_ingest_failures_for_compact': compact} + + # Mock context (not used in retry path, but required for handler signature) + mock_context = MagicMock() + + # Run the handler + result = populate_provider_documents(event, mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk_index was called exactly once (only for the failed provider) + self.assertEqual(1, mock_client_instance.bulk_index.call_count) + + # Verify the call was made with the correct provider document + bulk_index_calls = mock_client_instance.bulk_index.call_args_list + expected_call = self._generate_expected_call_for_document(compact) + self.assertEqual(expected_call, bulk_index_calls[0]) + + # Verify the result statistics + self.assertEqual( + { + 'compacts_processed': [ + {'compact': compact, 'providers_failed': 0, 'providers_indexed': 1, 'providers_processed': 1} + ], + 'completed': True, + 'total_providers_failed': 0, + 'total_providers_indexed': 1, + 'total_providers_processed': 1, + }, + result, + ) diff --git a/backend/compact-connect/lambdas/python/search/utils.py b/backend/compact-connect/lambdas/python/search/utils.py index 6fdd95541..04fd50aa5 100644 --- a/backend/compact-connect/lambdas/python/search/utils.py +++ b/backend/compact-connect/lambdas/python/search/utils.py @@ -7,10 +7,8 @@ """ import json -import time -from datetime import timedelta -from cc_common.config import config, logger +from cc_common.config import config from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema from cc_common.utils import ResponseEncoder @@ -41,64 +39,3 @@ def generate_provider_opensearch_document(compact: str, provider_id: str) -> dic # Serialize using ResponseEncoder to convert sets to lists and datetime objects to strings return json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) - - -def record_failed_indexing_batch( - failures: list[dict[str, str]], - *, - ttl_days: int = 7, -) -> None: - """ - Record multiple failed indexing operations to the search event state table using batch writes. - - This method stores the compact, provider ID, and sequence number for each failure so that - developers can replay failed indexing operations. Uses DynamoDB batch writer for efficient - bulk writes. - - :param failures: List of failure records, each containing 'compact', 'provider_id', and 'sequence_number' - :param ttl_days: TTL in days (default 7 days) - """ - if not failures: - return - - # Calculate TTL (Unix timestamp in seconds) - ttl = int(time.time()) + int(timedelta(days=ttl_days).total_seconds()) - - # Use batch writer for efficient bulk writes - try: - with config.search_event_state_table.batch_writer() as batch: - for failure in failures: - compact = failure['compact'] - provider_id = failure['provider_id'] - sequence_number = failure['sequence_number'] - - # Build partition and sort keys - # PK: COMPACT#{compact}#FAILED_INGEST - allows querying all failures for a provider - # SK: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - allows identifying the specific stream record - pk = f'COMPACT#{compact}#FAILED_INGEST' - sk = f'PROVIDER#{provider_id}#SEQUENCE#{sequence_number}' - - # Build item - item = { - 'pk': pk, - 'sk': sk, - 'compact': compact, - 'providerId': provider_id, - 'sequenceNumber': sequence_number, - 'ttl': ttl, - } - - batch.put_item(Item=item) - - logger.info( - 'Recorded failed indexing operations in batch', - failure_count=len(failures), - ttl_days=ttl_days, - ) - except Exception as e: # noqa: BLE001 - # Log error but don't fail the handler - this is tracking data, not critical path - logger.error( - 'Failed to record indexing failures in event state table', - failure_count=len(failures), - error=str(e), - ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index beb5cf264..308c9204e 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -104,8 +104,8 @@ def __init__( removal_policy=removal_policy, ) - # Grant the ingest role access to the search event state table for tracking failures - self.search_event_state_table.grant_write_data(self.opensearch_ingest_lambda_role) + # Grant the ingest role access to the search event state table for tracking and retrying failures + self.search_event_state_table.grant_read_write_data(self.opensearch_ingest_lambda_role) # Create the export results bucket for temporary CSV files self.export_results_bucket = ExportResultsBucket( @@ -148,6 +148,7 @@ def __init__( vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, + search_event_state_table=self.search_event_state_table, alarm_topic=persistent_stack.alarm_topic, ) From 19ba6b5ca37d2384343022bccd1cf63fcc170a82 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 13:20:25 -0600 Subject: [PATCH 085/137] Add table name env var to handler --- .../populate_provider_documents_handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py index d3c013ba4..1c95c00b5 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py @@ -12,6 +12,7 @@ from constructs import Construct from common_constructs.python_function import PythonFunction +from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.vpc_stack import VpcStack @@ -36,6 +37,7 @@ def __init__( vpc_subnets: SubnetSelection, lambda_role: IRole, provider_table: ITable, + search_event_state_table: SearchEventStateTable, alarm_topic: ITopic, ): """ @@ -48,6 +50,7 @@ def __init__( :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) :param provider_table: The DynamoDB provider table + :param search_event_state_table: The DynamoDB table for tracking failed indexing operations :param alarm_topic: The SNS topic for alarms """ super().__init__(scope, construct_id) @@ -67,6 +70,7 @@ def __init__( 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, 'PROVIDER_TABLE_NAME': provider_table.table_name, 'PROV_DATE_OF_UPDATE_INDEX_NAME': provider_table.provider_date_of_update_index_name, + 'SEARCH_EVENT_STATE_TABLE_NAME': search_event_state_table.table_name, **stack.common_env_vars, }, # Longer timeout for processing large datasets From 553ce7d625f5781b41227ec5df4ab669a912d718 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 14:11:52 -0600 Subject: [PATCH 086/137] reduce duplication --- .../handlers/populate_provider_documents.py | 77 +++++++++++-------- .../search/handlers/provider_update_ingest.py | 4 +- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 72c8828dc..c6096257c 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -43,7 +43,8 @@ def populate_provider_documents(event: dict, context: LambdaContext): This function can operate in two modes: 1. Normal mode: Scans all providers in the provider table using the providerDateOfUpdate GSI - 2. Retry mode: Re-indexes providers that previously failed ingestion (when 'retry_ingest_failures_for_compact' is provided) + 2. Retry mode: Re-indexes providers that previously failed ingestion (when 'retry_ingest_failures_for_compact' + is provided) Retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, and bulk indexes them into the appropriate OpenSearch indices. @@ -102,7 +103,6 @@ def populate_provider_documents(event: dict, context: LambdaContext): for compact_index, compact in enumerate(compacts_to_process): logger.info('Processing compact', compact=compact) - index_name = f'compact_{compact}_providers' documents_to_index = [] compact_stats = { @@ -134,19 +134,17 @@ def populate_provider_documents(event: dict, context: LambdaContext): ) # Index any remaining documents before returning - if documents_to_index: - try: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) - compact_stats['providers_indexed'] += indexed_count - except CCInternalException as e: - # Indexing failed after retries, return pagination info for manual retry - return _build_error_response( - stats, - compact_stats, - compact, - batch_start_key, - str(e), - ) + try: + _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) + except CCInternalException as e: + # Indexing failed after retries, return pagination info for manual retry + return _build_error_response( + stats, + compact_stats, + compact, + batch_start_key, + str(e), + ) # Update stats for current compact stats['total_providers_processed'] += compact_stats['providers_processed'] @@ -230,8 +228,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Bulk index when batch is full if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: try: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) - compact_stats['providers_indexed'] += indexed_count + _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) documents_to_index = [] except CCInternalException as e: # Indexing failed after retries, return pagination info for manual retry @@ -246,8 +243,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Index any remaining documents for this compact if documents_to_index: try: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) - compact_stats['providers_indexed'] += indexed_count + _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) except CCInternalException as e: # Indexing failed after retries, return pagination info for manual retry return _build_error_response( @@ -287,6 +283,23 @@ def populate_provider_documents(event: dict, context: LambdaContext): return stats +def _index_records_and_track_stats( + documents_to_index: list[dict], compact: str, opensearch_client: OpenSearchClient, compact_stats: dict +): + index_name = f'compact_{compact}_providers' + if documents_to_index: + failed_ids = _bulk_index_documents(opensearch_client, index_name, documents_to_index) + compact_stats['providers_indexed'] += len(documents_to_index) - len(failed_ids) + if failed_ids: + compact_stats['providers_failed'] += len(failed_ids) + logger.warning( + 'Some documents failed to index in batch', + compact=compact, + failed_count=len(failed_ids), + failed_document_ids=list(failed_ids), + ) + + def _build_error_response( stats: dict, compact_stats: dict, compact: str, batch_start_key: dict | None, error_message: str ) -> dict: @@ -335,48 +348,50 @@ def _build_error_response( return stats -def _bulk_index_documents(opensearch_client: OpenSearchClient, index_name: str, documents: list[dict]) -> int: +def _bulk_index_documents(opensearch_client: OpenSearchClient, index_name: str, documents: list[dict]) -> set[str]: """ Bulk index documents into OpenSearch. :param opensearch_client: The OpenSearch client :param index_name: The index to write to :param documents: List of documents to index - :return: Number of successfully indexed documents + :return: Set of failed document IDs (empty set if all documents succeeded) :raises CCInternalException: If bulk indexing fails after max retry attempts """ if not documents: - return 0 + return set() # This will raise CCInternalException if all retries fail response = opensearch_client.bulk_index(index_name=index_name, documents=documents) # Check for errors in the bulk response (individual document failures, not connection issues) if response.get('errors'): - error_count = 0 + failed_ids = set() for item in response.get('items', []): index_result = item.get('index', {}) if index_result.get('error'): - error_count += 1 + doc_id = index_result.get('_id') + failed_ids.add(doc_id) logger.warning( 'Bulk index item error', - document_id=index_result.get('_id'), + document_id=doc_id, error=index_result.get('error'), ) logger.warning( 'Bulk index completed with errors', index_name=index_name, total_documents=len(documents), - error_count=error_count, + error_count=len(failed_ids), + failed_document_ids=list(failed_ids), ) - return len(documents) - error_count + return failed_ids logger.info( 'Indexed documents', index_name=index_name, document_count=len(documents), ) - return len(documents) + return set() def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClient) -> dict: @@ -441,8 +456,7 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien # Bulk index when batch is full if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: try: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) - compact_stats['providers_indexed'] += indexed_count + _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) documents_to_index = [] except CCInternalException as e: logger.error( @@ -456,8 +470,7 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien # Index any remaining documents if documents_to_index: try: - indexed_count = _bulk_index_documents(opensearch_client, index_name, documents_to_index) - compact_stats['providers_indexed'] += indexed_count + _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) except CCInternalException as e: logger.error('Failed to index remaining documents during retry', error=str(e)) compact_stats['providers_failed'] += len(documents_to_index) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 9093eba0c..ac526a245 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -99,7 +99,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # if no provider records are found, the provider needs to be deleted from the index logger.warning( 'No provider records found. This may occur if a license upload rollback was performed or if records' - 'were manually deleted. Will delete provider document from index.', + ' were manually deleted. Will delete provider document from index.', provider_id=provider_id, compact=compact, error=str(e), @@ -118,7 +118,7 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: logger.warning( 'Some providers failed serialization', compact=compact, - failed_count=len(failed_providers[compact]), + failed_provider_ids=failed_providers[compact], successful_count=len(documents_to_index), ) From 213333c0b08fccca434a46a97019ea2df32b7295 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 14:13:21 -0600 Subject: [PATCH 087/137] remove unused var --- .../python/search/handlers/populate_provider_documents.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index c6096257c..001419940 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -427,7 +427,6 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien 'completed': True, } - index_name = f'compact_{compact}_providers' documents_to_index = [] compact_stats = { 'providers_processed': 0, From 934d2af040e5c71c73bde592bef8769747a9f993 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Wed, 10 Dec 2025 17:05:45 -0600 Subject: [PATCH 088/137] remove documents from index on retry if provider not found --- .../handlers/populate_provider_documents.py | 66 +++++++++++++++- .../search/handlers/provider_update_ingest.py | 22 ++---- .../python/search/opensearch_client.py | 26 +++++- .../test_populate_provider_documents.py | 79 ++++++++++++++++++- 4 files changed, 170 insertions(+), 23 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 001419940..32e4b76c2 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -300,6 +300,46 @@ def _index_records_and_track_stats( ) +def _delete_records_and_track_stats( + providers_to_delete: list[str], compact: str, opensearch_client: OpenSearchClient, compact_stats: dict +): + """ + Bulk delete provider documents from OpenSearch and track statistics. + + :param providers_to_delete: List of provider IDs to delete + :param compact: The compact abbreviation + :param opensearch_client: The OpenSearch client + :param compact_stats: Statistics dictionary to update (in/out) + """ + index_name = f'compact_{compact}_providers' + if not providers_to_delete: + return + + try: + failed_provider_ids = opensearch_client.bulk_delete(index_name=index_name, document_ids=providers_to_delete) + failed_deletes = len(failed_provider_ids) + + successful_deletes = len(providers_to_delete) - failed_deletes + compact_stats['providers_deleted'] += successful_deletes + compact_stats['providers_failed'] += failed_deletes + logger.info( + 'Bulk deleted documents', + index_name=index_name, + failed_provider_ids=list(failed_provider_ids), + document_count=len(providers_to_delete), + ) + except CCInternalException as e: + # All deletes for this batch failed + logger.error( + 'Failed to bulk delete documents after retries', + index_name=index_name, + failed_provider_ids=providers_to_delete, + document_count=len(providers_to_delete), + error=str(e), + ) + compact_stats['providers_failed'] += len(providers_to_delete) + + def _build_error_response( stats: dict, compact_stats: dict, compact: str, batch_start_key: dict | None, error_message: str ) -> dict: @@ -415,12 +455,14 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien return { 'total_providers_processed': 0, 'total_providers_indexed': 0, + 'total_providers_deleted': 0, 'total_providers_failed': 0, 'compacts_processed': [ { 'compact': compact, 'providers_processed': 0, 'providers_indexed': 0, + 'providers_deleted': 0, 'providers_failed': 0, } ], @@ -428,9 +470,11 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien } documents_to_index = [] + providers_to_delete = [] compact_stats = { 'providers_processed': 0, 'providers_indexed': 0, + 'providers_deleted': 0, 'providers_failed': 0, } @@ -442,7 +486,16 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien # Use the shared utility to process the provider serializable_document = generate_provider_opensearch_document(compact, provider_id) documents_to_index.append(serializable_document) - except (CCNotFoundException, ValidationError) as e: + except CCNotFoundException as e: + logger.warning( + 'No provider records found. This may occur if a license upload rollback was performed or if records' + ' were manually deleted. Will delete provider document from index.', + provider_id=provider_id, + compact=compact, + error=str(e), + ) + providers_to_delete.append(provider_id) + except ValidationError as e: logger.warning( 'Failed to process provider during retry', provider_id=provider_id, @@ -466,6 +519,11 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien compact_stats['providers_failed'] += len(documents_to_index) documents_to_index = [] + # Bulk delete when batch is full + if len(providers_to_delete) >= OPENSEARCH_BULK_SIZE: + _delete_records_and_track_stats(providers_to_delete, compact, opensearch_client, compact_stats) + providers_to_delete = [] + # Index any remaining documents if documents_to_index: try: @@ -474,17 +532,23 @@ def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClien logger.error('Failed to index remaining documents during retry', error=str(e)) compact_stats['providers_failed'] += len(documents_to_index) + # Delete any remaining providers that no longer exist + if providers_to_delete: + _delete_records_and_track_stats(providers_to_delete, compact, opensearch_client, compact_stats) + logger.info( 'Completed retrying failed ingestions', compact=compact, providers_processed=compact_stats['providers_processed'], providers_indexed=compact_stats['providers_indexed'], + providers_deleted=compact_stats['providers_deleted'], providers_failed=compact_stats['providers_failed'], ) return { 'total_providers_processed': compact_stats['providers_processed'], 'total_providers_indexed': compact_stats['providers_indexed'], + 'total_providers_deleted': compact_stats['providers_deleted'], 'total_providers_failed': compact_stats['providers_failed'], 'compacts_processed': [ { diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index ac526a245..0c41e2077 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -161,28 +161,16 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # Bulk delete providers that no longer exist if providers_to_delete: try: - response = opensearch_client.bulk_delete(index_name=index_name, document_ids=providers_to_delete) - - # Check for individual delete failures - if response.get('errors'): - for item in response.get('items', []): - delete_result = item.get('delete', {}) - if delete_result.get('error'): - doc_id = delete_result.get('_id') - # 404 (not_found) is not an error for delete - the document was already gone - if delete_result.get('status') != 404: - logger.error( - 'Document deletion failed', - provider_id=doc_id, - error=delete_result.get('error'), - ) - failed_providers[compact].add(doc_id) + failed_provider_ids = opensearch_client.bulk_delete( + index_name=index_name, document_ids=providers_to_delete + ) + failed_providers[compact].update(failed_provider_ids) logger.info( 'Bulk deleted documents', index_name=index_name, document_count=len(providers_to_delete), - had_errors=response.get('errors', False), + failed_provider_ids=list(failed_provider_ids), ) except CCInternalException as e: # All deletes for this compact failed diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index e0d285313..5b859bbb0 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -88,7 +88,7 @@ def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'pr return self._bulk_index_with_retry(actions=actions, index_name=index_name, document_count=len(documents)) - def bulk_delete(self, index_name: str, document_ids: list[str]) -> dict: + def bulk_delete(self, index_name: str, document_ids: list[str]) -> set[str]: """ Bulk delete multiple documents from the specified index. @@ -98,11 +98,12 @@ def bulk_delete(self, index_name: str, document_ids: list[str]) -> dict: :param index_name: The name of the index to delete from :param document_ids: List of document IDs to delete - :return: The bulk response from OpenSearch + :return: A list of document ids that failed to delete (if any) :raises CCInternalException: If all retry attempts fail due to connection issues """ + failed_document_ids = set() if not document_ids: - return {'items': [], 'errors': False} + return failed_document_ids actions = [] for doc_id in document_ids: @@ -112,10 +113,27 @@ def bulk_delete(self, index_name: str, document_ids: list[str]) -> dict: # indices in the request body for security purposes. actions.append({'delete': {'_id': doc_id}}) - return self._bulk_operation_with_retry( + response = self._bulk_operation_with_retry( actions=actions, index_name=index_name, operation_count=len(document_ids), operation_type='delete' ) + # Check for individual delete failures + if response.get('errors'): + for item in response.get('items', []): + delete_result = item.get('delete', {}) + if delete_result.get('error'): + doc_id = delete_result.get('_id') + # 404 (not_found) is not an error for delete - the document was already gone + if delete_result.get('status') != 404: + logger.error( + 'Document deletion failed', + provider_id=doc_id, + error=delete_result.get('error'), + ) + failed_document_ids.add(doc_id) + + return failed_document_ids + def _bulk_index_with_retry(self, actions: list, index_name: str, document_count: int) -> dict: """ Execute bulk index with retry logic and exponential backoff. diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 87598c7a9..3cf3fa5dd 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -397,12 +397,89 @@ def test_retry_ingest_failures_for_compact_indexes_only_failed_providers(self, m self.assertEqual( { 'compacts_processed': [ - {'compact': compact, 'providers_failed': 0, 'providers_indexed': 1, 'providers_processed': 1} + { + 'compact': compact, + 'providers_failed': 0, + 'providers_indexed': 1, + 'providers_deleted': 0, + 'providers_processed': 1, + } ], 'completed': True, 'total_providers_failed': 0, + 'total_providers_deleted': 0, 'total_providers_indexed': 1, 'total_providers_processed': 1, }, result, ) + + @patch('handlers.populate_provider_documents.OpenSearchClient') + def test_retry_ingest_failures_deletes_providers_when_not_found(self, mock_opensearch_client): + """Test that retry_ingest_failures_for_compact deletes providers from index when CCNotFoundException is raised. + + This test verifies: + 1. A failed ingest record is put in the search event state table + 2. NO provider records exist in the provider table (simulating deletion/rollback) + 3. When 'retry_ingest_failures_for_compact' is passed, bulk_delete is called + 4. The provider is deleted from the OpenSearch index + 5. Statistics reflect the deletion + """ + from handlers.populate_provider_documents import populate_provider_documents + + # Set up the mock opensearch client + mock_client_instance = Mock() + mock_opensearch_client.return_value = mock_client_instance + mock_client_instance.bulk_index.return_value = set() + mock_client_instance.bulk_delete.return_value = set() + + compact = 'aslp' + provider_id = MOCK_ASLP_PROVIDER_ID + sequence_number = '12345' + + # Put a failed ingest record in the search event state table + self._put_failed_ingest_record_in_search_event_state_table(compact, provider_id, sequence_number) + + # Do NOT create provider records in the provider table - this simulates the provider being deleted + + # Create event with retry_ingest_failures_for_compact + event = {'retry_ingest_failures_for_compact': compact} + + # Mock context (not used in retry path, but required for handler signature) + mock_context = MagicMock() + + # Run the handler + result = populate_provider_documents(event, mock_context) + + # Assert that the OpenSearchClient was instantiated + mock_opensearch_client.assert_called_once() + + # Assert that bulk_index was NOT called (no documents to index) + mock_client_instance.bulk_index.assert_not_called() + + # Assert that bulk_delete WAS called with the correct parameters + self.assertEqual(1, mock_client_instance.bulk_delete.call_count) + call_args = mock_client_instance.bulk_delete.call_args + self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) + self.assertEqual([MOCK_ASLP_PROVIDER_ID], call_args.kwargs['document_ids']) + + # Verify the result statistics + self.assertEqual( + { + 'compacts_processed': [ + { + 'compact': compact, + 'providers_deleted': 1, + 'providers_failed': 0, + 'providers_indexed': 0, + 'providers_processed': 1, + } + ], + 'completed': True, + 'total_providers_deleted': 1, + 'total_providers_failed': 0, + 'total_providers_indexed': 0, + 'total_providers_processed': 1, + }, + result, + ) From face0a33583855b51345d1236b887250d7cfdf68 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 09:43:50 -0600 Subject: [PATCH 089/137] Using event bridge pipe to process dynamodb stream --- .../queued_lambda_processor.py | 2 + .../lambdas/python/common/cc_common/utils.py | 48 +++ .../search/handlers/provider_update_ingest.py | 69 ++-- .../function/test_provider_update_ingest.py | 364 +++++++++--------- .../search_persistent_stack/__init__.py | 15 +- .../provider_update_ingest_handler.py | 106 +++-- .../provider_update_ingest_pipe.py | 106 +++++ 7 files changed, 427 insertions(+), 283 deletions(-) create mode 100644 backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py diff --git a/backend/compact-connect/common_constructs/queued_lambda_processor.py b/backend/compact-connect/common_constructs/queued_lambda_processor.py index d1deb4458..346a3c4fa 100644 --- a/backend/compact-connect/common_constructs/queued_lambda_processor.py +++ b/backend/compact-connect/common_constructs/queued_lambda_processor.py @@ -29,6 +29,7 @@ def __init__( encryption_key: IKey, alarm_topic: ITopic, dlq_count_alarm_threshold: int = 10, + dlq_retention_period: Duration | None = None, ): super().__init__(scope, construct_id) @@ -39,6 +40,7 @@ def __init__( encryption=QueueEncryption.KMS, encryption_master_key=encryption_key, enforce_ssl=True, + retention_period=dlq_retention_period, ) self.queue = Queue( diff --git a/backend/compact-connect/lambdas/python/common/cc_common/utils.py b/backend/compact-connect/lambdas/python/common/cc_common/utils.py index df351444e..8c6771ced 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/utils.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/utils.py @@ -468,6 +468,54 @@ def process_messages(event, context: LambdaContext): # noqa: ARG001 unused-argu return process_messages +def sqs_batch_handler(fn: Callable): + """Process a batch of messages from an SQS queue, passing all messages to the handler at once. + + This handler is similar to sqs_handler but passes ALL messages to the decorated function + at once, allowing for batch processing, deduplication, and bulk operations. The decorated + function is responsible for returning the batchItemFailures response directly. + + This handler uses batch item failure reporting: + https://docs.aws.amazon.com/lambda/latest/dg/example_serverless_SQS_Lambda_batch_item_failures_section.html + + The decorated function receives a list of records, where each record contains: + - 'messageId': The SQS message ID (used for batch item failure reporting) + - 'body': The parsed JSON body of the SQS message + + The decorated function must return: {'batchItemFailures': [{'itemIdentifier': messageId}, ...]} + """ + + @wraps(fn) + @metrics.log_metrics + @logger.inject_lambda_context + def process_messages(event, context: LambdaContext): # noqa: ARG001 unused-argument + sqs_records = event.get('Records', []) + logger.info('Starting batch processing', batch_count=len(sqs_records)) + + if not sqs_records: + logger.info('No records to process') + return {'batchItemFailures': []} + + # Parse all SQS message bodies and create records with messageId for failure tracking + records = [] + for sqs_record in sqs_records: + message_id = sqs_record['messageId'] + try: + body = json.loads(sqs_record['body']) + records.append({'messageId': message_id, 'body': body}) + except json.JSONDecodeError as e: + # If we can't parse the message body, log error but don't fail the whole batch + logger.error('Failed to parse SQS message body', message_id=message_id, exc_info=e) + # We can't process this message, but we also shouldn't retry it since it's malformed + # So we don't add it to failures - it will be deleted from the queue + + # Call the decorated function with all parsed records + # The function is responsible for returning {'batchItemFailures': [...]} + return fn(records) + + return process_messages + + def sqs_handler_with_notification_tracking(fn: Callable): """ Process messages from SQS with notification tracking capabilities. diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 0c41e2077..845172d74 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -1,27 +1,29 @@ """ -Lambda handler to process DynamoDB stream events and index provider documents into OpenSearch. +Lambda handler to process SQS messages containing DynamoDB stream events and index +provider documents into OpenSearch. -This Lambda is triggered by DynamoDB streams from the provider table. It processes -events in batches, deduplicates provider IDs by compact, and bulk indexes the -sanitized provider documents into the appropriate OpenSearch indices. +This Lambda is triggered by SQS (via EventBridge Pipe from DynamoDB streams) from +the provider table. It processes events in batches, deduplicates provider IDs by +compact, and bulk indexes the sanitized provider documents into the appropriate +OpenSearch indices. -The handler supports partial batch failures using the reportBatchItemFailures -response type, allowing successful records to be processed while failed records -are sent to the dead letter queue. +The handler uses the @sqs_batch_handler decorator which passes all SQS messages +to the handler at once, enabling batch processing and deduplication. The handler +returns batchItemFailures directly for partial success handling. """ -from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger from cc_common.exceptions import CCInternalException, CCNotFoundException +from cc_common.utils import sqs_batch_handler from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from search_event_state_client import record_failed_indexing_batch from utils import generate_provider_opensearch_document -def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: # noqa: ARG001 +@sqs_batch_handler +def provider_update_ingest_handler(records: list[dict]) -> dict: """ - Process DynamoDB stream events and index provider documents into OpenSearch. + Process DynamoDB stream events from SQS and index provider documents into OpenSearch. This function: 1. Creates a set for each compact to deduplicate provider IDs @@ -29,33 +31,32 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: 3. Processes each unique provider by compact using the shared utility 4. Bulk indexes the documents into the appropriate OpenSearch index - :param event: DynamoDB stream event containing records - :param context: Lambda context + :param records: List of SQS records, each containing 'messageId' and 'body' (DynamoDB stream record) :return: Response with batch item failures for partial success handling """ - records = event.get('Records', []) - if not records: logger.info('No records to process') return {'batchItemFailures': []} - logger.info('Processing DynamoDB stream batch', record_count=len(records)) + logger.info('Processing SQS batch with DynamoDB stream records', record_count=len(records)) # Create a set for each compact to deduplicate provider IDs providers_by_compact: dict[str, set[str]] = {compact: set() for compact in config.compacts} - # Track which sequence numbers correspond to which compact/provider for failure reporting - record_mapping: dict[str, tuple[str, str]] = {} # sequence_number -> (compact, provider_id) + # Track which message IDs correspond to which compact/provider for failure reporting + record_mapping: dict[str, tuple[str, str]] = {} # message_id -> (compact, provider_id) # Extract compact and providerId from each record for record in records: - sequence_number = record.get('dynamodb', {}).get('SequenceNumber') + message_id = record['messageId'] + # The body contains the DynamoDB stream record sent via EventBridge Pipe + stream_record = record['body'] # Try to get the data from NewImage first, fall back to OldImage for deletes - image = record.get('dynamodb', {}).get('NewImage') or record.get('dynamodb', {}).get('OldImage') + image = stream_record.get('dynamodb', {}).get('NewImage') or stream_record.get('dynamodb', {}).get('OldImage') if not image: - logger.warning('Record has no image data', record=record) + logger.error('Record has no image data', message_id=message_id, record=stream_record) continue # Extract compact and providerId from the DynamoDB image @@ -68,14 +69,14 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: logger.error( 'Record missing required fields', record_type=record_type, - sequence_number=sequence_number, + message_id=message_id, ) continue - # Add to the appropriate compact's set (deduplication happens automatically) + # Add to the appropriate compact's set to dedup provider ids if compact in providers_by_compact: providers_by_compact[compact].add(provider_id) - record_mapping[sequence_number] = (compact, provider_id) + record_mapping[message_id] = (compact, provider_id) else: logger.warning('Unknown compact in record', compact=compact, provider_id=provider_id) @@ -185,24 +186,10 @@ def provider_update_ingest_handler(event: dict, context: LambdaContext) -> dict: failed_providers[compact].add(provider_id) # Build batch item failures response for failed providers - # Map back from failed providers to their sequence numbers - # Also collect failures to record in the event state table for retry purposes - failures_to_record = [] - for sequence_number, (compact, provider_id) in record_mapping.items(): + # Map back from failed providers to their SQS message IDs + for message_id, (compact, provider_id) in record_mapping.items(): if provider_id in failed_providers[compact]: - batch_item_failures.append({'itemIdentifier': sequence_number}) - # Collect failure information for batch recording - failures_to_record.append( - { - 'compact': compact, - 'provider_id': provider_id, - 'sequence_number': sequence_number, - } - ) - - # Record all failures in the event state table using batch writer (for replays) - if failures_to_record: - record_failed_indexing_batch(failures_to_record) + batch_item_failures.append({'itemIdentifier': message_id}) if batch_item_failures: logger.warning('Reporting batch item failures', failure_count=len(batch_item_failures)) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index f5e504d41..761f3dbaf 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -1,3 +1,4 @@ +import json from unittest.mock import ANY, MagicMock, Mock, patch from common_test.test_constants import ( @@ -194,14 +195,19 @@ def test_opensearch_client_called_with_expected_parameters(self, mock_opensearch # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') - # Create a DynamoDB stream event + # Create an SQS event with DynamoDB stream record in the body event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + ) + ), + } ] } @@ -234,27 +240,42 @@ def test_provider_ids_are_deduped_only_one_document_indexed(self, mock_opensearc # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') - # Create multiple DynamoDB stream events for the SAME provider (simulating multiple updates) + # Create multiple SQS records for the SAME provider (simulating multiple updates) event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - event_name='INSERT', - ), - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12346', - event_name='MODIFY', - ), - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12347', - event_name='MODIFY', - ), + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number-1', + event_name='INSERT', + ) + ), + }, + { + 'messageId': '12346', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number-2', + event_name='MODIFY', + ) + ), + }, + { + 'messageId': '12347', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number-3', + event_name='MODIFY', + ) + ), + }, ] } @@ -294,14 +315,19 @@ def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_cli serialized_provider['compact'] = 'foo' self.config.provider_table.put_item(Item=serialized_provider) - # Create DynamoDB stream event for the provider without complete records + # Create SQS event with DynamoDB stream record in the body event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + ) + ), + } ] } @@ -309,7 +335,7 @@ def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_cli mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that the batch item failure is returned with the sequence number + # Verify that the batch item failure is returned with the message ID self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) @@ -352,19 +378,29 @@ def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opens self._put_test_provider_and_license_record_in_dynamodb_table('aslp') self._put_test_provider_and_license_record_in_dynamodb_table('octp') - # Create DynamoDB stream events for both providers + # Create SQS events with DynamoDB stream records in the body for both providers event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ), - self._create_dynamodb_stream_record( - compact='octp', - provider_id=MOCK_OCTP_PROVIDER_ID, - sequence_number='12346', - ), + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number-1', + ) + ), + }, + { + 'messageId': '12346', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='octp', + provider_id=MOCK_OCTP_PROVIDER_ID, + sequence_number='some-sequence-number-2', + ) + ), + }, ] } @@ -372,7 +408,7 @@ def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opens mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that only the failed document's sequence number is in batchItemFailures + # Verify that only the failed document's message ID is in batchItemFailures self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12346', result['batchItemFailures'][0]['itemIdentifier']) @@ -391,19 +427,29 @@ def test_bulk_index_exception_returns_all_batch_item_failures(self, mock_opensea self._put_test_provider_and_license_record_in_dynamodb_table('aslp') self._put_test_provider_and_license_record_in_dynamodb_table('octp') - # Create DynamoDB stream events for both providers + # Create SQS events with DynamoDB stream records in the body for both providers event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ), - self._create_dynamodb_stream_record( - compact='octp', - provider_id=MOCK_OCTP_PROVIDER_ID, - sequence_number='12346', - ), + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number-1', + ) + ), + }, + { + 'messageId': '12346', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='octp', + provider_id=MOCK_OCTP_PROVIDER_ID, + sequence_number='some-sequence-number-2', + ) + ), + }, ] } @@ -428,19 +474,29 @@ def test_multiple_compacts_indexed_separately(self, mock_opensearch_client): self._put_test_provider_and_license_record_in_dynamodb_table('aslp') self._put_test_provider_and_license_record_in_dynamodb_table('octp') - # Create DynamoDB stream events for both compacts + # Create SQS events with DynamoDB stream records in the body for both compacts event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ), - self._create_dynamodb_stream_record( - compact='octp', - provider_id=MOCK_OCTP_PROVIDER_ID, - sequence_number='12346', - ), + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number-1', + ) + ), + }, + { + 'messageId': '12346', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='octp', + provider_id=MOCK_OCTP_PROVIDER_ID, + sequence_number='some-sequence-number-2', + ) + ), + }, ] } @@ -498,16 +554,21 @@ def test_insert_event_without_old_image_indexes_successfully(self, mock_opensear # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') - # Create a DynamoDB stream event for INSERT (no OldImage) + # Create an SQS event with DynamoDB stream record in the body for INSERT (no OldImage) event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - event_name='INSERT', - include_old_image=False, # INSERT events don't have OldImage - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + event_name='INSERT', + include_old_image=False, # INSERT events don't have OldImage + ) + ), + } ] } @@ -579,14 +640,19 @@ def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opense # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') - # Create a DynamoDB stream event for REMOVE (only OldImage, no NewImage) + # Create an SQS event with DynamoDB stream record in the body for REMOVE (only OldImage, no NewImage) event = { 'Records': [ - self._create_dynamodb_stream_record_with_old_image_only( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record_with_old_image_only( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + ) + ), + } ] } @@ -626,16 +692,21 @@ def test_provider_deleted_from_index_when_no_records_found(self, mock_opensearch # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted - # Create a DynamoDB stream event for a provider that no longer exists + # Create an SQS event with DynamoDB stream record in the body for a provider that no longer exists event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - event_name='REMOVE', - include_old_image=False, - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + event_name='REMOVE', + include_old_image=False, + ) + ), + } ] } @@ -671,16 +742,21 @@ def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_cl # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted - # Create a DynamoDB stream event for a provider that no longer exists + # Create an SQS event with DynamoDB stream record in the body for a provider that no longer exists event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - event_name='REMOVE', - include_old_image=False, - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + event_name='REMOVE', + include_old_image=False, + ) + ), + } ] } @@ -688,7 +764,7 @@ def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_cl mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that the batch item failure is returned + # Verify that the batch item failure is returned with the message ID self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) @@ -731,13 +807,18 @@ def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock # Create a DynamoDB stream event for a provider that no longer exists event = { 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - event_name='REMOVE', - include_old_image=False, - ) + { + 'messageId': '12345', + 'body': json.dumps( + self._create_dynamodb_stream_record( + compact='aslp', + provider_id=MOCK_ASLP_PROVIDER_ID, + sequence_number='some-sequence-number', + event_name='REMOVE', + include_old_image=False, + ) + ), + } ] } @@ -750,78 +831,3 @@ def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock # Verify NO batch item failures - 404 is not treated as an error self.assertEqual({'batchItemFailures': []}, result) - - @patch('handlers.provider_update_ingest.OpenSearchClient') - def test_failed_indexing_recorded_in_event_state_table(self, mock_opensearch_client): - """Test that failed indexing operations are recorded in the search event state table. - - When a provider fails to index (e.g., validation error or OpenSearch error), - the handler should record the failure in the search event state table with the compact, - provider ID, and sequence number for retry purposes. - """ - from handlers.provider_update_ingest import provider_update_ingest_handler - - # Set up mock OpenSearch client - bulk_index returns error for one document - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - - # Simulate OpenSearch returning an error for one document - mock_client_instance.bulk_index.return_value = { - 'errors': True, - 'items': [ - { - 'index': { - '_id': MOCK_ASLP_PROVIDER_ID, - '_index': 'compact_aslp_providers', - 'status': 400, - 'error': { - 'type': 'mapper_parsing_exception', - 'reason': 'failed to parse field', - }, - } - } - ], - } - - # Create provider and license records in DynamoDB - self._put_test_provider_and_license_record_in_dynamodb_table('aslp') - - # Create DynamoDB stream event - event = { - 'Records': [ - self._create_dynamodb_stream_record( - compact='aslp', - provider_id=MOCK_ASLP_PROVIDER_ID, - sequence_number='12345', - ) - ] - } - - # Run the handler - mock_context = MagicMock() - result = provider_update_ingest_handler(event, mock_context) - - # Verify that the batch item failure is returned - self.assertEqual(1, len(result['batchItemFailures'])) - self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) - - # Verify that a record was written to the search event state table - response = self.config.search_event_state_table.get_item( - Key={ - 'pk': 'COMPACT#aslp#FAILED_INGEST', - 'sk': f'PROVIDER#{MOCK_ASLP_PROVIDER_ID}#SEQUENCE#12345', - } - ) - - self.assertEqual( - { - 'compact': 'aslp', - 'pk': 'COMPACT#aslp#FAILED_INGEST', - 'sk': f'PROVIDER#{MOCK_ASLP_PROVIDER_ID}#SEQUENCE#12345', - 'providerId': MOCK_ASLP_PROVIDER_ID, - 'sequenceNumber': '12345', - # verify that TTL is set - 'ttl': ANY, - }, - response['Item'], - ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 308c9204e..898312a43 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -10,6 +10,7 @@ from stacks.search_persistent_stack.populate_provider_documents_handler import PopulateProviderDocumentsHandler from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain from stacks.search_persistent_stack.provider_update_ingest_handler import ProviderUpdateIngestHandler +from stacks.search_persistent_stack.provider_update_ingest_pipe import ProviderUpdateIngestPipe from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.search_persistent_stack.search_handler import SearchHandler from stacks.vpc_stack import VpcStack @@ -152,8 +153,8 @@ def __init__( alarm_topic=persistent_stack.alarm_topic, ) - # Create the provider update ingest handler for DynamoDB stream processing - # This handler processes real-time updates from the provider table stream + # Create the provider update ingest handler for SQS-based stream processing + # This handler processes real-time updates from the provider table stream via EventBridge Pipe -> SQS self.provider_update_ingest_handler = ProviderUpdateIngestHandler( self, construct_id='providerUpdateIngestHandler', @@ -166,3 +167,13 @@ def __init__( encryption_key=self.opensearch_encryption_key, alarm_topic=persistent_stack.alarm_topic, ) + + # Create the EventBridge Pipe to connect DynamoDB stream to SQS queue + # This pipe reads from the provider table stream and sends events to the ingest handler's queue + self.provider_update_ingest_pipe = ProviderUpdateIngestPipe( + self, + construct_id='providerUpdateIngestPipe', + provider_table=persistent_stack.provider_table, + target_queue=self.provider_update_ingest_handler.queue, + encryption_key=self.opensearch_encryption_key, + ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 0c8f1f61b..29596b520 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -7,17 +7,15 @@ from aws_cdk.aws_ec2 import SubnetSelection from aws_cdk.aws_iam import IRole from aws_cdk.aws_kms import IKey -from aws_cdk.aws_lambda import StartingPosition -from aws_cdk.aws_lambda_event_sources import DynamoEventSource, SqsDlq from aws_cdk.aws_logs import RetentionDays from aws_cdk.aws_opensearchservice import Domain from aws_cdk.aws_sns import ITopic -from aws_cdk.aws_sqs import Queue, QueueEncryption from cdk_nag import NagSuppressions from common_constructs.stack import Stack from constructs import Construct from common_constructs.python_function import PythonFunction +from common_constructs.queued_lambda_processor import QueuedLambdaProcessor from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.vpc_stack import VpcStack @@ -26,11 +24,13 @@ class ProviderUpdateIngestHandler(Construct): """ Construct for the Provider Update Ingest Lambda function. - This construct creates the Lambda function that processes DynamoDB stream events - from the provider table and indexes the updated provider documents into OpenSearch. + This construct creates the Lambda function that processes SQS messages containing + DynamoDB stream events from the provider table and indexes the updated provider + documents into OpenSearch. - The Lambda is triggered by DynamoDB streams and processes events in batches, - deduplicating provider IDs by compact before bulk indexing into OpenSearch. + The Lambda is triggered by SQS (fed by EventBridge Pipe from DynamoDB streams) + and processes events in batches, deduplicating provider IDs by compact before + bulk indexing into OpenSearch. """ def __init__( @@ -55,7 +55,7 @@ def __init__( :param vpc_stack: The VPC stack :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) - :param provider_table: The DynamoDB provider table with stream enabled + :param provider_table: The DynamoDB provider table (used for fetching full provider records) :param search_event_state_table: The DynamoDB table for tracking failed indexing operations :param encryption_key: The KMS encryption key for the SQS queue :param alarm_topic: The SNS topic for alarms @@ -63,20 +63,12 @@ def __init__( super().__init__(scope, construct_id) stack = Stack.of(scope) - # Create the dead letter queue for failed stream events - self.dlq = Queue( - self, - 'ProviderUpdateIngestDLQ', - encryption=QueueEncryption.KMS, - encryption_master_key=encryption_key, - enforce_ssl=True, - ) - - # Create Lambda function for processing provider updates from DynamoDB streams + # Create Lambda function for processing provider updates from SQS self.handler = PythonFunction( self, 'ProviderUpdateIngestFunction', - description='Processes DynamoDB stream events and indexes provider documents into OpenSearch', + description='Processes SQS messages with DynamoDB stream events and indexes provider documents into ' + 'OpenSearch', index=os.path.join('handlers', 'provider_update_ingest.py'), lambda_dir='search', handler='provider_update_ingest_handler', @@ -89,7 +81,7 @@ def __init__( **stack.common_env_vars, }, # Allow enough time for processing large batches - timeout=Duration.minutes(5), + timeout=Duration.minutes(10), memory_size=512, vpc=vpc_stack.vpc, vpc_subnets=vpc_subnets, @@ -97,29 +89,45 @@ def __init__( alarm_topic=alarm_topic, ) - # Add DynamoDB stream as event source - self.handler.add_event_source( - DynamoEventSource( - provider_table, - starting_position=StartingPosition.TRIM_HORIZON, - batch_size=1000, - # Setting this to 15 seconds to give downstream updates time to be batched with initial - # updates to reduce the number of provider update calls. This can be adjusted as needed - max_batching_window=Duration.seconds(15), - bisect_batch_on_error=True, - retry_attempts=3, - on_failure=SqsDlq(self.dlq), - report_batch_item_failures=True, - ) + # Create the QueuedLambdaProcessor for SQS-based event processing + # The queue receives DynamoDB stream events from EventBridge Pipe + self.queue_processor = QueuedLambdaProcessor( + self, + 'ProviderUpdateIngest', + process_function=self.handler, + # Visibility timeout set to slightly longer than Lambda timeout. With batchItemFailures, + # failed messages are retried immediately, so we only need enough buffer for crash scenarios. + # If Lambda times out (10 min), messages become visible again after ~15 minutes for retry. + visibility_timeout=Duration.minutes(15), + # Retention period for the source queue (these should be processed fairly quickly, but setting this to + # account for retries) + retention_period=Duration.hours(4), + # Setting batch size to 5000 to give lambda headroom to process the events + # without timing out while reducing number of index requests. + batch_size=5000, + # Batching window to allow multiple events to be processed together + max_batching_window=Duration.seconds(15), + # Number of times to retry before sending to DLQ + max_receive_count=3, + encryption_key=encryption_key, + alarm_topic=alarm_topic, + # DLQ retention of 14 days for analysis and replay + dlq_retention_period=Duration.days(14), + # Alert immediately if any messages end up in the DLQ + dlq_count_alarm_threshold=1, ) + # Expose the queue and DLQ for use by the EventBridge Pipe + self.queue = self.queue_processor.queue + self.dlq = self.queue_processor.dlq + # Grant the handler write access to the OpenSearch domain opensearch_domain.grant_write(self.handler) # Grant the handler read access to the provider table for fetching full provider records provider_table.grant_read_data(self.handler) - # Grant the DLQ permission to use the encryption key + # Grant the handler permission to use the encryption key for SQS operations encryption_key.grant_encrypt_decrypt(self.handler) # Add alarm for Lambda errors @@ -130,20 +138,7 @@ def __init__( evaluation_periods=1, threshold=1, actions_enabled=True, - alarm_description=f'{self.handler.node.path} failed to process a DynamoDB stream batch', - comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=TreatMissingData.NOT_BREACHING, - ).add_alarm_action(SnsAction(alarm_topic)) - - # Add alarm for DLQ messages - Alarm( - self, - 'ProviderUpdateIngestDLQAlarm', - metric=self.dlq.metric_approximate_number_of_messages_visible(), - evaluation_periods=1, - threshold=1, - actions_enabled=True, - alarm_description=f'{self.dlq.node.path} has messages - provider update ingest failures', + alarm_description=f'{self.handler.node.path} failed to process an SQS message batch', comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, treat_missing_data=TreatMissingData.NOT_BREACHING, ).add_alarm_action(SnsAction(alarm_topic)) @@ -157,18 +152,7 @@ def __init__( 'id': 'AwsSolutions-IAM5', 'reason': 'The grant_write method requires wildcard permissions on the OpenSearch domain to ' 'write to indices. This is appropriate for a function that needs to index ' - 'provider documents. The DynamoDB grant_read_data also requires index permissions. ' - 'The DynamoDB stream permissions require wildcard access to stream resources.', - }, - ], - ) - - NagSuppressions.add_resource_suppressions( - self.dlq, - [ - { - 'id': 'AwsSolutions-SQS3', - 'reason': 'This queue serves as a dead letter queue for the DynamoDB stream event source.', + 'provider documents.', }, ], ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py new file mode 100644 index 000000000..ed655562e --- /dev/null +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py @@ -0,0 +1,106 @@ +from aws_cdk.aws_dynamodb import ITable +from aws_cdk.aws_iam import Effect, PolicyStatement, Role, ServicePrincipal +from aws_cdk.aws_kms import IKey +from aws_cdk.aws_pipes import CfnPipe +from aws_cdk.aws_sqs import IQueue +from cdk_nag import NagSuppressions +from common_constructs.stack import Stack +from constructs import Construct + + +class ProviderUpdateIngestPipe(Construct): + """ + Construct for the EventBridge Pipe that connects DynamoDB stream to SQS. + + This construct creates an EventBridge Pipe that: + - Reads events from the DynamoDB provider table stream + - Sends events to an SQS queue for processing by the provider update ingest Lambda + + The Pipe enables decoupling the DynamoDB stream from the Lambda function, allowing + for better scalability and resilience through SQS-based message processing. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + provider_table: ITable, + target_queue: IQueue, + encryption_key: IKey, + ): + """ + Initialize the ProviderUpdateIngestPipe construct. + + :param scope: The scope of the construct + :param construct_id: The id of the construct + :param provider_table: The DynamoDB provider table with stream enabled + :param target_queue: The SQS queue to send events to + :param encryption_key: The KMS encryption key used by the SQS queue + """ + super().__init__(scope, construct_id) + stack = Stack.of(scope) + + # Create IAM role for the EventBridge Pipe + self.pipe_role = Role( + self, + 'PipeRole', + assumed_by=ServicePrincipal('pipes.amazonaws.com'), + description='IAM role for EventBridge Pipe that reads from DynamoDB stream and sends to SQS', + ) + + # Grant permissions to read from DynamoDB stream + # The stream ARN is constructed from the table ARN + self.pipe_role.add_to_policy( + PolicyStatement( + effect=Effect.ALLOW, + actions=[ + 'dynamodb:DescribeStream', + 'dynamodb:GetRecords', + 'dynamodb:GetShardIterator', + 'dynamodb:ListStreams', + ], + resources=[ + f'{provider_table.table_arn}/stream/*', + ], + ) + ) + + # Grant permissions to send messages to SQS + target_queue.grant_send_messages(self.pipe_role) + + # Grant permissions to use the KMS key for encrypting SQS messages + encryption_key.grant_encrypt_decrypt(self.pipe_role) + # Grant permission to decrypt stream records from provider table + provider_table.encryption_key.grant_decrypt(self.pipe_role) + + # Create the EventBridge Pipe + # Using CfnPipe (L1 construct) as there's no L2 construct available yet + self.pipe = CfnPipe( + self, + 'Pipe', + role_arn=self.pipe_role.role_arn, + source=provider_table.table_stream_arn, + target=target_queue.queue_arn, + source_parameters=CfnPipe.PipeSourceParametersProperty( + dynamo_db_stream_parameters=CfnPipe.PipeSourceDynamoDBStreamParametersProperty( + starting_position='TRIM_HORIZON', + # send everything to SQS as it arrives + batch_size=1, + ), + ), + description='Pipe to send DynamoDB provider table stream events to SQS for OpenSearch indexing', + ) + + # Add CDK Nag suppressions + NagSuppressions.add_resource_suppressions_by_path( + stack, + f'{self.pipe_role.node.path}/DefaultPolicy/Resource', + [ + { + 'id': 'AwsSolutions-IAM5', + 'reason': 'The DynamoDB stream permissions require wildcard access to stream resources ' + 'as the stream ARN includes a timestamp component that changes on table recreation. ' + 'The SQS grant_send_messages also adds appropriate permissions.', + }, + ], + ) From ee4e033237cba08fff57d5eaa179867ea9d7a203 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 10:23:05 -0600 Subject: [PATCH 090/137] Remove unneeded search event state table Now that we can handle ingest retries through a proper SQS DLQ solution, this table is no longer needed. --- .../lambdas/python/common/cc_common/config.py | 14 -- .../handlers/populate_provider_documents.py | 184 +----------------- .../search/search_event_state_client.py | 131 ------------- .../lambdas/python/search/tests/__init__.py | 1 - .../python/search/tests/function/__init__.py | 14 -- .../test_populate_provider_documents.py | 136 ------------- .../search_persistent_stack/__init__.py | 17 -- .../populate_provider_documents_handler.py | 4 - .../provider_update_ingest_handler.py | 6 +- .../search_event_state_table.py | 52 ----- 10 files changed, 4 insertions(+), 555 deletions(-) delete mode 100644 backend/compact-connect/lambdas/python/search/search_event_state_client.py delete mode 100644 backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py diff --git a/backend/compact-connect/lambdas/python/common/cc_common/config.py b/backend/compact-connect/lambdas/python/common/cc_common/config.py index 3bcb6392a..e952271fa 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/config.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/config.py @@ -322,20 +322,6 @@ def event_state_client(self): return EventStateClient(self) - @property - def search_event_state_table_name(self): - return os.environ['SEARCH_EVENT_STATE_TABLE_NAME'] - - @property - def search_event_state_table(self): - return boto3.resource('dynamodb').Table(self.search_event_state_table_name) - - @cached_property - def search_event_state_client(self): - from search.search_event_state_client import SearchEventStateClient - - return SearchEventStateClient(self) - @cached_property def allowed_origins(self): return json.loads(os.environ['ALLOWED_ORIGINS']) diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index 32e4b76c2..a107d5a69 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -22,10 +22,9 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import config, logger -from cc_common.exceptions import CCInternalException, CCNotFoundException +from cc_common.exceptions import CCInternalException from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from search_event_state_client import get_failed_ingest_provider_ids from utils import generate_provider_opensearch_document # Batch size for DynamoDB pagination @@ -41,11 +40,6 @@ def populate_provider_documents(event: dict, context: LambdaContext): """ Populate OpenSearch indices with provider documents. - This function can operate in two modes: - 1. Normal mode: Scans all providers in the provider table using the providerDateOfUpdate GSI - 2. Retry mode: Re-indexes providers that previously failed ingestion (when 'retry_ingest_failures_for_compact' - is provided) - Retrieves complete provider records, sanitizes them using ProviderGeneralResponseSchema, and bulk indexes them into the appropriate OpenSearch indices. @@ -53,20 +47,14 @@ def populate_provider_documents(event: dict, context: LambdaContext): information that can be passed as input to continue processing. :param event: Lambda event with optional parameters: - - retry_ingest_failures_for_compact: If present, retry indexing for all failed providers for this compact - - startingCompact: The compact to start/resume processing from (normal mode only) - - startingLastKey: The DynamoDB pagination key to resume from (normal mode only) + - startingCompact: The compact to start/resume processing from + - startingLastKey: The DynamoDB pagination key to resume from :param context: Lambda context :return: Summary of indexing operation, including pagination info if incomplete """ data_client = config.data_client opensearch_client = OpenSearchClient() - # Check if this is a retry operation for failed ingestions - retry_compact = event.get('retry_ingest_failures_for_compact') - if retry_compact: - return _retry_failed_ingest_events(retry_compact, opensearch_client) - # Get optional pagination parameters from event for resumption (normal mode) starting_compact = event.get('startingCompact') starting_last_key = event.get('startingLastKey') @@ -300,46 +288,6 @@ def _index_records_and_track_stats( ) -def _delete_records_and_track_stats( - providers_to_delete: list[str], compact: str, opensearch_client: OpenSearchClient, compact_stats: dict -): - """ - Bulk delete provider documents from OpenSearch and track statistics. - - :param providers_to_delete: List of provider IDs to delete - :param compact: The compact abbreviation - :param opensearch_client: The OpenSearch client - :param compact_stats: Statistics dictionary to update (in/out) - """ - index_name = f'compact_{compact}_providers' - if not providers_to_delete: - return - - try: - failed_provider_ids = opensearch_client.bulk_delete(index_name=index_name, document_ids=providers_to_delete) - failed_deletes = len(failed_provider_ids) - - successful_deletes = len(providers_to_delete) - failed_deletes - compact_stats['providers_deleted'] += successful_deletes - compact_stats['providers_failed'] += failed_deletes - logger.info( - 'Bulk deleted documents', - index_name=index_name, - failed_provider_ids=list(failed_provider_ids), - document_count=len(providers_to_delete), - ) - except CCInternalException as e: - # All deletes for this batch failed - logger.error( - 'Failed to bulk delete documents after retries', - index_name=index_name, - failed_provider_ids=providers_to_delete, - document_count=len(providers_to_delete), - error=str(e), - ) - compact_stats['providers_failed'] += len(providers_to_delete) - - def _build_error_response( stats: dict, compact_stats: dict, compact: str, batch_start_key: dict | None, error_message: str ) -> dict: @@ -432,129 +380,3 @@ def _bulk_index_documents(opensearch_client: OpenSearchClient, index_name: str, document_count=len(documents), ) return set() - - -def _retry_failed_ingest_events(compact: str, opensearch_client: OpenSearchClient) -> dict: - """ - Retry indexing for all providers that previously failed ingestion for a specific compact. - - This function queries the search event state table for all failed ingest records, - then processes and indexes those providers. - - :param compact: The compact abbreviation to retry failed ingestions for - :param opensearch_client: The OpenSearch client - :return: Summary of indexing operation - """ - logger.info('Retrying failed ingestions for compact', compact=compact) - - # Get list of provider IDs that failed ingestion - failed_provider_ids = get_failed_ingest_provider_ids(compact) - - if not failed_provider_ids: - logger.info('No failed ingestions found for compact', compact=compact) - return { - 'total_providers_processed': 0, - 'total_providers_indexed': 0, - 'total_providers_deleted': 0, - 'total_providers_failed': 0, - 'compacts_processed': [ - { - 'compact': compact, - 'providers_processed': 0, - 'providers_indexed': 0, - 'providers_deleted': 0, - 'providers_failed': 0, - } - ], - 'completed': True, - } - - documents_to_index = [] - providers_to_delete = [] - compact_stats = { - 'providers_processed': 0, - 'providers_indexed': 0, - 'providers_deleted': 0, - 'providers_failed': 0, - } - - # Process each failed provider - for provider_id in failed_provider_ids: - compact_stats['providers_processed'] += 1 - - try: - # Use the shared utility to process the provider - serializable_document = generate_provider_opensearch_document(compact, provider_id) - documents_to_index.append(serializable_document) - except CCNotFoundException as e: - logger.warning( - 'No provider records found. This may occur if a license upload rollback was performed or if records' - ' were manually deleted. Will delete provider document from index.', - provider_id=provider_id, - compact=compact, - error=str(e), - ) - providers_to_delete.append(provider_id) - except ValidationError as e: - logger.warning( - 'Failed to process provider during retry', - provider_id=provider_id, - compact=compact, - error=str(e), - ) - compact_stats['providers_failed'] += 1 - continue - - # Bulk index when batch is full - if len(documents_to_index) >= OPENSEARCH_BULK_SIZE: - try: - _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) - documents_to_index = [] - except CCInternalException as e: - logger.error( - 'Failed to bulk index during retry', - compact=compact, - error=str(e), - ) - compact_stats['providers_failed'] += len(documents_to_index) - documents_to_index = [] - - # Bulk delete when batch is full - if len(providers_to_delete) >= OPENSEARCH_BULK_SIZE: - _delete_records_and_track_stats(providers_to_delete, compact, opensearch_client, compact_stats) - providers_to_delete = [] - - # Index any remaining documents - if documents_to_index: - try: - _index_records_and_track_stats(documents_to_index, compact, opensearch_client, compact_stats) - except CCInternalException as e: - logger.error('Failed to index remaining documents during retry', error=str(e)) - compact_stats['providers_failed'] += len(documents_to_index) - - # Delete any remaining providers that no longer exist - if providers_to_delete: - _delete_records_and_track_stats(providers_to_delete, compact, opensearch_client, compact_stats) - - logger.info( - 'Completed retrying failed ingestions', - compact=compact, - providers_processed=compact_stats['providers_processed'], - providers_indexed=compact_stats['providers_indexed'], - providers_deleted=compact_stats['providers_deleted'], - providers_failed=compact_stats['providers_failed'], - ) - - return { - 'total_providers_processed': compact_stats['providers_processed'], - 'total_providers_indexed': compact_stats['providers_indexed'], - 'total_providers_deleted': compact_stats['providers_deleted'], - 'total_providers_failed': compact_stats['providers_failed'], - 'compacts_processed': [ - { - 'compact': compact, - **compact_stats, - } - ], - 'completed': True, - } diff --git a/backend/compact-connect/lambdas/python/search/search_event_state_client.py b/backend/compact-connect/lambdas/python/search/search_event_state_client.py deleted file mode 100644 index faeda02b0..000000000 --- a/backend/compact-connect/lambdas/python/search/search_event_state_client.py +++ /dev/null @@ -1,131 +0,0 @@ -import time -from datetime import timedelta - -from boto3.dynamodb.conditions import Key -from cc_common.config import config, logger - - -def record_failed_indexing_batch( - failures: list[dict[str, str]], - *, - ttl_days: int = 7, -) -> None: - """ - Record multiple failed indexing operations to the search event state table using batch writes. - - This method stores the compact, provider ID, and sequence number for each failure so that - developers can replay failed indexing operations. Uses DynamoDB batch writer for efficient - bulk writes. - - :param failures: List of failure records, each containing 'compact', 'provider_id', and 'sequence_number' - :param ttl_days: TTL in days (default 7 days) - """ - if not failures: - return - - # Calculate TTL (Unix timestamp in seconds) - ttl = int(time.time()) + int(timedelta(days=ttl_days).total_seconds()) - - # Use batch writer for efficient bulk writes - try: - with config.search_event_state_table.batch_writer() as batch: - for failure in failures: - compact = failure['compact'] - provider_id = failure['provider_id'] - sequence_number = failure['sequence_number'] - - # Build partition and sort keys - # PK: COMPACT#{compact}#FAILED_INGEST - allows querying all failures for a provider - # SK: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - allows identifying the specific stream record - pk = f'COMPACT#{compact}#FAILED_INGEST' - sk = f'PROVIDER#{provider_id}#SEQUENCE#{sequence_number}' - - # Build item - item = { - 'pk': pk, - 'sk': sk, - 'compact': compact, - 'providerId': provider_id, - 'sequenceNumber': sequence_number, - 'ttl': ttl, - } - - batch.put_item(Item=item) - - logger.info( - 'Recorded failed indexing operations in batch', - failure_count=len(failures), - ttl_days=ttl_days, - ) - except Exception as e: # noqa: BLE001 - # Log error but don't fail the handler - this is tracking data, not critical path - logger.error( - 'Failed to record indexing failures in event state table', - failure_count=len(failures), - error=str(e), - ) - - -def get_failed_ingest_provider_ids(compact: str) -> list[str]: - """ - Query the search event state table for all failed ingest records for a compact. - - Returns a deduplicated list of provider IDs that have failed indexing operations. - This can be used to retry indexing for providers that previously failed. - - :param compact: The compact abbreviation (e.g., 'aslp') - :return: List of unique provider IDs that have failed indexing operations - """ - pk = f'COMPACT#{compact}#FAILED_INGEST' - provider_ids = set() - - try: - # Query all records with the partition key - # The SK pattern is PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - # so we can extract provider IDs from the SK - response = config.search_event_state_table.query( - KeyConditionExpression=Key('pk').eq(pk), - ) - - # Extract provider IDs from the sort keys - for item in response.get('Items', []): - sk = item.get('sk', '') - # SK format: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - if sk.startswith('PROVIDER#'): - # Extract provider ID from SK - # Format: PROVIDER#{provider_id}#SEQUENCE#{sequence_number} - parts = sk.split('#') - if len(parts) >= 2: - provider_id = parts[1] # The provider ID is the second part - provider_ids.add(provider_id) - - # Handle pagination - while 'LastEvaluatedKey' in response: - response = config.search_event_state_table.query( - KeyConditionExpression=Key('pk').eq(pk), - ExclusiveStartKey=response['LastEvaluatedKey'], - ) - - for item in response.get('Items', []): - sk = item.get('sk', '') - if sk.startswith('PROVIDER#'): - parts = sk.split('#') - if len(parts) >= 2: - provider_id = parts[1] - provider_ids.add(provider_id) - - provider_ids_list = sorted(list(provider_ids)) - logger.info( - 'Retrieved failed ingest provider IDs', - compact=compact, - provider_count=len(provider_ids_list), - ) - return provider_ids_list - - except Exception as e: # noqa: BLE001 - logger.error( - 'Failed to query failed ingest records', - compact=compact, - error=str(e), - ) - return [] diff --git a/backend/compact-connect/lambdas/python/search/tests/__init__.py b/backend/compact-connect/lambdas/python/search/tests/__init__.py index dcc0127b6..53d1a14ab 100644 --- a/backend/compact-connect/lambdas/python/search/tests/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/__init__.py @@ -24,7 +24,6 @@ def setUpClass(cls): 'LICENSE_GSI_NAME': 'licenseGSI', 'OPENSEARCH_HOST_ENDPOINT': 'vpc-providersearchd-5bzuqxhpxffk-w6dkpddu.us-east-1.es.amazonaws.com', 'EXPORT_RESULTS_BUCKET_NAME': 'test-export-results-bucket', - 'SEARCH_EVENT_STATE_TABLE_NAME': 'search-event-state-table', 'JURISDICTIONS': json.dumps( [ 'al', diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py index 414868cb4..9044cc9c8 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -27,11 +27,9 @@ def setUp(self): # noqa: N801 invalid-name def build_resources(self): self.create_provider_table() self.create_export_results_bucket() - self.create_search_event_state_table() def delete_resources(self): self._provider_table.delete() - self._search_event_state_table.delete() # must delete all objects in the bucket before deleting the bucket self._bucket.objects.delete() self._bucket.delete() @@ -93,15 +91,3 @@ def create_provider_table(self): }, ], ) - - def create_search_event_state_table(self): - """Create the mock DynamoDB table for search event state tracking""" - self._search_event_state_table = boto3.resource('dynamodb').create_table( - AttributeDefinitions=[ - {'AttributeName': 'pk', 'AttributeType': 'S'}, - {'AttributeName': 'sk', 'AttributeType': 'S'}, - ], - TableName=os.environ['SEARCH_EVENT_STATE_TABLE_NAME'], - KeySchema=[{'AttributeName': 'pk', 'KeyType': 'HASH'}, {'AttributeName': 'sk', 'KeyType': 'RANGE'}], - BillingMode='PAY_PER_REQUEST', - ) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index 3cf3fa5dd..b0fd319d5 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -347,139 +347,3 @@ def test_returns_pagination_info_when_bulk_indexing_fails_after_retries(self, mo # Verify octp and coun were indexed self.assertEqual(2, second_result['total_providers_indexed']) self.assertEqual(2, mock_client_instance.bulk_index.call_count) - - @patch('handlers.populate_provider_documents.OpenSearchClient') - def test_retry_ingest_failures_for_compact_indexes_only_failed_providers(self, mock_opensearch_client): - """Test that retry_ingest_failures_for_compact only indexes providers from the search event state table. - - This test verifies: - 1. A failed ingest record is put in the search event state table - 2. A provider record exists in the provider table - 3. When 'retry_ingest_failures_for_compact' is passed, only that provider is indexed - 4. The opensearch_client is called with the correct provider document - """ - from handlers.populate_provider_documents import populate_provider_documents - - # Set up the mock opensearch client - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) - - compact = 'aslp' - provider_id = MOCK_ASLP_PROVIDER_ID - sequence_number = '12345' - - # Put a failed ingest record in the search event state table - self._put_failed_ingest_record_in_search_event_state_table(compact, provider_id, sequence_number) - - # Put provider and license records in the provider table - self._put_test_provider_and_license_record_in_dynamodb_table(compact) - - # Create event with retry_ingest_failures_for_compact - event = {'retry_ingest_failures_for_compact': compact} - - # Mock context (not used in retry path, but required for handler signature) - mock_context = MagicMock() - - # Run the handler - result = populate_provider_documents(event, mock_context) - - # Assert that the OpenSearchClient was instantiated - mock_opensearch_client.assert_called_once() - - # Assert that bulk_index was called exactly once (only for the failed provider) - self.assertEqual(1, mock_client_instance.bulk_index.call_count) - - # Verify the call was made with the correct provider document - bulk_index_calls = mock_client_instance.bulk_index.call_args_list - expected_call = self._generate_expected_call_for_document(compact) - self.assertEqual(expected_call, bulk_index_calls[0]) - - # Verify the result statistics - self.assertEqual( - { - 'compacts_processed': [ - { - 'compact': compact, - 'providers_failed': 0, - 'providers_indexed': 1, - 'providers_deleted': 0, - 'providers_processed': 1, - } - ], - 'completed': True, - 'total_providers_failed': 0, - 'total_providers_deleted': 0, - 'total_providers_indexed': 1, - 'total_providers_processed': 1, - }, - result, - ) - - @patch('handlers.populate_provider_documents.OpenSearchClient') - def test_retry_ingest_failures_deletes_providers_when_not_found(self, mock_opensearch_client): - """Test that retry_ingest_failures_for_compact deletes providers from index when CCNotFoundException is raised. - - This test verifies: - 1. A failed ingest record is put in the search event state table - 2. NO provider records exist in the provider table (simulating deletion/rollback) - 3. When 'retry_ingest_failures_for_compact' is passed, bulk_delete is called - 4. The provider is deleted from the OpenSearch index - 5. Statistics reflect the deletion - """ - from handlers.populate_provider_documents import populate_provider_documents - - # Set up the mock opensearch client - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.bulk_index.return_value = set() - mock_client_instance.bulk_delete.return_value = set() - - compact = 'aslp' - provider_id = MOCK_ASLP_PROVIDER_ID - sequence_number = '12345' - - # Put a failed ingest record in the search event state table - self._put_failed_ingest_record_in_search_event_state_table(compact, provider_id, sequence_number) - - # Do NOT create provider records in the provider table - this simulates the provider being deleted - - # Create event with retry_ingest_failures_for_compact - event = {'retry_ingest_failures_for_compact': compact} - - # Mock context (not used in retry path, but required for handler signature) - mock_context = MagicMock() - - # Run the handler - result = populate_provider_documents(event, mock_context) - - # Assert that the OpenSearchClient was instantiated - mock_opensearch_client.assert_called_once() - - # Assert that bulk_index was NOT called (no documents to index) - mock_client_instance.bulk_index.assert_not_called() - - # Assert that bulk_delete WAS called with the correct parameters - self.assertEqual(1, mock_client_instance.bulk_delete.call_count) - call_args = mock_client_instance.bulk_delete.call_args - self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) - self.assertEqual([MOCK_ASLP_PROVIDER_ID], call_args.kwargs['document_ids']) - - # Verify the result statistics - self.assertEqual( - { - 'compacts_processed': [ - { - 'compact': compact, - 'providers_deleted': 1, - 'providers_failed': 0, - 'providers_indexed': 0, - 'providers_processed': 1, - } - ], - 'completed': True, - 'total_providers_deleted': 1, - 'total_providers_failed': 0, - 'total_providers_indexed': 0, - 'total_providers_processed': 1, - }, - result, - ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 898312a43..c14f40b6e 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -11,7 +11,6 @@ from stacks.search_persistent_stack.provider_search_domain import ProviderSearchDomain from stacks.search_persistent_stack.provider_update_ingest_handler import ProviderUpdateIngestHandler from stacks.search_persistent_stack.provider_update_ingest_pipe import ProviderUpdateIngestPipe -from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.search_persistent_stack.search_handler import SearchHandler from stacks.vpc_stack import VpcStack @@ -94,20 +93,6 @@ def __init__( self.domain = self.provider_search_domain.domain self.opensearch_encryption_key = self.provider_search_domain.encryption_key - # Determine removal policy based on environment - removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY - - # Create the search event state table for tracking failed indexing operations - self.search_event_state_table = SearchEventStateTable( - self, - 'SearchEventStateTable', - encryption_key=self.opensearch_encryption_key, - removal_policy=removal_policy, - ) - - # Grant the ingest role access to the search event state table for tracking and retrying failures - self.search_event_state_table.grant_read_write_data(self.opensearch_ingest_lambda_role) - # Create the export results bucket for temporary CSV files self.export_results_bucket = ExportResultsBucket( self, @@ -149,7 +134,6 @@ def __init__( vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, - search_event_state_table=self.search_event_state_table, alarm_topic=persistent_stack.alarm_topic, ) @@ -163,7 +147,6 @@ def __init__( vpc_subnets=self.provider_search_domain.vpc_subnets, lambda_role=self.opensearch_ingest_lambda_role, provider_table=persistent_stack.provider_table, - search_event_state_table=self.search_event_state_table, encryption_key=self.opensearch_encryption_key, alarm_topic=persistent_stack.alarm_topic, ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py index 1c95c00b5..d3c013ba4 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/populate_provider_documents_handler.py @@ -12,7 +12,6 @@ from constructs import Construct from common_constructs.python_function import PythonFunction -from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.vpc_stack import VpcStack @@ -37,7 +36,6 @@ def __init__( vpc_subnets: SubnetSelection, lambda_role: IRole, provider_table: ITable, - search_event_state_table: SearchEventStateTable, alarm_topic: ITopic, ): """ @@ -50,7 +48,6 @@ def __init__( :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) :param provider_table: The DynamoDB provider table - :param search_event_state_table: The DynamoDB table for tracking failed indexing operations :param alarm_topic: The SNS topic for alarms """ super().__init__(scope, construct_id) @@ -70,7 +67,6 @@ def __init__( 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, 'PROVIDER_TABLE_NAME': provider_table.table_name, 'PROV_DATE_OF_UPDATE_INDEX_NAME': provider_table.provider_date_of_update_index_name, - 'SEARCH_EVENT_STATE_TABLE_NAME': search_event_state_table.table_name, **stack.common_env_vars, }, # Longer timeout for processing large datasets diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 29596b520..1c735bc85 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -16,7 +16,6 @@ from common_constructs.python_function import PythonFunction from common_constructs.queued_lambda_processor import QueuedLambdaProcessor -from stacks.search_persistent_stack.search_event_state_table import SearchEventStateTable from stacks.vpc_stack import VpcStack @@ -42,7 +41,6 @@ def __init__( vpc_subnets: SubnetSelection, lambda_role: IRole, provider_table: ITable, - search_event_state_table: SearchEventStateTable, encryption_key: IKey, alarm_topic: ITopic, ): @@ -56,7 +54,6 @@ def __init__( :param vpc_subnets: The VPC subnets for Lambda deployment :param lambda_role: The IAM role for the Lambda function (should have OpenSearch write access) :param provider_table: The DynamoDB provider table (used for fetching full provider records) - :param search_event_state_table: The DynamoDB table for tracking failed indexing operations :param encryption_key: The KMS encryption key for the SQS queue :param alarm_topic: The SNS topic for alarms """ @@ -77,7 +74,6 @@ def __init__( environment={ 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, 'PROVIDER_TABLE_NAME': provider_table.table_name, - 'SEARCH_EVENT_STATE_TABLE_NAME': search_event_state_table.table_name, **stack.common_env_vars, }, # Allow enough time for processing large batches @@ -114,7 +110,7 @@ def __init__( # DLQ retention of 14 days for analysis and replay dlq_retention_period=Duration.days(14), # Alert immediately if any messages end up in the DLQ - dlq_count_alarm_threshold=1, + dlq_count_alarm_threshold=0, ) # Expose the queue and DLQ for use by the EventBridge Pipe diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py b/backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py deleted file mode 100644 index bf6b2cda6..000000000 --- a/backend/compact-connect/stacks/search_persistent_stack/search_event_state_table.py +++ /dev/null @@ -1,52 +0,0 @@ -from aws_cdk import RemovalPolicy -from aws_cdk.aws_dynamodb import ( - AttributeType, - BillingMode, - PointInTimeRecoverySpecification, - Table, - TableEncryption, -) -from aws_cdk.aws_kms import IKey -from cdk_nag import NagSuppressions -from constructs import Construct - - -class SearchEventStateTable(Table): - """ - DynamoDB table for tracking state of search update events for failure retries. - - This table is used to maintain idempotency and track failures of various indexing operations - performed when provider records are updated. - """ - - def __init__( - self, - scope: Construct, - construct_id: str, - encryption_key: IKey, - removal_policy: RemovalPolicy, - ) -> None: - super().__init__( - scope, - construct_id, - billing_mode=BillingMode.PAY_PER_REQUEST, - encryption=TableEncryption.CUSTOMER_MANAGED, - encryption_key=encryption_key, - partition_key={'name': 'pk', 'type': AttributeType.STRING}, - sort_key={'name': 'sk', 'type': AttributeType.STRING}, - point_in_time_recovery_specification=PointInTimeRecoverySpecification(point_in_time_recovery_enabled=True), - removal_policy=removal_policy, - deletion_protection=True if removal_policy == RemovalPolicy.RETAIN else False, - time_to_live_attribute='ttl', - ) - - NagSuppressions.add_resource_suppressions( - self, - suppressions=[ - { - 'id': 'HIPAA.Security-DynamoDBInBackupPlan', - 'reason': 'These records are not intended to be backed up. This table is only for temporary event ' - 'state tracking for retries and all records expire after several weeks.', - }, - ], - ) From 9f5505a55a92f2aacf688595a477665ff0b87f99 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 10:32:34 -0600 Subject: [PATCH 091/137] linter/logs --- .../python/search/handlers/provider_update_ingest.py | 6 ++++++ .../search/tests/function/test_provider_update_ingest.py | 2 +- .../stacks/search_persistent_stack/__init__.py | 2 -- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 845172d74..2892a2f5a 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -189,6 +189,12 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: # Map back from failed providers to their SQS message IDs for message_id, (compact, provider_id) in record_mapping.items(): if provider_id in failed_providers[compact]: + logger.info( + 'Returning message id in batch item failures for failed provider', + compact=compact, + provider_id=provider_id, + message_id=message_id, + ) batch_item_failures.append({'itemIdentifier': message_id}) if batch_item_failures: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index 761f3dbaf..676c79761 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -1,5 +1,5 @@ import json -from unittest.mock import ANY, MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, patch from common_test.test_constants import ( DEFAULT_LICENSE_EXPIRATION_DATE, diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index c14f40b6e..ebc90cc91 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,9 +1,7 @@ -from aws_cdk import RemovalPolicy from aws_cdk.aws_iam import Role, ServicePrincipal from common_constructs.stack import AppStack from constructs import Construct -from common_constructs.constants import PROD_ENV_NAME from stacks.persistent_stack import PersistentStack from stacks.search_persistent_stack.export_results_bucket import ExportResultsBucket from stacks.search_persistent_stack.index_manager import IndexManagerCustomResource From c5d07836d9800e4c6e1803a38a44c63ac9ddaa49 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 10:45:05 -0600 Subject: [PATCH 092/137] Add comments to clarify SQS retry behavior --- .../provider_update_ingest_handler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 1c735bc85..a8e11ca17 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -91,9 +91,9 @@ def __init__( self, 'ProviderUpdateIngest', process_function=self.handler, - # Visibility timeout set to slightly longer than Lambda timeout. With batchItemFailures, - # failed messages are retried immediately, so we only need enough buffer for crash scenarios. - # If Lambda times out (10 min), messages become visible again after ~15 minutes for retry. + # Visibility timeout controls when failed messages (in batchItemFailures) become visible for retry. + # Set to slightly longer than Lambda timeout (10 min) to prevent duplicate processing during + # Lambda execution. Failed messages will retry after this timeout expires (~15 minutes). visibility_timeout=Duration.minutes(15), # Retention period for the source queue (these should be processed fairly quickly, but setting this to # account for retries) @@ -103,7 +103,8 @@ def __init__( batch_size=5000, # Batching window to allow multiple events to be processed together max_batching_window=Duration.seconds(15), - # Number of times to retry before sending to DLQ + # Max receive count = total attempts before DLQ (1 initial + 2 retries = 3 total) + # Failed messages retry after visibility_timeout expires (15 min between attempts) max_receive_count=3, encryption_key=encryption_key, alarm_topic=alarm_topic, From de916a773fc6cd6af896d842695433d69254e360 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 11:56:44 -0600 Subject: [PATCH 093/137] Reduce ingest batch size to 2000 --- .../provider_update_ingest_handler.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index a8e11ca17..37d39f91b 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -98,10 +98,15 @@ def __init__( # Retention period for the source queue (these should be processed fairly quickly, but setting this to # account for retries) retention_period=Duration.hours(4), - # Setting batch size to 5000 to give lambda headroom to process the events - # without timing out while reducing number of index requests. - batch_size=5000, - # Batching window to allow multiple events to be processed together + # OpenSearch recommends performing bulk indexing with sizes between 5 - 15 MB per operation. + # see https://www.elastic.co/guide/en/elasticsearch/guide/2.x/indexing-performance.html#_using_and_sizing_bulk_requests + # A basic provider document without any additional records (privileges, adverse actions, etc.) is + # around 2KB on average. We expect these provider documents to grow over time as providers accumulate + # privileges and other records. Setting a batch size of 2000 places the initial bulk operations around + # 4MB max size per request (2KB * 2000 = 4 MB). This puts us below that range but provides headroom for + # these documents to grow over time, while still processing license uploads in a timely manner. + batch_size=2000, + # Batching window to allow multiple events for the same provider to be processed together max_batching_window=Duration.seconds(15), # Max receive count = total attempts before DLQ (1 initial + 2 retries = 3 total) # Failed messages retry after visibility_timeout expires (15 min between attempts) From 77cac60cb9604e5c9712ef6794cfc681889f6368 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 11:58:31 -0600 Subject: [PATCH 094/137] PR feedback - deserialize records --- .../search/handlers/provider_update_ingest.py | 23 ++++--------------- .../provider_update_ingest_pipe.py | 4 +++- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 2892a2f5a..bbbee232a 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -11,7 +11,7 @@ to the handler at once, enabling batch processing and deduplication. The handler returns batchItemFailures directly for partial success handling. """ - +from boto3.dynamodb.types import TypeDeserializer from cc_common.config import config, logger from cc_common.exceptions import CCInternalException, CCNotFoundException from cc_common.utils import sqs_batch_handler @@ -61,9 +61,10 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: # Extract compact and providerId from the DynamoDB image # The format is {'S': 'value'} for string attributes - compact = _extract_string_value(image.get('compact')) - provider_id = _extract_string_value(image.get('providerId')) - record_type = _extract_string_value(image.get('type')) + deserialized_image = TypeDeserializer().deserialize(value={'M': image}) + compact = deserialized_image.get('compact') + provider_id = deserialized_image.get('providerId') + record_type = deserialized_image.get('type') if not compact or not provider_id: logger.error( @@ -201,17 +202,3 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: logger.warning('Reporting batch item failures', failure_count=len(batch_item_failures)) return {'batchItemFailures': batch_item_failures} - - -def _extract_string_value(dynamo_attribute: dict | None) -> str | None: - """ - Extract a string value from a DynamoDB attribute. - - DynamoDB stream records use the format {'S': 'value'} for string attributes. - - :param dynamo_attribute: The DynamoDB attribute dict - :return: The string value, or None if not present - """ - if dynamo_attribute is None: - return None - return dynamo_attribute.get('S') diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py index ed655562e..677e83ceb 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py @@ -74,7 +74,7 @@ def __init__( provider_table.encryption_key.grant_decrypt(self.pipe_role) # Create the EventBridge Pipe - # Using CfnPipe (L1 construct) as there's no L2 construct available yet + # Using CfnPipe (L1 construct) as there's no stable L2 construct available yet self.pipe = CfnPipe( self, 'Pipe', @@ -83,6 +83,8 @@ def __init__( target=target_queue.queue_arn, source_parameters=CfnPipe.PipeSourceParametersProperty( dynamo_db_stream_parameters=CfnPipe.PipeSourceDynamoDBStreamParametersProperty( + # 'TRIM_HORIZON' starts processing from the earliest + # available stream record (oldest data in the DynamoDB stream, up to 24 hours ago) starting_position='TRIM_HORIZON', # send everything to SQS as it arrives batch_size=1, From 105dc7cf3738752f507dd83c50f7b637fa76dc76 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 12:23:33 -0600 Subject: [PATCH 095/137] Add query definitions for ingest and search --- .../search_persistent_stack/__init__.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index ebc90cc91..4a209da89 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -1,4 +1,5 @@ from aws_cdk.aws_iam import Role, ServicePrincipal +from aws_cdk.aws_logs import QueryDefinition, QueryString from common_constructs.stack import AppStack from constructs import Construct @@ -158,3 +159,29 @@ def __init__( target_queue=self.provider_update_ingest_handler.queue, encryption_key=self.opensearch_encryption_key, ) + + # add log insights for provider ingest + QueryDefinition( + self, + 'IngestQuery', + query_definition_name=f'{self.node.id}/ProviderUpdateIngest', + query_string=QueryString( + fields=['@timestamp', '@log', 'level', 'message', 'compact', 'provider_id', '@message'], + filter_statements=['level in ["INFO", "WARNING", "ERROR"]'], + sort='@timestamp asc', + ), + log_groups=[self.provider_update_ingest_handler.handler.log_group], + ) + + # add log insights for search requests + QueryDefinition( + self, + 'SearchLambdaQuery', + query_definition_name=f'{self.node.id}/SearchAPILambda', + query_string=QueryString( + fields=['@timestamp', '@log', 'level', 'message', 'compact', '@message'], + filter_statements=['level in ["INFO", "WARNING", "ERROR"]'], + sort='@timestamp asc', + ), + log_groups=[self.search_handler.handler.log_group], + ) From 58b4d36239b4d79137745938458d995db0a206e1 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 13:27:34 -0600 Subject: [PATCH 096/137] Tweak ingest config based on load tests --- .../provider_update_ingest_handler.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 37d39f91b..099debfb5 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -82,6 +82,14 @@ def __init__( vpc=vpc_stack.vpc, vpc_subnets=vpc_subnets, security_groups=[vpc_stack.lambda_security_group], + # We set a limit to the number of concurrent executions that can be started before being throttled. + # This protects us in several ways. First, it prevents ingest from taking concurrent execution count from + # our api lambdas, which if left unchecked could cause them to get throttled if we hit our account limit + # (currently at the default of 1000). It also prevents the OpenSearch Domain from getting slammed during + # high volume. This reserved limit can result in messages waiting a bit longer on the queue during high + # volume while the reserved executions complete their tasks before grabbing the next batch. This can be + # adjusted as needed, but based on initial load testing this seems like a reasonable limit. + reserved_concurrent_executions=25, alarm_topic=alarm_topic, ) @@ -97,15 +105,15 @@ def __init__( visibility_timeout=Duration.minutes(15), # Retention period for the source queue (these should be processed fairly quickly, but setting this to # account for retries) - retention_period=Duration.hours(4), + retention_period=Duration.hours(2), # OpenSearch recommends performing bulk indexing with sizes between 5 - 15 MB per operation. # see https://www.elastic.co/guide/en/elasticsearch/guide/2.x/indexing-performance.html#_using_and_sizing_bulk_requests # A basic provider document without any additional records (privileges, adverse actions, etc.) is - # around 2KB on average. We expect these provider documents to grow over time as providers accumulate - # privileges and other records. Setting a batch size of 2000 places the initial bulk operations around - # 4MB max size per request (2KB * 2000 = 4 MB). This puts us below that range but provides headroom for + # around 2 KB on average. We expect these provider documents to grow over time as providers accumulate + # privileges and other records. Setting a batch size of 3000 places the initial bulk operations around + # 6 MB max size per request (2KB * 3000 = 6 MB). This puts us within that range and provides headroom for # these documents to grow over time, while still processing license uploads in a timely manner. - batch_size=2000, + batch_size=3000, # Batching window to allow multiple events for the same provider to be processed together max_batching_window=Duration.seconds(15), # Max receive count = total attempts before DLQ (1 initial + 2 retries = 3 total) From 7e2c26d5afbb1fb8cc63bdef22e87274fb114f1b Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 13:53:26 -0600 Subject: [PATCH 097/137] PR feedback --- .../python/search/handlers/provider_update_ingest.py | 5 +++-- .../lambdas/python/search/tests/__init__.py | 1 + .../lambdas/python/search/tests/function/__init__.py | 2 +- .../provider_update_ingest_handler.py | 7 ++++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index bbbee232a..027ae3134 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -56,7 +56,7 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: image = stream_record.get('dynamodb', {}).get('NewImage') or stream_record.get('dynamodb', {}).get('OldImage') if not image: - logger.error('Record has no image data', message_id=message_id, record=stream_record) + logger.error('Record has no image data', message_id=message_id) continue # Extract compact and providerId from the DynamoDB image @@ -157,7 +157,8 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: error=str(e), ) # Mark all providers in this compact as failed - for provider_id in provider_ids: + document_provider_ids = [document['providerId'] for document in documents_to_index] + for provider_id in document_provider_ids: failed_providers[compact].add(provider_id) # Bulk delete providers that no longer exist diff --git a/backend/compact-connect/lambdas/python/search/tests/__init__.py b/backend/compact-connect/lambdas/python/search/tests/__init__.py index 53d1a14ab..d00640792 100644 --- a/backend/compact-connect/lambdas/python/search/tests/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/__init__.py @@ -22,6 +22,7 @@ def setUpClass(cls): 'PROV_DATE_OF_UPDATE_INDEX_NAME': 'providerDateOfUpdate', 'PROV_FAM_GIV_MID_INDEX_NAME': 'providerFamGivMid', 'LICENSE_GSI_NAME': 'licenseGSI', + 'LICENSE_UPLOAD_DATE_INDEX_NAME': 'licenseUploadDateGSI', 'OPENSEARCH_HOST_ENDPOINT': 'vpc-providersearchd-5bzuqxhpxffk-w6dkpddu.us-east-1.es.amazonaws.com', 'EXPORT_RESULTS_BUCKET_NAME': 'test-export-results-bucket', 'JURISDICTIONS': json.dumps( diff --git a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py index 9044cc9c8..3d3f139f9 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/__init__.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/__init__.py @@ -79,7 +79,7 @@ def create_provider_table(self): 'Projection': {'ProjectionType': 'ALL'}, }, { - 'IndexName': 'licenseUploadDateGSI', + 'IndexName': os.environ['LICENSE_UPLOAD_DATE_INDEX_NAME'], 'KeySchema': [ {'AttributeName': 'licenseUploadDateGSIPK', 'KeyType': 'HASH'}, {'AttributeName': 'licenseUploadDateGSISK', 'KeyType': 'RANGE'}, diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 099debfb5..05990a2f9 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -78,7 +78,7 @@ def __init__( }, # Allow enough time for processing large batches timeout=Duration.minutes(10), - memory_size=512, + memory_size=1024, vpc=vpc_stack.vpc, vpc_subnets=vpc_subnets, security_groups=[vpc_stack.lambda_security_group], @@ -87,8 +87,9 @@ def __init__( # our api lambdas, which if left unchecked could cause them to get throttled if we hit our account limit # (currently at the default of 1000). It also prevents the OpenSearch Domain from getting slammed during # high volume. This reserved limit can result in messages waiting a bit longer on the queue during high - # volume while the reserved executions complete their tasks before grabbing the next batch. This can be - # adjusted as needed, but based on initial load testing this seems like a reasonable limit. + # volume while the reserved executions complete their tasks before grabbing the next batch. We have an alert + # in place to fire if this lambda is ever throttled. This limit can be adjusted as needed, but based on + # initial load testing this seems like a reasonable limit. reserved_concurrent_executions=25, alarm_topic=alarm_topic, ) From 2f278ec73f7feb647f9934cfa2a6d51c91b4a011 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 15:26:50 -0600 Subject: [PATCH 098/137] Update bootstrap stack permission boundary to allow new services --- .../resources/bootstrap-stack-beta.yaml | 78 +++++++++++++++++++ .../resources/bootstrap-stack-prod.yaml | 78 +++++++++++++++++++ .../resources/bootstrap-stack-test.yaml | 78 +++++++++++++++++++ 3 files changed, 234 insertions(+) diff --git a/backend/compact-connect/resources/bootstrap-stack-beta.yaml b/backend/compact-connect/resources/bootstrap-stack-beta.yaml index 827283393..393bac9be 100644 --- a/backend/compact-connect/resources/bootstrap-stack-beta.yaml +++ b/backend/compact-connect/resources/bootstrap-stack-beta.yaml @@ -614,6 +614,10 @@ Resources: - kms:* # AWS Lambda - lambda:* + # Amazon OpenSearch Service + - es:* + # Amazon EventBridge Pipes + - pipes:* # Amazon Route 53 - route53:* # Amazon S3 @@ -637,6 +641,80 @@ Resources: - sts:GetCallerIdentity - sts:TagSession Resource: "*" + # VPC Resources - Restricted EC2 permissions for VPC networking only + - Sid: AllowVpcNetworkingResources + Effect: Allow + Action: + # VPC management + - ec2:CreateVpc + - ec2:DeleteVpc + - ec2:DescribeVpcs + - ec2:ModifyVpcAttribute + - ec2:DescribeVpcAttribute + # Subnet management + - ec2:CreateSubnet + - ec2:DeleteSubnet + - ec2:DescribeSubnets + - ec2:ModifySubnetAttribute + # Route table management + - ec2:CreateRouteTable + - ec2:DeleteRouteTable + - ec2:DescribeRouteTables + - ec2:AssociateRouteTable + - ec2:DisassociateRouteTable + - ec2:CreateRoute + - ec2:DeleteRoute + - ec2:ReplaceRoute + # Security group management + - ec2:CreateSecurityGroup + - ec2:DeleteSecurityGroup + - ec2:DescribeSecurityGroups + - ec2:DescribeSecurityGroupRules + - ec2:AuthorizeSecurityGroupIngress + - ec2:AuthorizeSecurityGroupEgress + - ec2:RevokeSecurityGroupIngress + - ec2:RevokeSecurityGroupEgress + - ec2:UpdateSecurityGroupRuleDescriptionsIngress + - ec2:UpdateSecurityGroupRuleDescriptionsEgress + # VPC Endpoint management + - ec2:CreateVpcEndpoint + - ec2:DeleteVpcEndpoints + - ec2:DescribeVpcEndpoints + - ec2:ModifyVpcEndpoint + - ec2:DescribeVpcEndpointServices + - ec2:DescribePrefixLists + # VPC Flow Logs + - ec2:CreateFlowLogs + - ec2:DeleteFlowLogs + - ec2:DescribeFlowLogs + # Tagging + - ec2:CreateTags + - ec2:DeleteTags + # General describe operations needed by CDK + - ec2:DescribeAvailabilityZones + - ec2:DescribeNetworkInterfaces + Resource: "*" + # Explicitly deny EC2 instance operations + - Sid: DenyEc2InstanceOperations + Effect: Deny + Action: + - ec2:RunInstances + - ec2:StartInstances + - ec2:StopInstances + - ec2:TerminateInstances + - ec2:RebootInstances + - ec2:CreateImage + - ec2:RegisterImage + - ec2:ImportInstance + - ec2:ImportImage + - ec2:RequestSpotInstances + - ec2:RequestSpotFleet + - ec2:ModifyInstanceAttribute + - ec2:ModifySpotFleetRequest + - ec2:CreateLaunchTemplate + - ec2:CreateLaunchTemplateVersion + - ec2:ModifyLaunchTemplate + Resource: "*" - Sid: DenyDangerousActions Effect: Deny Action: diff --git a/backend/compact-connect/resources/bootstrap-stack-prod.yaml b/backend/compact-connect/resources/bootstrap-stack-prod.yaml index a18b8d50d..29e6a3959 100644 --- a/backend/compact-connect/resources/bootstrap-stack-prod.yaml +++ b/backend/compact-connect/resources/bootstrap-stack-prod.yaml @@ -614,6 +614,10 @@ Resources: - kms:* # AWS Lambda - lambda:* + # Amazon OpenSearch Service + - es:* + # Amazon EventBridge Pipes + - pipes:* # Amazon Route 53 - route53:* # Amazon S3 @@ -637,6 +641,80 @@ Resources: - sts:GetCallerIdentity - sts:TagSession Resource: "*" + # VPC Resources - Restricted EC2 permissions for VPC networking only + - Sid: AllowVpcNetworkingResources + Effect: Allow + Action: + # VPC management + - ec2:CreateVpc + - ec2:DeleteVpc + - ec2:DescribeVpcs + - ec2:ModifyVpcAttribute + - ec2:DescribeVpcAttribute + # Subnet management + - ec2:CreateSubnet + - ec2:DeleteSubnet + - ec2:DescribeSubnets + - ec2:ModifySubnetAttribute + # Route table management + - ec2:CreateRouteTable + - ec2:DeleteRouteTable + - ec2:DescribeRouteTables + - ec2:AssociateRouteTable + - ec2:DisassociateRouteTable + - ec2:CreateRoute + - ec2:DeleteRoute + - ec2:ReplaceRoute + # Security group management + - ec2:CreateSecurityGroup + - ec2:DeleteSecurityGroup + - ec2:DescribeSecurityGroups + - ec2:DescribeSecurityGroupRules + - ec2:AuthorizeSecurityGroupIngress + - ec2:AuthorizeSecurityGroupEgress + - ec2:RevokeSecurityGroupIngress + - ec2:RevokeSecurityGroupEgress + - ec2:UpdateSecurityGroupRuleDescriptionsIngress + - ec2:UpdateSecurityGroupRuleDescriptionsEgress + # VPC Endpoint management + - ec2:CreateVpcEndpoint + - ec2:DeleteVpcEndpoints + - ec2:DescribeVpcEndpoints + - ec2:ModifyVpcEndpoint + - ec2:DescribeVpcEndpointServices + - ec2:DescribePrefixLists + # VPC Flow Logs + - ec2:CreateFlowLogs + - ec2:DeleteFlowLogs + - ec2:DescribeFlowLogs + # Tagging + - ec2:CreateTags + - ec2:DeleteTags + # General describe operations needed by CDK + - ec2:DescribeAvailabilityZones + - ec2:DescribeNetworkInterfaces + Resource: "*" + # Explicitly deny EC2 instance operations + - Sid: DenyEc2InstanceOperations + Effect: Deny + Action: + - ec2:RunInstances + - ec2:StartInstances + - ec2:StopInstances + - ec2:TerminateInstances + - ec2:RebootInstances + - ec2:CreateImage + - ec2:RegisterImage + - ec2:ImportInstance + - ec2:ImportImage + - ec2:RequestSpotInstances + - ec2:RequestSpotFleet + - ec2:ModifyInstanceAttribute + - ec2:ModifySpotFleetRequest + - ec2:CreateLaunchTemplate + - ec2:CreateLaunchTemplateVersion + - ec2:ModifyLaunchTemplate + Resource: "*" - Sid: DenyDangerousActions Effect: Deny Action: diff --git a/backend/compact-connect/resources/bootstrap-stack-test.yaml b/backend/compact-connect/resources/bootstrap-stack-test.yaml index f7aa1b5f8..ae16a6dbc 100644 --- a/backend/compact-connect/resources/bootstrap-stack-test.yaml +++ b/backend/compact-connect/resources/bootstrap-stack-test.yaml @@ -614,6 +614,10 @@ Resources: - kms:* # AWS Lambda - lambda:* + # Amazon OpenSearch Service + - es:* + # Amazon EventBridge Pipes + - pipes:* # Amazon Route 53 - route53:* # Amazon S3 @@ -637,6 +641,80 @@ Resources: - sts:GetCallerIdentity - sts:TagSession Resource: "*" + # VPC Resources - Restricted EC2 permissions for VPC networking only + - Sid: AllowVpcNetworkingResources + Effect: Allow + Action: + # VPC management + - ec2:CreateVpc + - ec2:DeleteVpc + - ec2:DescribeVpcs + - ec2:ModifyVpcAttribute + - ec2:DescribeVpcAttribute + # Subnet management + - ec2:CreateSubnet + - ec2:DeleteSubnet + - ec2:DescribeSubnets + - ec2:ModifySubnetAttribute + # Route table management + - ec2:CreateRouteTable + - ec2:DeleteRouteTable + - ec2:DescribeRouteTables + - ec2:AssociateRouteTable + - ec2:DisassociateRouteTable + - ec2:CreateRoute + - ec2:DeleteRoute + - ec2:ReplaceRoute + # Security group management + - ec2:CreateSecurityGroup + - ec2:DeleteSecurityGroup + - ec2:DescribeSecurityGroups + - ec2:DescribeSecurityGroupRules + - ec2:AuthorizeSecurityGroupIngress + - ec2:AuthorizeSecurityGroupEgress + - ec2:RevokeSecurityGroupIngress + - ec2:RevokeSecurityGroupEgress + - ec2:UpdateSecurityGroupRuleDescriptionsIngress + - ec2:UpdateSecurityGroupRuleDescriptionsEgress + # VPC Endpoint management + - ec2:CreateVpcEndpoint + - ec2:DeleteVpcEndpoints + - ec2:DescribeVpcEndpoints + - ec2:ModifyVpcEndpoint + - ec2:DescribeVpcEndpointServices + - ec2:DescribePrefixLists + # VPC Flow Logs + - ec2:CreateFlowLogs + - ec2:DeleteFlowLogs + - ec2:DescribeFlowLogs + # Tagging + - ec2:CreateTags + - ec2:DeleteTags + # General describe operations needed by CDK + - ec2:DescribeAvailabilityZones + - ec2:DescribeNetworkInterfaces + Resource: "*" + # Explicitly deny EC2 instance operations + - Sid: DenyEc2InstanceOperations + Effect: Deny + Action: + - ec2:RunInstances + - ec2:StartInstances + - ec2:StopInstances + - ec2:TerminateInstances + - ec2:RebootInstances + - ec2:CreateImage + - ec2:RegisterImage + - ec2:ImportInstance + - ec2:ImportImage + - ec2:RequestSpotInstances + - ec2:RequestSpotFleet + - ec2:ModifyInstanceAttribute + - ec2:ModifySpotFleetRequest + - ec2:CreateLaunchTemplate + - ec2:CreateLaunchTemplateVersion + - ec2:ModifyLaunchTemplate + Resource: "*" - Sid: DenyDangerousActions Effect: Deny Action: From 59b726612c6f53a39d04a74a81e072465b458ce7 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Thu, 11 Dec 2025 17:15:19 -0600 Subject: [PATCH 099/137] Add opensearch service link role to bootstrap templates --- .../compact-connect/resources/bootstrap-stack-beta.yaml | 9 +++++++++ .../compact-connect/resources/bootstrap-stack-prod.yaml | 9 +++++++++ .../compact-connect/resources/bootstrap-stack-test.yaml | 9 +++++++++ backend/multi-account/README.md | 2 ++ 4 files changed, 29 insertions(+) diff --git a/backend/compact-connect/resources/bootstrap-stack-beta.yaml b/backend/compact-connect/resources/bootstrap-stack-beta.yaml index 393bac9be..323675af7 100644 --- a/backend/compact-connect/resources/bootstrap-stack-beta.yaml +++ b/backend/compact-connect/resources/bootstrap-stack-beta.yaml @@ -107,6 +107,15 @@ Conditions: - Ref: PublicAccessBlockConfiguration Resources: + # Service-linked role for Amazon OpenSearch Service to access VPC resources + # This role allows OpenSearch to create and manage ENIs in VPCs + # NOTE: If this role already exists in the account, remove this resource from the template + OpenSearchServiceLinkedRole: + Type: AWS::IAM::ServiceLinkedRole + Properties: + AWSServiceName: opensearchservice.amazonaws.com + Description: Service-linked role for Amazon OpenSearch Service VPC access + FileAssetsBucketEncryptionKey: Type: AWS::KMS::Key Properties: diff --git a/backend/compact-connect/resources/bootstrap-stack-prod.yaml b/backend/compact-connect/resources/bootstrap-stack-prod.yaml index 29e6a3959..abb7a4331 100644 --- a/backend/compact-connect/resources/bootstrap-stack-prod.yaml +++ b/backend/compact-connect/resources/bootstrap-stack-prod.yaml @@ -107,6 +107,15 @@ Conditions: - Ref: PublicAccessBlockConfiguration Resources: + # Service-linked role for Amazon OpenSearch Service to access VPC resources + # This role allows OpenSearch to create and manage ENIs in VPCs + # NOTE: If this role already exists in the account, remove this resource from the template + OpenSearchServiceLinkedRole: + Type: AWS::IAM::ServiceLinkedRole + Properties: + AWSServiceName: opensearchservice.amazonaws.com + Description: Service-linked role for Amazon OpenSearch Service VPC access + FileAssetsBucketEncryptionKey: Type: AWS::KMS::Key Properties: diff --git a/backend/compact-connect/resources/bootstrap-stack-test.yaml b/backend/compact-connect/resources/bootstrap-stack-test.yaml index ae16a6dbc..8f23c9dce 100644 --- a/backend/compact-connect/resources/bootstrap-stack-test.yaml +++ b/backend/compact-connect/resources/bootstrap-stack-test.yaml @@ -107,6 +107,15 @@ Conditions: - Ref: PublicAccessBlockConfiguration Resources: + # Service-linked role for Amazon OpenSearch Service to access VPC resources + # This role allows OpenSearch to create and manage ENIs in VPCs + # NOTE: If this role already exists in the account, remove this resource from the template + OpenSearchServiceLinkedRole: + Type: AWS::IAM::ServiceLinkedRole + Properties: + AWSServiceName: opensearchservice.amazonaws.com + Description: Service-linked role for Amazon OpenSearch Service VPC access + FileAssetsBucketEncryptionKey: Type: AWS::KMS::Key Properties: diff --git a/backend/multi-account/README.md b/backend/multi-account/README.md index f1f308503..1be0424b6 100644 --- a/backend/multi-account/README.md +++ b/backend/multi-account/README.md @@ -216,6 +216,8 @@ For enhanced security, use the secure bootstrap templates that trust only specif --cloudformation-execution-policies 'arn:aws:iam::aws:policy/AdministratorAccess' ``` +**Note on OpenSearch Service-Linked Role**: The bootstrap templates include creation of a service-linked role for Amazon OpenSearch Service VPC access. This role can only exist once per AWS account. If the role already exists in the account (e.g., from previous OpenSearch usage), the bootstrap deployment will fail. In that case, simply remove the `OpenSearchServiceLinkedRole` resource from the template before running the bootstrap command. + ### Bootstrap the secondary accounts See ./backups/README for instructions on setting up the secondary accounts and backup resources. From 5d3b007df5d4a99e2e92099ccfc07cdf1a7b2bb9 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 12 Dec 2025 09:48:45 -0600 Subject: [PATCH 100/137] update multi-account project requirements to latest --- .../backups/requirements-dev.txt | 32 ++++++------- .../multi-account/backups/requirements.txt | 16 +++---- .../control-tower/requirements-dev.txt | 46 ++++++++++--------- .../control-tower/requirements.txt | 18 ++++---- .../log-aggregation/requirements-dev.txt | 8 ++-- .../log-aggregation/requirements.txt | 16 +++---- 6 files changed, 69 insertions(+), 67 deletions(-) diff --git a/backend/multi-account/backups/requirements-dev.txt b/backend/multi-account/backups/requirements-dev.txt index 88d4637ce..9a922a328 100644 --- a/backend/multi-account/backups/requirements-dev.txt +++ b/backend/multi-account/backups/requirements-dev.txt @@ -1,27 +1,27 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --no-emit-index-url backups/requirements-dev.in +# pip-compile --no-emit-index-url --no-strip-extras backups/requirements-dev.in # -boto3==1.40.33 +boto3==1.42.8 # via moto -botocore==1.40.33 +botocore==1.42.8 # via # boto3 # moto # s3transfer -certifi==2025.8.3 +certifi==2025.11.12 # via requests cffi==2.0.0 # via cryptography -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 # via requests -cryptography==46.0.1 +cryptography==46.0.3 # via moto -idna==3.10 +idna==3.11 # via requests -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest jinja2==3.1.6 # via moto @@ -29,11 +29,11 @@ jmespath==1.0.1 # via # boto3 # botocore -markupsafe==3.0.2 +markupsafe==3.0.3 # via # jinja2 # werkzeug -moto==5.1.12 +moto==5.1.18 # via -r backups/requirements-dev.in packaging==25.0 # via pytest @@ -43,13 +43,13 @@ pycparser==2.23 # via cffi pygments==2.19.2 # via pytest -pytest==8.4.2 +pytest==9.0.2 # via -r backups/requirements-dev.in python-dateutil==2.9.0.post0 # via # botocore # moto -pyyaml==6.0.2 +pyyaml==6.0.3 # via responses requests==2.32.5 # via @@ -57,16 +57,16 @@ requests==2.32.5 # responses responses==0.25.8 # via moto -s3transfer==0.14.0 +s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.5.0 +urllib3==2.6.2 # via # botocore # requests # responses -werkzeug==3.1.3 +werkzeug==3.1.4 # via moto xmltodict==1.0.2 # via moto diff --git a/backend/multi-account/backups/requirements.txt b/backend/multi-account/backups/requirements.txt index 536510e88..8dfcf26f0 100644 --- a/backend/multi-account/backups/requirements.txt +++ b/backend/multi-account/backups/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --no-emit-index-url backups/requirements.in +# pip-compile --no-emit-index-url --no-strip-extras backups/requirements.in # -attrs==25.3.0 +attrs==25.4.0 # via # cattrs # jsii @@ -12,19 +12,19 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-cloud-assembly-schema==48.10.0 +aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.215.0 +aws-cdk-lib==2.232.1 # via -r backups/requirements.in -cattrs==25.2.0 +cattrs==25.3.0 # via jsii -constructs==10.4.2 +constructs==10.4.4 # via # -r backups/requirements.in # aws-cdk-lib importlib-resources==6.5.2 # via jsii -jsii==1.114.1 +jsii==1.121.0 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 diff --git a/backend/multi-account/control-tower/requirements-dev.txt b/backend/multi-account/control-tower/requirements-dev.txt index 21864a695..81ca07257 100644 --- a/backend/multi-account/control-tower/requirements-dev.txt +++ b/backend/multi-account/control-tower/requirements-dev.txt @@ -1,36 +1,36 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --no-emit-index-url control-tower/requirements-dev.in +# pip-compile --no-emit-index-url --no-strip-extras control-tower/requirements-dev.in # boolean-py==5.0 # via license-expression build==1.3.0 # via pip-tools -cachecontrol[filecache]==0.14.3 +cachecontrol[filecache]==0.14.4 # via # cachecontrol # pip-audit -certifi==2025.8.3 +certifi==2025.11.12 # via requests -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 # via requests -click==8.2.1 +click==8.3.1 # via pip-tools -coverage[toml]==7.10.6 +coverage[toml]==7.13.0 # via # -r control-tower/requirements-dev.in # pytest-cov -cyclonedx-python-lib==9.1.0 +cyclonedx-python-lib==11.6.0 # via pip-audit defusedxml==0.7.1 # via py-serializable -filelock==3.19.1 +filelock==3.20.0 # via cachecontrol -idna==3.10 +idna==3.11 # via requests -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest license-expression==30.4.4 # via cyclonedx-python-lib @@ -38,9 +38,9 @@ markdown-it-py==4.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -msgpack==1.1.1 +msgpack==1.1.2 # via cachecontrol -packageurl-python==0.17.5 +packageurl-python==0.17.6 # via cyclonedx-python-lib packaging==25.0 # via @@ -50,13 +50,13 @@ packaging==25.0 # pytest pip-api==0.0.34 # via pip-audit -pip-audit==2.9.0 +pip-audit==2.10.0 # via -r control-tower/requirements-dev.in pip-requirements-parser==32.0.1 # via pip-audit -pip-tools==7.5.0 +pip-tools==7.5.2 # via -r control-tower/requirements-dev.in -platformdirs==4.4.0 +platformdirs==4.5.1 # via pip-audit pluggy==1.6.0 # via @@ -68,13 +68,13 @@ pygments==2.19.2 # via # pytest # rich -pyparsing==3.2.4 +pyparsing==3.2.5 # via pip-requirements-parser pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.4.2 +pytest==9.0.2 # via # -r control-tower/requirements-dev.in # pytest-cov @@ -84,15 +84,17 @@ requests==2.32.5 # via # cachecontrol # pip-audit -rich==14.1.0 +rich==14.2.0 # via pip-audit -ruff==0.13.0 +ruff==0.14.9 # via -r control-tower/requirements-dev.in sortedcontainers==2.4.0 # via cyclonedx-python-lib -toml==0.10.2 +tomli==2.3.0 # via pip-audit -urllib3==2.5.0 +tomli-w==1.2.0 + # via pip-audit +urllib3==2.6.2 # via requests wheel==0.45.1 # via pip-tools diff --git a/backend/multi-account/control-tower/requirements.txt b/backend/multi-account/control-tower/requirements.txt index 2f24b1a7a..daaa7bb30 100644 --- a/backend/multi-account/control-tower/requirements.txt +++ b/backend/multi-account/control-tower/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --no-emit-index-url control-tower/requirements.in +# pip-compile --no-emit-index-url --no-strip-extras control-tower/requirements.in # -attrs==25.3.0 +attrs==25.4.0 # via # cattrs # jsii @@ -12,24 +12,24 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-cloud-assembly-schema==48.10.0 +aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.215.0 +aws-cdk-lib==2.232.1 # via # -r control-tower/requirements.in # cdk-nag -cattrs==25.2.0 +cattrs==25.3.0 # via jsii -cdk-nag==2.37.29 +cdk-nag==2.37.55 # via -r control-tower/requirements.in -constructs==10.4.2 +constructs==10.4.4 # via # -r control-tower/requirements.in # aws-cdk-lib # cdk-nag importlib-resources==6.5.2 # via jsii -jsii==1.114.1 +jsii==1.121.0 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 diff --git a/backend/multi-account/log-aggregation/requirements-dev.txt b/backend/multi-account/log-aggregation/requirements-dev.txt index e002df771..255223201 100644 --- a/backend/multi-account/log-aggregation/requirements-dev.txt +++ b/backend/multi-account/log-aggregation/requirements-dev.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --no-emit-index-url log-aggregation/requirements-dev.in +# pip-compile --no-emit-index-url --no-strip-extras log-aggregation/requirements-dev.in # -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest packaging==25.0 # via pytest @@ -12,5 +12,5 @@ pluggy==1.6.0 # via pytest pygments==2.19.2 # via pytest -pytest==8.4.2 +pytest==9.0.2 # via -r log-aggregation/requirements-dev.in diff --git a/backend/multi-account/log-aggregation/requirements.txt b/backend/multi-account/log-aggregation/requirements.txt index 338d35ef1..c29c4a7b6 100644 --- a/backend/multi-account/log-aggregation/requirements.txt +++ b/backend/multi-account/log-aggregation/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --no-emit-index-url log-aggregation/requirements.in +# pip-compile --no-emit-index-url --no-strip-extras log-aggregation/requirements.in # -attrs==25.3.0 +attrs==25.4.0 # via # cattrs # jsii @@ -12,19 +12,19 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-cloud-assembly-schema==48.10.0 +aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.215.0 +aws-cdk-lib==2.232.1 # via -r log-aggregation/requirements.in -cattrs==25.2.0 +cattrs==25.3.0 # via jsii -constructs==10.4.2 +constructs==10.4.4 # via # -r log-aggregation/requirements.in # aws-cdk-lib importlib-resources==6.5.2 # via jsii -jsii==1.114.1 +jsii==1.121.0 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 From 9ac7c161e6a8b44bb73a8d63452836d98105be6e Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 12 Dec 2025 11:14:56 -0600 Subject: [PATCH 101/137] Add additional domain config settings based on testing --- .../provider_search_domain.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index f7a17b681..a8baeb6f1 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -1,7 +1,7 @@ from aws_cdk import Duration, Fn, RemovalPolicy from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Metric, TreatMissingData from aws_cdk.aws_cloudwatch_actions import SnsAction -from aws_cdk.aws_ec2 import SubnetSelection, SubnetType +from aws_cdk.aws_ec2 import EbsDeviceVolumeType, SubnetSelection, SubnetType from aws_cdk.aws_iam import Effect, IRole, PolicyStatement, ServicePrincipal from aws_cdk.aws_kms import Key from aws_cdk.aws_logs import LogGroup, ResourcePolicy, RetentionDays @@ -13,7 +13,7 @@ EngineVersion, LoggingOptions, TLSSecurityPolicy, - ZoneAwarenessConfig, + ZoneAwarenessConfig, WindowStartTime, ) from aws_cdk.aws_sns import ITopic from cdk_nag import NagSuppressions @@ -165,6 +165,11 @@ def __init__( # both the search API and search persistent stacks needed to be destroyed, redeployed, and re-indexed. version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, + enable_auto_software_update=True, + enable_version_upgrade=True, + # We set the off-peak window to 9AM UTC (1AM PST) + # this determines when automatic updates are performed on the domain. + off_peak_window_start=WindowStartTime(hours=9, minutes=0), # VPC configuration for network isolation vpc=vpc_stack.vpc, vpc_subnets=[self.vpc_subnets], @@ -172,7 +177,9 @@ def __init__( # EBS volume configuration ebs=EbsOptions( enabled=True, - volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, + volume_size=PROD_EBS_VOLUME_SIZE if environment_name != PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, + # this type is required for medium instances + volume_type=EbsDeviceVolumeType.GP3 ), # Encryption settings encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.encryption_key), From a5b222d7be30d36cc0c0787429b9193539229524 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 12 Dec 2025 12:59:19 -0600 Subject: [PATCH 102/137] Tweak master node instance type based on testing --- .../search_persistent_stack/provider_search_domain.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index a8baeb6f1..d34dc8943 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -296,11 +296,15 @@ def _get_capacity_config(self, environment_name: str) -> CapacityConfig: # 3 dedicated master nodes + 3 data nodes across 3 AZs with standby # Multi-AZ with standby does not support t3 instance types return CapacityConfig( - # Data nodes - m7g.medium provides 4 vCPUs and 8GB RAM + # Data nodes - m7g.medium provides 1 vCPU and 4GB RAM data_node_instance_type='m7g.medium.search', + # we require at least 3 data nodes and master nodes to support multi-az with standby + # for high availability data_nodes=3, # Dedicated master nodes for cluster management - master_node_instance_type='m7g.medium.search', + # r8g.medium provides 8GB RAM, which the master nodes + # need based on our domain size + master_node_instance_type='r8g.medium.search', master_nodes=3, # Multi-AZ with standby for high availability multi_az_with_standby_enabled=True, From 899a9a23165cb60a4af02bd22d22bc0c3bf63acf Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 12 Dec 2025 15:36:21 -0600 Subject: [PATCH 103/137] Fix PROD index shard configuration Also set dependency ordering so that domain is indexed before we attempt ingest --- .../search_persistent_stack/__init__.py | 4 +++ .../search_persistent_stack/index_manager.py | 25 ++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 4a209da89..ff35736a7 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -149,6 +149,8 @@ def __init__( encryption_key=self.opensearch_encryption_key, alarm_topic=persistent_stack.alarm_topic, ) + # don't deploy ingest resources until index manager has set proper index configuration + self.provider_update_ingest_handler.queue.node.add_dependency(self.index_manager_custom_resource) # Create the EventBridge Pipe to connect DynamoDB stream to SQS queue # This pipe reads from the provider table stream and sends events to the ingest handler's queue @@ -159,6 +161,8 @@ def __init__( target_queue=self.provider_update_ingest_handler.queue, encryption_key=self.opensearch_encryption_key, ) + # don't deploy ingest resources until index manager has set proper index configuration + self.provider_update_ingest_pipe.node.add_dependency(self.index_manager_custom_resource) # add log insights for provider ingest QueryDefinition( diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index 394ec5f98..aed3154f3 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -16,11 +16,12 @@ # Index configuration constants # Non-prod environments use a single data node, so no replicas are needed -# Production uses 3 data nodes across 3 AZs, so 1 replica ensures data availability NON_PROD_NUMBER_OF_SHARDS = 1 NON_PROD_NUMBER_OF_REPLICAS = 0 +# Production uses 3 data nodes across 3 AZs, so 1 primary and 2 replica ensures data availability +# if this is updated, the total of primary + replica shards must be a multiple of 3 PROD_NUMBER_OF_SHARDS = 1 -PROD_NUMBER_OF_REPLICAS = 1 +PROD_NUMBER_OF_REPLICAS = 2 class IndexManagerCustomResource(Construct): @@ -168,8 +169,6 @@ def __init__( ], ) - # Determine index configuration based on environment - number_of_shards, number_of_replicas = self._get_index_configuration(environment_name) # Create custom resource for managing indices # This custom resource will create versioned indices (e.g., 'compact_aslp_providers_v1') @@ -181,21 +180,7 @@ def __init__( resource_type='Custom::IndexManager', service_token=provider.service_token, properties={ - 'numberOfShards': number_of_shards, - 'numberOfReplicas': number_of_replicas, + 'numberOfShards': PROD_NUMBER_OF_SHARDS if environment_name == PROD_ENV_NAME else NON_PROD_NUMBER_OF_SHARDS, + 'numberOfReplicas': PROD_NUMBER_OF_REPLICAS if environment_name == PROD_ENV_NAME else NON_PROD_NUMBER_OF_REPLICAS, }, ) - - def _get_index_configuration(self, environment_name: str) -> tuple[int, int]: - """ - Determine OpenSearch index configuration based on environment. - - Non-prod environments use a single data node, so no replicas are needed. - Production uses 3 data nodes across 3 AZs, so 1 replica ensures data availability. - - :param environment_name: The deployment environment name - :return: Tuple of (number_of_shards, number_of_replicas) - """ - if environment_name == PROD_ENV_NAME: - return PROD_NUMBER_OF_SHARDS, PROD_NUMBER_OF_REPLICAS - return NON_PROD_NUMBER_OF_SHARDS, NON_PROD_NUMBER_OF_REPLICAS From 46f0045d21c073a4e805d71fd02f35a72f69d3f1 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 12 Dec 2025 15:38:33 -0600 Subject: [PATCH 104/137] Set ingest pipeline to latest starting position Given that we will need to perform a full ingest using the populate lambda, we don't need to perform extra effort of grabbing the last 24 hours worth of updates when we initially deploy --- .../search_persistent_stack/provider_update_ingest_pipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py index 677e83ceb..4a316fa3b 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py @@ -83,9 +83,9 @@ def __init__( target=target_queue.queue_arn, source_parameters=CfnPipe.PipeSourceParametersProperty( dynamo_db_stream_parameters=CfnPipe.PipeSourceDynamoDBStreamParametersProperty( - # 'TRIM_HORIZON' starts processing from the earliest - # available stream record (oldest data in the DynamoDB stream, up to 24 hours ago) - starting_position='TRIM_HORIZON', + # 'LATEST' starts processing from the latest available stream record + # from the moment the pipe is created + starting_position='LATEST', # send everything to SQS as it arrives batch_size=1, ), From eaa5406605cd2f95aadbe136adab63b6d130224c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Fri, 12 Dec 2025 17:34:25 -0600 Subject: [PATCH 105/137] Add upgrade strategy notes based on findings from testing --- .../search_persistent_stack/__init__.py | 17 ++++------ .../provider_search_domain.py | 31 +++++++++++++------ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index ff35736a7..4cafeacd1 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -24,16 +24,11 @@ class SearchPersistentStack(AppStack): - Node-to-node encryption and HTTPS enforcement - Environment-specific instance sizing and cluster configuration - Instance sizing by environment: - - Non-prod (sandbox/test/beta): t3.small.search, 1 node - - Prod: m7g.medium.search, 3 master + 3 data nodes (with standby) - - IMPORTANT NOTE: Updating the OpenSearch domain may require a blue/green deployment, which is known to get stuck - on occasion requiring AWS support intervention (every time we attempted to update the engine version during - development, the deployment never completed). If you intend to update any field that will require a blue/green - deployment as described here: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html - Note that worst case scenario, you may have to delete the entire stack, re-deploy it, and re-index all the data from - the provider table. In light of this, DO NOT place any resources in this stack that should never be deleted. + IMPORTANT NOTE: Avoid updating the OpenSearch domain in a way that requires a blue/green deployment, + which is known to get stuck. See provider_search_domain.py for detailed upgrade notes, root causes, + and recovery steps. Note that worst case scenario, you may have to delete the entire stack, re-deploy it, and + re-index all the data from the provider table. In light of this, DO NOT place any resources in this stack that + should never be deleted. """ def __init__( @@ -150,7 +145,7 @@ def __init__( alarm_topic=persistent_stack.alarm_topic, ) # don't deploy ingest resources until index manager has set proper index configuration - self.provider_update_ingest_handler.queue.node.add_dependency(self.index_manager_custom_resource) + self.provider_update_ingest_handler.node.add_dependency(self.index_manager_custom_resource) # Create the EventBridge Pipe to connect DynamoDB stream to SQS queue # This pipe reads from the provider table stream and sends events to the ingest handler's queue diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index d34dc8943..e8f8b05a1 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -154,15 +154,26 @@ def __init__( self.domain = Domain( self, 'Domain', - # IMPORTANT NOTE: updating the engine version requires a blue/green deployment, which is known to get stuck - # on occasion requiring AWS support intervention. If you intend to update this field, or any other field - # that will require a blue/green deployment as described here: + # IMPORTANT NOTE: updating the engine version requires a blue/green deployment, which has consistently + # failed to complete in both production and non-production environments due to failed dashboard health + # checks. We suspect this is because of the 'rest.action.multi.allow_explicit_index: false' setting + # interfering with dashboard internal multi-index operations during upgrades. If you intend to update + # this field, or any other field that will require a blue/green deployment as described here: # https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html - # You should consider working with stakeholders to schedule a maintenance window during low-traffic periods - # where advanced search may become inaccessible during the update. During development, we found that if a - # blue/green deployment became stuck, the search endpoints were still able to serve data, but the - # CloudFormation deployment would fail waiting for the domain to become active. Worst case scenario, - # both the search API and search persistent stacks needed to be destroyed, redeployed, and re-indexed. + # You should consider the following migration process instead: + # 1. Deploy a NEW domain with the target version (use different construct ID) + # 2. Reindex data from provider table using PopulateProviderDocumentsHandler + # 3. Update search API to point to new domain + # 4. Decommission old domain + # This approach provides full rollback capability and avoids blue/green issues entirely. + # + # During significant upgrades, consider working with stakeholders to schedule a maintenance window during + # low-traffic periods where advanced search may become inaccessible during the update. During development, + # we found that if a blue/green deployment became stuck, the search endpoints were still able to serve data, + # but the CloudFormation deployment would fail waiting for the domain to become active. In such cases you + # may have to work with AWS support to get it out of that state. Worst case scenario, both the search API + # and search persistent stacks will need to be destroyed, redeployed, and re-indexed, hence why we recommend + # you create an entirely different domain and avoid the blue/green deployment altogether. version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, enable_auto_software_update=True, @@ -177,7 +188,7 @@ def __init__( # EBS volume configuration ebs=EbsOptions( enabled=True, - volume_size=PROD_EBS_VOLUME_SIZE if environment_name != PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, + volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, # this type is required for medium instances volume_type=EbsDeviceVolumeType.GP3 ), @@ -286,7 +297,7 @@ def _get_capacity_config(self, environment_name: str) -> CapacityConfig: Determine OpenSearch cluster capacity configuration based on environment. Non-prod (sandbox, test, beta, etc.): Single t3.small.search node - Prod: 3 dedicated master (m7g.medium.search) + 3 data nodes (m7g.medium.search) with standby + Prod: 3 dedicated master (r8g.medium.search) + 3 data nodes (m7g.medium.search) with standby :param environment_name: The deployment environment name :return: CapacityConfig with appropriate instance types and counts From ec0d65f8c0abe08ba03617d9273fcfea7641628a Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 09:27:17 -0600 Subject: [PATCH 106/137] PR feedback --- .../search/handlers/provider_update_ingest.py | 1 + .../stacks/search_persistent_stack/__init__.py | 1 + .../search_persistent_stack/index_manager.py | 9 ++++++--- .../provider_search_domain.py | 14 ++++++-------- .../provider_update_ingest_handler.py | 4 ++-- .../provider_update_ingest_pipe.py | 7 ++++--- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index 027ae3134..de6b8b2d3 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -11,6 +11,7 @@ to the handler at once, enabling batch processing and deduplication. The handler returns batchItemFailures directly for partial success handling. """ + from boto3.dynamodb.types import TypeDeserializer from cc_common.config import config, logger from cc_common.exceptions import CCInternalException, CCNotFoundException diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 4cafeacd1..7cc9d4b44 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -75,6 +75,7 @@ def __init__( self, 'ProviderSearchDomain', environment_name=environment_name, + region=self.region, vpc_stack=vpc_stack, compact_abbreviations=persistent_stack.get_list_of_compact_abbreviations(), alarm_topic=persistent_stack.alarm_topic, diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index aed3154f3..2ab6cf8cd 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -169,7 +169,6 @@ def __init__( ], ) - # Create custom resource for managing indices # This custom resource will create versioned indices (e.g., 'compact_aslp_providers_v1') # with aliases (e.g., 'compact_aslp_providers') for each compact. @@ -180,7 +179,11 @@ def __init__( resource_type='Custom::IndexManager', service_token=provider.service_token, properties={ - 'numberOfShards': PROD_NUMBER_OF_SHARDS if environment_name == PROD_ENV_NAME else NON_PROD_NUMBER_OF_SHARDS, - 'numberOfReplicas': PROD_NUMBER_OF_REPLICAS if environment_name == PROD_ENV_NAME else NON_PROD_NUMBER_OF_REPLICAS, + 'numberOfShards': PROD_NUMBER_OF_SHARDS + if environment_name == PROD_ENV_NAME + else NON_PROD_NUMBER_OF_SHARDS, + 'numberOfReplicas': PROD_NUMBER_OF_REPLICAS + if environment_name == PROD_ENV_NAME + else NON_PROD_NUMBER_OF_REPLICAS, }, ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index e8f8b05a1..4825b2da2 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -13,7 +13,8 @@ EngineVersion, LoggingOptions, TLSSecurityPolicy, - ZoneAwarenessConfig, WindowStartTime, + WindowStartTime, + ZoneAwarenessConfig, ) from aws_cdk.aws_sns import ITopic from cdk_nag import NagSuppressions @@ -49,6 +50,7 @@ def __init__( construct_id: str, *, environment_name: str, + region: str, vpc_stack: VpcStack, compact_abbreviations: list[str], alarm_topic: ITopic, @@ -62,6 +64,7 @@ def __init__( :param scope: The scope of the construct :param construct_id: The id of the construct :param environment_name: The deployment environment name (e.g., 'prod', 'test') + :param region: The deployment region (e.g., 'us-east-1') :param vpc_stack: The VPC stack containing network resources :param compact_abbreviations: List of compact abbreviations for index access policies :param alarm_topic: The SNS topic for capacity alarms @@ -94,7 +97,7 @@ def __init__( self.encryption_key.grant_encrypt_decrypt(opensearch_principal) # Grant cloudwatch service principal permission to use the key - log_principal = ServicePrincipal('logs.amazonaws.com') + log_principal = ServicePrincipal(f'logs.{region}.amazonaws.com') self.encryption_key.grant_encrypt_decrypt(log_principal) # Create CloudWatch log groups for OpenSearch logging @@ -190,7 +193,7 @@ def __init__( enabled=True, volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, # this type is required for medium instances - volume_type=EbsDeviceVolumeType.GP3 + volume_type=EbsDeviceVolumeType.GP3, ), # Encryption settings encryption_at_rest=EncryptionAtRestOptions(enabled=True, kms_key=self.encryption_key), @@ -554,11 +557,6 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): def _add_domain_suppressions(self, environment_name: str): """ Add CDK Nag suppressions for OpenSearch Domain configuration. - - Some security best practices are not applicable or will be implemented later: - - Fine-grained access control: Will be added with full API implementation - - Access policies: Will be configured when Lambda functions are added - - Dedicated master nodes: Only needed for prod (>3 nodes) """ NagSuppressions.add_resource_suppressions( self.domain, diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 05990a2f9..7cb572247 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -3,7 +3,6 @@ from aws_cdk import Duration from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Stats, TreatMissingData from aws_cdk.aws_cloudwatch_actions import SnsAction -from aws_cdk.aws_dynamodb import ITable from aws_cdk.aws_ec2 import SubnetSelection from aws_cdk.aws_iam import IRole from aws_cdk.aws_kms import IKey @@ -16,6 +15,7 @@ from common_constructs.python_function import PythonFunction from common_constructs.queued_lambda_processor import QueuedLambdaProcessor +from stacks.persistent_stack import ProviderTable from stacks.vpc_stack import VpcStack @@ -40,7 +40,7 @@ def __init__( vpc_stack: VpcStack, vpc_subnets: SubnetSelection, lambda_role: IRole, - provider_table: ITable, + provider_table: ProviderTable, encryption_key: IKey, alarm_topic: ITopic, ): diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py index 4a316fa3b..548d3f525 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_pipe.py @@ -1,4 +1,3 @@ -from aws_cdk.aws_dynamodb import ITable from aws_cdk.aws_iam import Effect, PolicyStatement, Role, ServicePrincipal from aws_cdk.aws_kms import IKey from aws_cdk.aws_pipes import CfnPipe @@ -7,6 +6,8 @@ from common_constructs.stack import Stack from constructs import Construct +from stacks.persistent_stack import ProviderTable + class ProviderUpdateIngestPipe(Construct): """ @@ -24,7 +25,7 @@ def __init__( self, scope: Construct, construct_id: str, - provider_table: ITable, + provider_table: ProviderTable, target_queue: IQueue, encryption_key: IKey, ): @@ -83,7 +84,7 @@ def __init__( target=target_queue.queue_arn, source_parameters=CfnPipe.PipeSourceParametersProperty( dynamo_db_stream_parameters=CfnPipe.PipeSourceDynamoDBStreamParametersProperty( - # 'LATEST' starts processing from the latest available stream record + # 'LATEST' starts processing from the latest available stream record # from the moment the pipe is created starting_position='LATEST', # send everything to SQS as it arrives From 2a86ea5be26e24b7309c8dc55d772cd9004203f9 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 09:56:00 -0600 Subject: [PATCH 107/137] simplify check for prod env --- .../search_persistent_stack/index_manager.py | 6 ++++-- .../provider_search_domain.py | 14 ++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index 2ab6cf8cd..521cd68c6 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -57,6 +57,8 @@ def __init__( super().__init__(scope, construct_id) stack = Stack.of(scope) + self._is_prod_environment = environment_name == PROD_ENV_NAME + # Create Lambda function for managing OpenSearch indices self.manage_function = PythonFunction( self, @@ -180,10 +182,10 @@ def __init__( service_token=provider.service_token, properties={ 'numberOfShards': PROD_NUMBER_OF_SHARDS - if environment_name == PROD_ENV_NAME + if self._is_prod_environment else NON_PROD_NUMBER_OF_SHARDS, 'numberOfReplicas': PROD_NUMBER_OF_REPLICAS - if environment_name == PROD_ENV_NAME + if self._is_prod_environment else NON_PROD_NUMBER_OF_REPLICAS, }, ) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index 4825b2da2..5d125bdfe 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -80,8 +80,10 @@ def __init__( self._index_manager_lambda_role = index_manager_lambda_role self._search_api_lambda_role = search_api_lambda_role + self._is_prod_environment = environment_name == PROD_ENV_NAME + # Determine removal policy based on environment - removal_policy = RemovalPolicy.RETAIN if environment_name == PROD_ENV_NAME else RemovalPolicy.DESTROY + removal_policy = RemovalPolicy.RETAIN if self._is_prod_environment else RemovalPolicy.DESTROY # Create dedicated KMS key for OpenSearch domain encryption self.encryption_key = Key( @@ -191,7 +193,7 @@ def __init__( # EBS volume configuration ebs=EbsOptions( enabled=True, - volume_size=PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE, + volume_size=PROD_EBS_VOLUME_SIZE if self._is_prod_environment else NON_PROD_EBS_VOLUME_SIZE, # this type is required for medium instances volume_type=EbsDeviceVolumeType.GP3, ), @@ -305,7 +307,7 @@ def _get_capacity_config(self, environment_name: str) -> CapacityConfig: :param environment_name: The deployment environment name :return: CapacityConfig with appropriate instance types and counts """ - if environment_name == PROD_ENV_NAME: + if self._is_prod_environment: # Production configuration with high availability # 3 dedicated master nodes + 3 data nodes across 3 AZs with standby # Multi-AZ with standby does not support t3 instance types @@ -344,7 +346,7 @@ def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConf :param environment_name: The deployment environment name :return: ZoneAwarenessConfig with appropriate settings """ - if environment_name == PROD_ENV_NAME: + if self._is_prod_environment: return ZoneAwarenessConfig(enabled=True, availability_zone_count=3) # Non-prod environments only use one data node, hence we don't enable zone awareness @@ -361,7 +363,7 @@ def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> Subnet :param vpc_stack: The VPC stack containing the private subnets :return: SubnetSelection with appropriate subnet configuration """ - if environment_name == PROD_ENV_NAME: + if self._is_prod_environment: # Production: Use all private isolated subnets from the VPC. # VPC is configured with max_azs=3, so this will select exactly 3 subnets return SubnetSelection(subnet_type=SubnetType.PRIVATE_ISOLATED) @@ -411,7 +413,7 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): stack = Stack.of(self) # Get the volume size for calculating storage threshold - volume_size_gb = PROD_EBS_VOLUME_SIZE if environment_name == PROD_ENV_NAME else NON_PROD_EBS_VOLUME_SIZE + volume_size_gb = PROD_EBS_VOLUME_SIZE if self._is_prod_environment else NON_PROD_EBS_VOLUME_SIZE # 50% threshold in MB (FreeStorageSpace metric is reported in megabytes) # Formula: GB * 1024 MB/GB * 0.5 for 50% threshold storage_threshold_mb = volume_size_gb * 1024 * 0.5 From 16ac23188be9fd2053a94b9af1aab02a873fac14 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 10:04:05 -0600 Subject: [PATCH 108/137] remove unused params --- .../search_persistent_stack/index_manager.py | 4 +-- .../provider_search_domain.py | 26 ++++++++----------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index 521cd68c6..bddd595a8 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -181,9 +181,7 @@ def __init__( resource_type='Custom::IndexManager', service_token=provider.service_token, properties={ - 'numberOfShards': PROD_NUMBER_OF_SHARDS - if self._is_prod_environment - else NON_PROD_NUMBER_OF_SHARDS, + 'numberOfShards': PROD_NUMBER_OF_SHARDS if self._is_prod_environment else NON_PROD_NUMBER_OF_SHARDS, 'numberOfReplicas': PROD_NUMBER_OF_REPLICAS if self._is_prod_environment else NON_PROD_NUMBER_OF_REPLICAS, diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index 5d125bdfe..cd19c9fc3 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -149,11 +149,11 @@ def __init__( ) # Determine instance type and capacity based on environment - capacity_config = self._get_capacity_config(environment_name) + capacity_config = self._get_capacity_config() # Determine AZ awareness based on environment - zone_awareness_config = self._get_zone_awareness_config(environment_name) + zone_awareness_config = self._get_zone_awareness_config() # Determine subnet selection based on environment - self.vpc_subnets = self._get_vpc_subnets(environment_name, vpc_stack) + self.vpc_subnets = self._get_vpc_subnets(vpc_stack) # Create OpenSearch Domain self.domain = Domain( @@ -232,14 +232,14 @@ def __init__( self.domain.grant_read_write(self._index_manager_lambda_role) # Add CDK Nag suppressions - self._add_domain_suppressions(environment_name) + self._add_domain_suppressions() self._add_access_policy_lambda_suppressions() self._add_lambda_role_suppressions(self._search_api_lambda_role) self._add_lambda_role_suppressions(self._ingest_lambda_role) self._add_lambda_role_suppressions(self._index_manager_lambda_role) # Add capacity monitoring alarms - self._add_capacity_alarms(environment_name, alarm_topic) + self._add_capacity_alarms(alarm_topic) def _configure_access_policies(self, compact_abbreviations: list[str]): """ @@ -297,14 +297,13 @@ def _configure_access_policies(self, compact_abbreviations: list[str]): search_api_policy, ) - def _get_capacity_config(self, environment_name: str) -> CapacityConfig: + def _get_capacity_config(self) -> CapacityConfig: """ Determine OpenSearch cluster capacity configuration based on environment. Non-prod (sandbox, test, beta, etc.): Single t3.small.search node Prod: 3 dedicated master (r8g.medium.search) + 3 data nodes (m7g.medium.search) with standby - :param environment_name: The deployment environment name :return: CapacityConfig with appropriate instance types and counts """ if self._is_prod_environment: @@ -337,13 +336,12 @@ def _get_capacity_config(self, environment_name: str) -> CapacityConfig: multi_az_with_standby_enabled=False, ) - def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConfig: + def _get_zone_awareness_config(self) -> ZoneAwarenessConfig: """ Determine OpenSearch cluster availability zone awareness based on environment. 3 for production, not enabled for all other non-prod environments - :param environment_name: The deployment environment name :return: ZoneAwarenessConfig with appropriate settings """ if self._is_prod_environment: @@ -352,14 +350,13 @@ def _get_zone_awareness_config(self, environment_name: str) -> ZoneAwarenessConf # Non-prod environments only use one data node, hence we don't enable zone awareness return ZoneAwarenessConfig(enabled=False) - def _get_vpc_subnets(self, environment_name: str, vpc_stack: VpcStack) -> SubnetSelection: + def _get_vpc_subnets(self, vpc_stack: VpcStack) -> SubnetSelection: """ Determine VPC subnet selection based on environment. Production: All private isolated subnets (3 AZs) for zone awareness and high availability Non-prod: Single subnet (privateSubnet1 with CIDR 10.0.0.0/20) for single-node deployment - :param environment_name: The deployment environment name :param vpc_stack: The VPC stack containing the private subnets :return: SubnetSelection with appropriate subnet configuration """ @@ -396,7 +393,7 @@ def _find_subnet_by_name(self, vpc, subnet_name: str): return subnet_construct - def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): + def _add_capacity_alarms(self, alarm_topic: ITopic): """ Add CloudWatch alarms to monitor OpenSearch capacity and alert before hitting limits. @@ -407,7 +404,6 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): - Cluster Status (red/yellow) for critical and degraded states - Automated Snapshot Failure for backup issues - :param environment_name: The deployment environment name :param alarm_topic: The SNS topic to send alarm notifications to """ stack = Stack.of(self) @@ -556,7 +552,7 @@ def _add_capacity_alarms(self, environment_name: str, alarm_topic: ITopic): ), ).add_alarm_action(SnsAction(alarm_topic)) - def _add_domain_suppressions(self, environment_name: str): + def _add_domain_suppressions(self): """ Add CDK Nag suppressions for OpenSearch Domain configuration. """ @@ -580,7 +576,7 @@ def _add_domain_suppressions(self, environment_name: str): ], apply_to_children=True, ) - if environment_name != PROD_ENV_NAME: + if not self._is_prod_environment: NagSuppressions.add_resource_suppressions( self.domain, suppressions=[ From 3d449e5683179a657ed417d3327b5acaedb93432 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 10:58:30 -0600 Subject: [PATCH 109/137] Check for cross-index queries --- .../data_model/schema/provider/api.py | 56 ++++++++++++ .../tests/function/test_search_privileges.py | 89 +++++++++++++++++++ .../tests/function/test_search_providers.py | 89 +++++++++++++++++++ 3 files changed, 234 insertions(+) diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py index 316c40cb8..36d464caa 100644 --- a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -31,6 +31,37 @@ PrivilegeReadPrivateResponseSchema, ) +# Keys that indicate cross-index query attempts in OpenSearch DSL +# These are used by terms lookup, more_like_this, and other queries to reference external indices +_CROSS_INDEX_KEYS = frozenset({'index', '_index'}) + + +def _validate_no_cross_index_keys(obj, path: str = 'query') -> None: + """ + Recursively validate that an object does not contain cross-index lookup keys. + + This function traverses the query structure looking for keys that would indicate + an attempt to access data from other indices: + - 'index': Used in terms lookup queries to specify an external index + - '_index': Used in more_like_this queries to reference documents from other indices + + These keys should never appear in legitimate single-index queries against the + provider search index. + + :param obj: The object to validate (dict, list, or scalar) + :param path: The current path in the object for error messages + :raises ValidationError: If a cross-index key is found + """ + if isinstance(obj, dict): + for key, value in obj.items(): + if key in _CROSS_INDEX_KEYS: + raise ValidationError(f"Cross-index queries are not allowed. Found '{key}' at {path}.{key}") + _validate_no_cross_index_keys(value, path=f'{path}.{key}') + elif isinstance(obj, list): + for i, item in enumerate(obj): + _validate_no_cross_index_keys(item, path=f'{path}[{i}]') + # Scalar values (str, int, bool, None) are safe - we only check keys + class ProviderSSNResponseSchema(ForgivingSchema): """ @@ -481,6 +512,22 @@ class SearchProvidersRequestSchema(CCRequestSchema): # Example: ["provider-uuid-123", "2024-01-15T10:30:00Z"] search_after = Raw(required=False, allow_none=False) + @validates_schema + def validate_no_cross_index_queries(self, data, **kwargs): + """ + Validate that the query does not contain cross-index lookup attempts. + + This is a defense-in-depth security measure to prevent queries that attempt to access + data from other compact indices. The primary protection is the OpenSearch domain setting + `rest.action.multi.allow_explicit_index: false`, but this validation provides an + additional application-layer check. + + Dangerous patterns blocked: + - Terms lookup with external index: {"terms": {"field": {"index": "other_index", ...}}} + - More like this with external docs: {"more_like_this": {"like": [{"_index": "other_index"}]}} + """ + _validate_no_cross_index_keys(data.get('query', {})) + class ExportPrivilegesRequestSchema(CCRequestSchema): """ @@ -495,3 +542,12 @@ class ExportPrivilegesRequestSchema(CCRequestSchema): # The OpenSearch query body - we use Raw to allow the full flexibility of OpenSearch queries query = Raw(required=True, allow_none=False) + + @validates_schema + def validate_no_cross_index_queries(self, data, **kwargs): + """ + Validate that the query does not contain cross-index lookup attempts. + + This is a defense-in-depth security measure. See SearchProvidersRequestSchema for details. + """ + _validate_no_cross_index_keys(data.get('query', {})) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 4a7f9e503..91f6970c9 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -650,3 +650,92 @@ def test_missing_scopes_returns_403(self): self.assertEqual(403, response['statusCode']) body = json.loads(response['body']) self.assertIn('Access denied', body['message']) + + def test_export_query_with_index_key_returns_400(self): + """Test that export queries containing 'index' key are rejected with 400 error.""" + from handlers.search import search_api_handler + + # Test with 'index' key (terms lookup attack pattern) + event = self._create_api_event( + 'aslp', + body={ + 'query': { + 'terms': { + 'providerId': { + 'index': 'compact_octp_providers', + 'id': 'some-uuid', + 'path': 'providerId', + } + } + } + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Cross-index queries are not allowed', body['message']) + self.assertIn("'index'", body['message']) + + def test_export_query_with_underscore_index_key_returns_400(self): + """Test that export queries containing '_index' key are rejected with 400 error.""" + from handlers.search import search_api_handler + + # Test with '_index' key (more_like_this attack pattern) + event = self._create_api_event( + 'aslp', + body={ + 'query': { + 'more_like_this': { + 'fields': ['familyName', 'givenName'], + 'like': [ + { + '_index': 'compact_octp_providers', + '_id': 'target-provider-uuid', + } + ], + } + } + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Cross-index queries are not allowed', body['message']) + self.assertIn("'_index'", body['message']) + + def test_export_query_with_nested_index_key_returns_400(self): + """Test that export queries with nested 'index' key at any level are rejected.""" + from handlers.search import search_api_handler + + # Test with 'index' key nested deep in the query structure + event = self._create_api_event( + 'aslp', + body={ + 'query': { + 'bool': { + 'should': [ + { + 'terms': { + 'familyName.keyword': { + 'index': 'compact_octp_providers', + 'id': 'target-uuid', + 'path': 'familyName.keyword', + } + } + } + ] + } + } + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Cross-index queries are not allowed', body['message']) + self.assertIn("'index'", body['message']) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index a854e98d4..12741c90b 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -345,3 +345,92 @@ def test_missing_scopes_returns_403(self): self.assertEqual(403, response['statusCode']) body = json.loads(response['body']) self.assertIn('Access denied', body['message']) + + def test_query_with_index_key_returns_400(self): + """Test that queries containing 'index' key are rejected with 400 error.""" + from handlers.search import search_api_handler + + # Test with 'index' key (terms lookup attack pattern) + event = self._create_api_event( + 'aslp', + body={ + 'query': { + 'terms': { + 'providerId': { + 'index': 'compact_octp_providers', + 'id': 'some-uuid', + 'path': 'providerId', + } + } + } + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Cross-index queries are not allowed', body['message']) + self.assertIn("'index'", body['message']) + + def test_query_with_underscore_index_key_returns_400(self): + """Test that queries containing '_index' key are rejected with 400 error.""" + from handlers.search import search_api_handler + + # Test with '_index' key (more_like_this attack pattern) + event = self._create_api_event( + 'aslp', + body={ + 'query': { + 'more_like_this': { + 'fields': ['familyName', 'givenName'], + 'like': [ + { + '_index': 'compact_octp_providers', + '_id': 'target-provider-uuid', + } + ], + } + } + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Cross-index queries are not allowed', body['message']) + self.assertIn("'_index'", body['message']) + + def test_query_with_nested_index_key_returns_400(self): + """Test that queries with nested 'index' key at any level are rejected.""" + from handlers.search import search_api_handler + + # Test with 'index' key nested deep in the query structure + event = self._create_api_event( + 'aslp', + body={ + 'query': { + 'bool': { + 'should': [ + { + 'terms': { + 'familyName.keyword': { + 'index': 'compact_octp_providers', + 'id': 'target-uuid', + 'path': 'familyName.keyword', + } + } + } + ] + } + } + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Cross-index queries are not allowed', body['message']) + self.assertIn("'index'", body['message']) From d9e4be89ccddb4aedbd150d7ed19a5d11ad1797f Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 11:13:15 -0600 Subject: [PATCH 110/137] remove outdated comment --- .../stacks/search_persistent_stack/provider_search_domain.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index cd19c9fc3..a755a356f 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -598,10 +598,6 @@ def _add_domain_suppressions(self): def _add_access_policy_lambda_suppressions(self): """ Add CDK Nag suppressions for the auto-generated Lambda function created by add_access_policies. - - The CDK Domain.add_access_policies() method creates an AwsCustomResource Lambda to manage - the domain's access policy. CDK generates these with IDs starting with 'AWS' followed by a hash. - We find these dynamically to avoid relying on a specific hash value. """ stack = Stack.of(self) From dd9f7bbec7a62477a932cd2f253932776400a0f4 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 12:39:13 -0600 Subject: [PATCH 111/137] Return query errors to client --- .../python/search/opensearch_client.py | 22 ++++++++-- .../tests/function/test_search_providers.py | 38 ++++++++++++++++ .../tests/unit/test_opensearch_client.py | 44 ++++++++++++++++++- 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 5b859bbb0..31e049f7d 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -2,9 +2,9 @@ import boto3 from cc_common.config import config, logger -from cc_common.exceptions import CCInternalException +from cc_common.exceptions import CCInternalException, CCInvalidRequestException from opensearchpy import AWSV4SignerAuth, OpenSearch, RequestsHttpConnection -from opensearchpy.exceptions import ConnectionTimeout, TransportError +from opensearchpy.exceptions import ConnectionTimeout, RequestError, TransportError # Retry configuration for bulk indexing MAX_RETRY_ATTEMPTS = 5 @@ -46,8 +46,24 @@ def search(self, index_name: str, body: dict) -> dict: :param index_name: The name of the index to search :param body: The OpenSearch query body :return: The search response from OpenSearch + :raises CCInvalidRequestException: If the query is invalid (400 error from OpenSearch) """ - return self._client.search(index=index_name, body=body) + try: + return self._client.search(index=index_name, body=body) + except RequestError as e: + if e.status_code == 400: + # Extract the error message from the RequestError + # RequestError contains: status_code, error (type), and info (message or dict) + error_message = e.info if isinstance(e.info, str) else str(e.error) + logger.warning( + 'OpenSearch search request failed', + index_name=index_name, + status_code=e.status_code, + error=str(e), + ) + raise CCInvalidRequestException(f'Invalid search query: {error_message}') from e + # Re-raise non-400 RequestErrors + raise def index_document(self, index_name: str, document_id: str, document: dict) -> dict: """ diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 12741c90b..dd7745a83 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -2,6 +2,7 @@ from unittest.mock import Mock, patch from moto import mock_aws +from opensearchpy.exceptions import RequestError from . import TstFunction @@ -434,3 +435,40 @@ def test_query_with_nested_index_key_returns_400(self): body = json.loads(response['body']) self.assertIn('Cross-index queries are not allowed', body['message']) self.assertIn("'index'", body['message']) + + @patch('opensearch_client.OpenSearch') + def test_opensearch_request_error_returns_400_with_error_message(self, mock_opensearch_client): + """Test that OpenSearch RequestError with status 400 returns error message to caller.""" + from handlers.search import search_api_handler + + # Configure the mock internal OpenSearch client to raise a RequestError + mock_internal_client = Mock() + mock_opensearch_client.return_value = mock_internal_client + + # Create a RequestError similar to what OpenSearch returns for invalid queries + # RequestError(status_code, error_type, info) + error_message = ( + 'Text fields are not optimised for operations that require per-document field data ' + 'like aggregations and sorting, so these operations are disabled by default. ' + 'Please use a keyword field instead.' + ) + mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_message) + + event = self._create_api_event( + 'aslp', + body={ + 'query': {'match_all': {}}, + 'sort': [{'familyName': 'asc'}], # Sorting on text field causes this error + }, + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual( + 'Invalid search query: Text fields are not optimised for operations that ' + 'require per-document field data like aggregations and sorting, so these ' + 'operations are disabled by default. Please use a keyword field instead.', + body['message'], + ) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index bb4ba3130..18a912b79 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -1,8 +1,8 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from cc_common.exceptions import CCInternalException -from opensearchpy.exceptions import ConnectionTimeout, TransportError +from cc_common.exceptions import CCInternalException, CCInvalidRequestException +from opensearchpy.exceptions import ConnectionTimeout, RequestError, TransportError class TestOpenSearchClient(TestCase): @@ -113,6 +113,46 @@ def test_search_calls_internal_client_with_expected_arguments(self): ) self.assertEqual(expected_response, result) + def test_search_raises_cc_invalid_request_exception_on_400_request_error(self): + """Test that search raises CCInvalidRequestException when OpenSearch returns a 400 RequestError.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + query_body = {'query': {'match_all': {}}, 'sort': [{'familyName': 'asc'}]} + + # Simulate OpenSearch returning a 400 error for invalid query + error_message = ( + 'Text fields are not optimised for operations that require per-document field data ' + 'like aggregations and sorting, so these operations are disabled by default.' + ) + mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_message) + + with self.assertRaises(CCInvalidRequestException) as context: + client.search(index_name=index_name, body=query_body) + + # Verify the exception message contains useful info + self.assertEqual( + 'Invalid search query: Text fields are not optimised for operations that ' + 'require per-document field data like aggregations and sorting, so these ' + 'operations are disabled by default.', + str(context.exception), + ) + + def test_search_reraises_non_400_request_error(self): + """Test that search re-raises RequestError for non-400 status codes.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + query_body = {'query': {'match_all': {}}} + + # Simulate OpenSearch returning a 500 error + mock_internal_client.search.side_effect = RequestError(500, 'internal_error', 'Something went wrong') + + with self.assertRaises(RequestError) as context: + client.search(index_name=index_name, body=query_body) + + self.assertEqual(500, context.exception.status_code) + def test_index_document_calls_internal_client_with_expected_arguments(self): """Test that index_document calls the internal client's index method correctly.""" client, mock_internal_client = self._create_client_with_mock() From d759714e525ba8f29eb0232ca8f71881d074c94e Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 14:15:28 -0600 Subject: [PATCH 112/137] Add domain health check logic to index manager to avoid timeouts --- .../handlers/manage_opensearch_indices.py | 90 ++++++++++- .../python/search/opensearch_client.py | 124 ++++++++++++-- .../test_manage_opensearch_indices.py | 74 +++++++++ .../tests/unit/test_opensearch_client.py | 151 +++++++++++++++++- .../search_persistent_stack/index_manager.py | 2 +- 5 files changed, 424 insertions(+), 17 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index e148cb417..9d62269ff 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -1,10 +1,18 @@ +import time + from cc_common.config import config, logger +from cc_common.exceptions import CCInternalException from custom_resource_handler import CustomResourceHandler, CustomResourceResponse from opensearch_client import OpenSearchClient # Initial index version for new deployments INITIAL_INDEX_VERSION = 'v1' +# Readiness check configuration +# OpenSearch domains may take time to become responsive after CloudFormation reports them as created. +DOMAIN_READINESS_CHECK_INTERVAL_SECONDS = 10 +DOMAIN_READINESS_MAX_ATTEMPTS = 30 # 30 attempts * 10 seconds = 5 minutes max wait + class OpenSearchIndexManager(CustomResourceHandler): """ @@ -21,8 +29,13 @@ def on_create(self, properties: dict) -> CustomResourceResponse | None: """ Create the versioned indices and aliases on creation. """ - logger.info('Connecting to OpenSearch domain') - client = OpenSearchClient() + logger.info( + 'Starting OpenSearch index creation', + opensearch_host=config.opensearch_host_endpoint, + ) + + # Wait for domain to become responsive + client = self._wait_for_domain_ready() # Get index configuration from custom resource properties number_of_shards = int(properties['numberOfShards']) @@ -58,6 +71,79 @@ def on_delete(self, _properties: dict) -> CustomResourceResponse | None: No-op on delete. """ + def _wait_for_domain_ready(self) -> OpenSearchClient: + """ + Wait for the OpenSearch domain to become responsive. + + Newly created OpenSearch domains may not be immediately responsive even after + CloudFormation reports them as created. This method attempts to create a client + and verify connectivity with retries before proceeding with index creation. + + :return: A connected OpenSearchClient instance + :raises CCInternalException: If the domain is not responsive after max attempts + """ + last_exception = None + + for attempt in range(1, DOMAIN_READINESS_MAX_ATTEMPTS + 1): + try: + logger.info( + 'Attempting to connect to OpenSearch domain', + attempt=attempt, + max_attempts=DOMAIN_READINESS_MAX_ATTEMPTS, + ) + client = OpenSearchClient() + # Perform a lightweight health check to verify connectivity + # This will use the client's internal retry logic + cluster_health = client.cluster_health() + logger.info( + 'Successfully connected to OpenSearch domain', + cluster_status=cluster_health.get('status'), + number_of_nodes=cluster_health.get('number_of_nodes'), + ) + return client + except CCInternalException as e: + # CCInternalException is raised by OpenSearchClient after its internal retries are exhausted + last_exception = e + if attempt < DOMAIN_READINESS_MAX_ATTEMPTS: + logger.warning( + 'Domain not yet responsive, waiting before retry', + attempt=attempt, + max_attempts=DOMAIN_READINESS_MAX_ATTEMPTS, + wait_seconds=DOMAIN_READINESS_CHECK_INTERVAL_SECONDS, + error=str(e), + ) + time.sleep(DOMAIN_READINESS_CHECK_INTERVAL_SECONDS) + else: + logger.error( + 'Domain did not become responsive within timeout', + attempts=DOMAIN_READINESS_MAX_ATTEMPTS, + error=str(e), + ) + except Exception as e: + # Handle unexpected exceptions (e.g., connection errors during client initialization) + last_exception = e + if attempt < DOMAIN_READINESS_MAX_ATTEMPTS: + logger.warning( + 'Connection attempt failed, waiting before retry', + attempt=attempt, + max_attempts=DOMAIN_READINESS_MAX_ATTEMPTS, + wait_seconds=DOMAIN_READINESS_CHECK_INTERVAL_SECONDS, + error=str(e), + ) + time.sleep(DOMAIN_READINESS_CHECK_INTERVAL_SECONDS) + else: + logger.error( + 'Failed to connect to OpenSearch domain after max attempts', + attempts=DOMAIN_READINESS_MAX_ATTEMPTS, + error=str(e), + ) + + raise CCInternalException( + f'OpenSearch domain did not become responsive after {DOMAIN_READINESS_MAX_ATTEMPTS} attempts ' + f'({DOMAIN_READINESS_MAX_ATTEMPTS * DOMAIN_READINESS_CHECK_INTERVAL_SECONDS} seconds). ' + f'Last error: {last_exception}' + ) + def _create_provider_index_with_alias( self, client: OpenSearchClient, diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 31e049f7d..cc6168376 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -6,14 +6,16 @@ from opensearchpy import AWSV4SignerAuth, OpenSearch, RequestsHttpConnection from opensearchpy.exceptions import ConnectionTimeout, RequestError, TransportError -# Retry configuration for bulk indexing +# Retry configuration for operations MAX_RETRY_ATTEMPTS = 5 -INITIAL_BACKOFF_SECONDS = 1 +INITIAL_BACKOFF_SECONDS = 2 MAX_BACKOFF_SECONDS = 32 +DEFAULT_TIMEOUT = 30 + class OpenSearchClient: - def __init__(self): + def __init__(self, timeout: int = DEFAULT_TIMEOUT): lambda_credentials = boto3.Session().get_credentials() auth = AWSV4SignerAuth(credentials=lambda_credentials, region=config.environment_region, service='es') self._client = OpenSearch( @@ -22,22 +24,124 @@ def __init__(self): use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, + timeout=timeout, pool_maxsize=20, ) def create_index(self, index_name: str, index_mapping: dict) -> None: - self._client.indices.create(index=index_name, body=index_mapping) + """ + Create an index with the specified mapping. + + :param index_name: The name of the index to create + :param index_mapping: The index configuration including settings and mappings + :raises CCInternalException: If all retry attempts fail + """ + self._execute_with_retry( + operation=lambda: self._client.indices.create(index=index_name, body=index_mapping), + operation_name=f'create_index({index_name})', + ) def index_exists(self, index_name: str) -> bool: - return self._client.indices.exists(index=index_name) + """ + Check if an index exists. + + :param index_name: The name of the index to check + :return: True if the index exists, False otherwise + :raises CCInternalException: If all retry attempts fail + """ + return self._execute_with_retry( + operation=lambda: self._client.indices.exists(index=index_name), + operation_name=f'index_exists({index_name})', + ) def alias_exists(self, alias_name: str) -> bool: - """Check if an alias exists.""" - return self._client.indices.exists_alias(name=alias_name) + """ + Check if an alias exists. + + :param alias_name: The name of the alias to check + :return: True if the alias exists, False otherwise + :raises CCInternalException: If all retry attempts fail + """ + return self._execute_with_retry( + operation=lambda: self._client.indices.exists_alias(name=alias_name), + operation_name=f'alias_exists({alias_name})', + ) def create_alias(self, index_name: str, alias_name: str) -> None: - """Create an alias pointing to the specified index.""" - self._client.indices.put_alias(index=index_name, name=alias_name) + """ + Create an alias pointing to the specified index. + + :param index_name: The index to create the alias for + :param alias_name: The name of the alias to create + :raises CCInternalException: If all retry attempts fail + """ + self._execute_with_retry( + operation=lambda: self._client.indices.put_alias(index=index_name, name=alias_name), + operation_name=f'create_alias({alias_name} -> {index_name})', + ) + + def cluster_health(self) -> dict: + """ + Get the cluster health status. + + Implements retry logic with exponential backoff for transient connection issues. + This is useful for checking if the cluster is responsive, especially after + a new domain is created. + + :return: The cluster health response from OpenSearch + :raises CCInternalException: If all retry attempts fail + """ + return self._execute_with_retry( + operation=lambda: self._client.cluster.health(), + operation_name='cluster_health', + ) + + def _execute_with_retry(self, operation: callable, operation_name: str): + """ + Execute an operation with retry logic and exponential backoff. + + This handles transient connection issues that can occur when: + - OpenSearch domain was just created and is still warming up + - Network connectivity issues within the VPC + - Temporary high load on the OpenSearch cluster + + :param operation: A callable that performs the operation + :param operation_name: A descriptive name for the operation (for logging) + :return: The result of the operation + :raises CCInternalException: If all retry attempts fail + """ + last_exception = None + backoff_seconds = INITIAL_BACKOFF_SECONDS + + for attempt in range(1, MAX_RETRY_ATTEMPTS + 1): + try: + return operation() + except (ConnectionTimeout, TransportError) as e: + last_exception = e + if attempt < MAX_RETRY_ATTEMPTS: + logger.warning( + 'Operation failed, retrying with backoff', + operation=operation_name, + attempt=attempt, + max_attempts=MAX_RETRY_ATTEMPTS, + backoff_seconds=backoff_seconds, + error=str(e), + ) + time.sleep(backoff_seconds) + # Exponential backoff with cap + backoff_seconds = min(backoff_seconds * 2, MAX_BACKOFF_SECONDS) + else: + logger.error( + 'Operation failed after max retry attempts', + operation=operation_name, + attempts=MAX_RETRY_ATTEMPTS, + error=str(e), + ) + + # All retry attempts failed + raise CCInternalException( + f'{operation_name} failed after {MAX_RETRY_ATTEMPTS} attempts. Last error: {last_exception}' + ) def search(self, index_name: str, body: dict) -> dict: """ @@ -182,7 +286,7 @@ def _bulk_operation_with_retry( for attempt in range(1, MAX_RETRY_ATTEMPTS + 1): try: - return self._client.bulk(body=actions, index=index_name, timeout=30) + return self._client.bulk(body=actions, index=index_name, timeout=DEFAULT_TIMEOUT) except (ConnectionTimeout, TransportError) as e: last_exception = e if attempt < MAX_RETRY_ATTEMPTS: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index 19e0a95c4..35ab18bd7 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -44,6 +44,13 @@ def _when_testing_mock_opensearch_client( mock_client_instance = Mock() mock_opensearch_client.return_value = mock_client_instance + # Configure cluster_health mock (used by _wait_for_domain_ready) + mock_client_instance.cluster_health.return_value = { + 'status': 'green', + 'number_of_nodes': 1, + 'cluster_name': 'test-cluster', + } + # Configure alias_exists mock if isinstance(alias_exists_return_value, dict): mock_client_instance.alias_exists.side_effect = lambda alias_name: alias_exists_return_value.get( @@ -483,3 +490,70 @@ def test_on_delete_is_noop(self, mock_opensearch_client): # Result should be None (no-op) self.assertIsNone(result) + + @patch('handlers.manage_opensearch_indices.time.sleep') + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_create_retries_when_domain_not_immediately_responsive(self, mock_opensearch_client, mock_sleep): + """Test that on_create retries connecting to the domain when it's not immediately responsive.""" + from cc_common.exceptions import CCInternalException + from handlers.manage_opensearch_indices import on_event + + # First two calls fail, third succeeds + mock_client_instance = Mock() + mock_client_instance.cluster_health.return_value = { + 'status': 'green', + 'number_of_nodes': 1, + } + mock_client_instance.alias_exists.return_value = True # Skip index creation for simplicity + + call_count = 0 + + def side_effect(): + nonlocal call_count + call_count += 1 + if call_count <= 2: + raise CCInternalException('cluster_health failed after 5 attempts. Last error: ConnectionTimeout') + return mock_client_instance + + mock_opensearch_client.side_effect = side_effect + + # Create the event for a 'Create' request + event = self._create_event('Create') + + # Call the handler + on_event(event, self.mock_context) + + # Assert that OpenSearchClient was instantiated 3 times (2 failures + 1 success) + self.assertEqual(3, mock_opensearch_client.call_count) + + # Assert that sleep was called twice (once between each retry) + self.assertEqual(2, mock_sleep.call_count) + + @patch('handlers.manage_opensearch_indices.time.sleep') + @patch('handlers.manage_opensearch_indices.OpenSearchClient') + def test_on_create_raises_after_max_retries(self, mock_opensearch_client, mock_sleep): + """Test that on_create raises CCInternalException after max retries are exhausted.""" + from cc_common.exceptions import CCInternalException + from handlers.manage_opensearch_indices import ( + DOMAIN_READINESS_MAX_ATTEMPTS, + on_event, + ) + + # All calls fail + mock_opensearch_client.side_effect = CCInternalException( + 'cluster_health failed after 5 attempts. Last error: ConnectionTimeout' + ) + + # Create the event for a 'Create' request + event = self._create_event('Create') + + # Call the handler and expect an exception + with self.assertRaises(CCInternalException) as context: + on_event(event, self.mock_context) + + # Verify the error message mentions the number of attempts + self.assertIn(str(DOMAIN_READINESS_MAX_ATTEMPTS), str(context.exception)) + self.assertIn('did not become responsive', str(context.exception)) + + # Assert that OpenSearchClient was instantiated max attempts times + self.assertEqual(DOMAIN_READINESS_MAX_ATTEMPTS, mock_opensearch_client.call_count) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index 18a912b79..42b1ac64f 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -233,6 +233,8 @@ def test_bulk_index_returns_early_for_empty_documents(self): @patch('opensearch_client.time.sleep') def test_bulk_index_retries_on_connection_timeout_and_succeeds(self, mock_sleep): """Test that bulk_index retries on ConnectionTimeout and eventually succeeds.""" + from opensearch_client import INITIAL_BACKOFF_SECONDS + client, mock_internal_client = self._create_client_with_mock() index_name = 'test_index' @@ -250,10 +252,10 @@ def test_bulk_index_retries_on_connection_timeout_and_succeeds(self, mock_sleep) # Verify bulk was called 3 times self.assertEqual(3, mock_internal_client.bulk.call_count) - # Verify sleep was called with exponential backoff (1s, 2s) + # Verify sleep was called with exponential backoff self.assertEqual(2, mock_sleep.call_count) - mock_sleep.assert_any_call(1) - mock_sleep.assert_any_call(2) + mock_sleep.assert_any_call(INITIAL_BACKOFF_SECONDS) + mock_sleep.assert_any_call(INITIAL_BACKOFF_SECONDS * 2) # Verify we got the successful response self.assertEqual(expected_response, result) @@ -322,8 +324,149 @@ def test_bulk_index_exponential_backoff_caps_at_max(self, mock_sleep): with self.assertRaises(CCInternalException): client.bulk_index(index_name=index_name, documents=documents) - # Verify backoff values: 1, 2, 4, 8 (all should be <= MAX_BACKOFF_SECONDS) + # Verify backoff values: 2, 4, 8, 16 (all should be <= MAX_BACKOFF_SECONDS) # With MAX_RETRY_ATTEMPTS = 5, we have 4 sleeps sleep_calls = [call[0][0] for call in mock_sleep.call_args_list] for sleep_value in sleep_calls: self.assertLessEqual(sleep_value, MAX_BACKOFF_SECONDS) + + +class TestOpenSearchClientIndexManagementRetry(TestCase): + """Test suite for OpenSearchClient index management operations with retry logic.""" + + def _create_client_with_mock(self): + """Create an OpenSearchClient with a mocked internal client.""" + with ( + patch('opensearch_client.boto3'), + patch('opensearch_client.config'), + patch('opensearch_client.OpenSearch') as mock_opensearch_class, + ): + mock_internal_client = MagicMock() + mock_opensearch_class.return_value = mock_internal_client + + from opensearch_client import OpenSearchClient + + client = OpenSearchClient() + return client, mock_internal_client + + @patch('opensearch_client.time.sleep') + def test_create_index_retries_on_connection_timeout_and_succeeds(self, mock_sleep): + """Test that create_index retries on ConnectionTimeout and eventually succeeds.""" + from opensearch_client import INITIAL_BACKOFF_SECONDS + + client, mock_internal_client = self._create_client_with_mock() + + # First call fails, second succeeds + mock_internal_client.indices.create.side_effect = [ + ConnectionTimeout('Connection timed out', 503, 'some error'), + {'acknowledged': True}, + ] + + # Should not raise + client.create_index(index_name='test_index', index_mapping={'settings': {}}) + + # Verify create was called 2 times + self.assertEqual(2, mock_internal_client.indices.create.call_count) + # Verify sleep was called once + self.assertEqual(1, mock_sleep.call_count) + mock_sleep.assert_called_with(INITIAL_BACKOFF_SECONDS) + + @patch('opensearch_client.time.sleep') + def test_create_index_raises_after_max_retries(self, mock_sleep): + """Test that create_index raises CCInternalException after max retries.""" + from opensearch_client import MAX_RETRY_ATTEMPTS + + client, mock_internal_client = self._create_client_with_mock() + + # All calls fail + mock_internal_client.indices.create.side_effect = ConnectionTimeout('Connection timed out', 503, 'some error') + + with self.assertRaises(CCInternalException) as context: + client.create_index(index_name='test_index', index_mapping={'settings': {}}) + + # Verify create was called MAX_RETRY_ATTEMPTS times + self.assertEqual(MAX_RETRY_ATTEMPTS, mock_internal_client.indices.create.call_count) + self.assertIn('create_index', str(context.exception)) + + @patch('opensearch_client.time.sleep') + def test_index_exists_retries_on_transport_error_and_succeeds(self, mock_sleep): + """Test that index_exists retries on TransportError and eventually succeeds.""" + client, mock_internal_client = self._create_client_with_mock() + + # First call fails, second succeeds + mock_internal_client.indices.exists.side_effect = [ + TransportError(503, 'ReadTimeout'), + True, + ] + + result = client.index_exists(index_name='test_index') + + self.assertTrue(result) + self.assertEqual(2, mock_internal_client.indices.exists.call_count) + + @patch('opensearch_client.time.sleep') + def test_alias_exists_retries_on_connection_timeout_and_succeeds(self, mock_sleep): + """Test that alias_exists retries on ConnectionTimeout and eventually succeeds.""" + client, mock_internal_client = self._create_client_with_mock() + + # First call fails, second succeeds + mock_internal_client.indices.exists_alias.side_effect = [ + ConnectionTimeout('Connection timed out', 503, 'some error'), + True, + ] + + result = client.alias_exists(alias_name='test_alias') + + self.assertTrue(result) + self.assertEqual(2, mock_internal_client.indices.exists_alias.call_count) + + @patch('opensearch_client.time.sleep') + def test_create_alias_retries_on_connection_timeout_and_succeeds(self, mock_sleep): + """Test that create_alias retries on ConnectionTimeout and eventually succeeds.""" + client, mock_internal_client = self._create_client_with_mock() + + # First call fails, second succeeds + mock_internal_client.indices.put_alias.side_effect = [ + ConnectionTimeout('Connection timed out', 503, 'some error'), + {'acknowledged': True}, + ] + + # Should not raise + client.create_alias(index_name='test_index', alias_name='test_alias') + + self.assertEqual(2, mock_internal_client.indices.put_alias.call_count) + + @patch('opensearch_client.time.sleep') + def test_cluster_health_retries_on_connection_timeout_and_succeeds(self, mock_sleep): + """Test that cluster_health retries on ConnectionTimeout and eventually succeeds.""" + client, mock_internal_client = self._create_client_with_mock() + + expected_response = {'status': 'green', 'number_of_nodes': 3} + + # First call fails, second succeeds + mock_internal_client.cluster.health.side_effect = [ + ConnectionTimeout('Connection timed out', 503, 'some error'), + expected_response, + ] + + result = client.cluster_health() + + self.assertEqual(expected_response, result) + self.assertEqual(2, mock_internal_client.cluster.health.call_count) + + @patch('opensearch_client.time.sleep') + def test_cluster_health_raises_after_max_retries(self, mock_sleep): + """Test that cluster_health raises CCInternalException after max retries.""" + from opensearch_client import MAX_RETRY_ATTEMPTS + + client, mock_internal_client = self._create_client_with_mock() + + # All calls fail + mock_internal_client.cluster.health.side_effect = ConnectionTimeout('Connection timed out', 503, 'some error') + + with self.assertRaises(CCInternalException) as context: + client.cluster_health() + + # Verify health was called MAX_RETRY_ATTEMPTS times + self.assertEqual(MAX_RETRY_ATTEMPTS, mock_internal_client.cluster.health.call_count) + self.assertIn('cluster_health', str(context.exception)) diff --git a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py index bddd595a8..e8e8108e5 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/index_manager.py +++ b/backend/compact-connect/stacks/search_persistent_stack/index_manager.py @@ -72,7 +72,7 @@ def __init__( 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, **stack.common_env_vars, }, - timeout=Duration.minutes(5), + timeout=Duration.minutes(10), memory_size=256, vpc=vpc_stack.vpc, vpc_subnets=vpc_subnets, From c2ad9fafdc8941eb15a4ce4d8c19f315d7186026 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 16:08:59 -0600 Subject: [PATCH 113/137] Returning specific client error message --- .../python/search/opensearch_client.py | 40 +++++++++++++-- .../test_manage_opensearch_indices.py | 2 +- .../tests/function/test_search_providers.py | 24 ++++++--- .../tests/unit/test_opensearch_client.py | 50 +++++++++++++++---- 4 files changed, 92 insertions(+), 24 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index cc6168376..083afe70f 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -12,6 +12,7 @@ MAX_BACKOFF_SECONDS = 32 DEFAULT_TIMEOUT = 30 +SEARCH_TIMEOUT = 20 class OpenSearchClient: @@ -143,32 +144,61 @@ def _execute_with_retry(self, operation: callable, operation_name: str): f'{operation_name} failed after {MAX_RETRY_ATTEMPTS} attempts. Last error: {last_exception}' ) - def search(self, index_name: str, body: dict) -> dict: + def search(self, index_name: str, body: dict, timeout: int = SEARCH_TIMEOUT) -> dict: """ Execute a search query against the specified index. :param index_name: The name of the index to search :param body: The OpenSearch query body + :param timeout: How long to wait before raising a connection timeout exception :return: The search response from OpenSearch :raises CCInvalidRequestException: If the query is invalid (400 error from OpenSearch) """ try: - return self._client.search(index=index_name, body=body) + return self._client.search(index=index_name, body=body, timeout=timeout) except RequestError as e: if e.status_code == 400: # Extract the error message from the RequestError - # RequestError contains: status_code, error (type), and info (message or dict) - error_message = e.info if isinstance(e.info, str) else str(e.error) + error_message = self._extract_opensearch_error_reason(e) logger.warning( 'OpenSearch search request failed', index_name=index_name, status_code=e.status_code, - error=str(e), + error_message=error_message, ) raise CCInvalidRequestException(f'Invalid search query: {error_message}') from e # Re-raise non-400 RequestErrors raise + @staticmethod + def _extract_opensearch_error_reason(e: RequestError) -> str: + """ + Extract a human-readable error reason from an OpenSearch RequestError. + + The error info structure is typically: + {"error": {"root_cause": [{"type": "...", "reason": "..."}], ...}, "status": 400} + + :param e: The RequestError exception + :return: The extracted error reason, or a fallback string representation + """ + if not e.info: + return str(e.error) + + try: + # Navigate to error.root_cause[0].reason + root_causes = e.info.get('error', {}).get('root_cause', []) + if root_causes and isinstance(root_causes, list) and len(root_causes) > 0: + reason = root_causes[0].get('reason') + if reason: + return str(reason) + except (AttributeError, TypeError, KeyError): + # If navigation fails, fall back to string representation + logger.warning( + 'Failed to extract error reason from OpenSearch RequestError', + error=str(e), + ) + return str(e.error) + def index_document(self, index_name: str, document_id: str, document: dict) -> dict: """ Index a single document into the specified index. diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index 35ab18bd7..ca250efa9 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -531,7 +531,7 @@ def side_effect(): @patch('handlers.manage_opensearch_indices.time.sleep') @patch('handlers.manage_opensearch_indices.OpenSearchClient') - def test_on_create_raises_after_max_retries(self, mock_opensearch_client, mock_sleep): + def test_on_create_raises_after_max_retries(self, mock_opensearch_client, mock_sleep): # noqa ARG002 unused-argument """Test that on_create raises CCInternalException after max retries are exhausted.""" from cc_common.exceptions import CCInternalException from handlers.manage_opensearch_indices import ( diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index dd7745a83..b465e23e4 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -445,14 +445,26 @@ def test_opensearch_request_error_returns_400_with_error_message(self, mock_open mock_internal_client = Mock() mock_opensearch_client.return_value = mock_internal_client - # Create a RequestError similar to what OpenSearch returns for invalid queries - # RequestError(status_code, error_type, info) - error_message = ( + # Create a RequestError with realistic OpenSearch error structure + error_reason = ( 'Text fields are not optimised for operations that require per-document field data ' 'like aggregations and sorting, so these operations are disabled by default. ' 'Please use a keyword field instead.' ) - mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_message) + error_info = { + 'error': { + 'root_cause': [ + { + 'type': 'illegal_argument_exception', + 'reason': error_reason, + } + ], + 'type': 'search_phase_execution_exception', + 'reason': 'all shards failed', + }, + 'status': 400, + } + mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_info) event = self._create_api_event( 'aslp', @@ -467,8 +479,6 @@ def test_opensearch_request_error_returns_400_with_error_message(self, mock_open self.assertEqual(400, response['statusCode']) body = json.loads(response['body']) self.assertEqual( - 'Invalid search query: Text fields are not optimised for operations that ' - 'require per-document field data like aggregations and sorting, so these ' - 'operations are disabled by default. Please use a keyword field instead.', + f'Invalid search query: {error_reason}', body['message'], ) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index 42b1ac64f..f3d0195cf 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -1,3 +1,4 @@ +# ruff: noqa ARG002 unused-argument from unittest import TestCase from unittest.mock import MagicMock, patch @@ -107,10 +108,7 @@ def test_search_calls_internal_client_with_expected_arguments(self): result = client.search(index_name=index_name, body=query_body) - mock_internal_client.search.assert_called_once_with( - index=index_name, - body=query_body, - ) + mock_internal_client.search.assert_called_once_with(index=index_name, body=query_body, timeout=20) self.assertEqual(expected_response, result) def test_search_raises_cc_invalid_request_exception_on_400_request_error(self): @@ -120,21 +118,51 @@ def test_search_raises_cc_invalid_request_exception_on_400_request_error(self): index_name = 'test_index' query_body = {'query': {'match_all': {}}, 'sort': [{'familyName': 'asc'}]} - # Simulate OpenSearch returning a 400 error for invalid query - error_message = ( + # Simulate OpenSearch returning a 400 error with realistic error structure + error_reason = ( 'Text fields are not optimised for operations that require per-document field data ' 'like aggregations and sorting, so these operations are disabled by default.' ) - mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_message) + error_info = { + 'error': { + 'root_cause': [ + { + 'type': 'illegal_argument_exception', + 'reason': error_reason, + } + ], + 'type': 'search_phase_execution_exception', + 'reason': 'all shards failed', + }, + 'status': 400, + } + mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_info) with self.assertRaises(CCInvalidRequestException) as context: client.search(index_name=index_name, body=query_body) - # Verify the exception message contains useful info + # Verify the exception message extracts the reason from root_cause + self.assertEqual( + f'Invalid search query: {error_reason}', + str(context.exception), + ) + + def test_search_raises_cc_invalid_request_exception_with_fallback_on_missing_root_cause(self): + """Test that search falls back to error type when root_cause is missing.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + query_body = {'query': {'match_all': {}}} + + # Simulate OpenSearch returning a 400 error without root_cause structure + mock_internal_client.search.side_effect = RequestError(400, 'parsing_exception', None) + + with self.assertRaises(CCInvalidRequestException) as context: + client.search(index_name=index_name, body=query_body) + + # Verify the exception falls back to the error type self.assertEqual( - 'Invalid search query: Text fields are not optimised for operations that ' - 'require per-document field data like aggregations and sorting, so these ' - 'operations are disabled by default.', + 'Invalid search query: parsing_exception', str(context.exception), ) From ba3f2b85cd3b64a5983808072b443b204ff6db9f Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 16:26:11 -0600 Subject: [PATCH 114/137] Handling search timeouts --- .../python/search/opensearch_client.py | 14 ++++++++++++- .../tests/unit/test_opensearch_client.py | 21 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 083afe70f..b4e3236c6 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -152,10 +152,22 @@ def search(self, index_name: str, body: dict, timeout: int = SEARCH_TIMEOUT) -> :param body: The OpenSearch query body :param timeout: How long to wait before raising a connection timeout exception :return: The search response from OpenSearch - :raises CCInvalidRequestException: If the query is invalid (400 error from OpenSearch) + :raises CCInvalidRequestException: If the query is invalid (400 error) or times out """ try: return self._client.search(index=index_name, body=body, timeout=timeout) + except ConnectionTimeout as e: + logger.warning( + 'OpenSearch search request timed out', + index_name=index_name, + timeout=timeout, + error=str(e), + ) + # We are returning this as an invalid request exception so the UI client picks it up as + # a 400 and displays the message to the client + raise CCInvalidRequestException( + 'Search request timed out. Please try again or narrow your search criteria.' + ) from e except RequestError as e: if e.status_code == 400: # Extract the error message from the RequestError diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index f3d0195cf..47fceee9f 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -181,6 +181,27 @@ def test_search_reraises_non_400_request_error(self): self.assertEqual(500, context.exception.status_code) + def test_search_raises_cc_invalid_request_exception_on_timeout(self): + """Test that search raises CCInvalidRequestException when the request times out.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'test_index' + query_body = {'query': {'match_all': {}}} + + # Simulate OpenSearch timing out + mock_internal_client.search.side_effect = ConnectionTimeout( + 'Connection timed out', 503, 'Read timed out' + ) + + with self.assertRaises(CCInvalidRequestException) as context: + client.search(index_name=index_name, body=query_body) + + # Verify the exception message tells the user to try again + self.assertEqual( + 'Search request timed out. Please try again or narrow your search criteria.', + str(context.exception), + ) + def test_index_document_calls_internal_client_with_expected_arguments(self): """Test that index_document calls the internal client's index method correctly.""" client, mock_internal_client = self._create_client_with_mock() From 26c16bb73ad2851c10a9012381ece3eb11ec3ef5 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 16:26:24 -0600 Subject: [PATCH 115/137] linter --- .../lambdas/python/search/handlers/manage_opensearch_indices.py | 2 +- .../search/tests/function/test_manage_opensearch_indices.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py index 9d62269ff..c6a56d9d0 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -119,7 +119,7 @@ def _wait_for_domain_ready(self) -> OpenSearchClient: attempts=DOMAIN_READINESS_MAX_ATTEMPTS, error=str(e), ) - except Exception as e: + except Exception as e: # noqa BLE001 # Handle unexpected exceptions (e.g., connection errors during client initialization) last_exception = e if attempt < DOMAIN_READINESS_MAX_ATTEMPTS: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index ca250efa9..2ae84681d 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -531,7 +531,7 @@ def side_effect(): @patch('handlers.manage_opensearch_indices.time.sleep') @patch('handlers.manage_opensearch_indices.OpenSearchClient') - def test_on_create_raises_after_max_retries(self, mock_opensearch_client, mock_sleep): # noqa ARG002 unused-argument + def test_on_create_raises_after_max_retries(self, mock_opensearch_client, mock_sleep): # noqa ARG002 unused-argument """Test that on_create raises CCInternalException after max retries are exhausted.""" from cc_common.exceptions import CCInternalException from handlers.manage_opensearch_indices import ( From 6e790b1a96018fee13e93dc718ec1c5054f08904 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 15 Dec 2025 17:56:39 -0600 Subject: [PATCH 116/137] update comments --- .../provider_search_domain.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index a755a356f..f42938370 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -162,8 +162,10 @@ def __init__( # IMPORTANT NOTE: updating the engine version requires a blue/green deployment, which has consistently # failed to complete in both production and non-production environments due to failed dashboard health # checks. We suspect this is because of the 'rest.action.multi.allow_explicit_index: false' setting - # interfering with dashboard internal multi-index operations during upgrades. If you intend to update - # this field, or any other field that will require a blue/green deployment as described here: + # interfering with the OpenSearch dashboard health checks during upgrades. + # See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html#ac-advanced + # If you intend to update this field, or any other field that will require a blue/green deployment as + # described here: # https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html # You should consider the following migration process instead: # 1. Deploy a NEW domain with the target version (use different construct ID) @@ -172,13 +174,15 @@ def __init__( # 4. Decommission old domain # This approach provides full rollback capability and avoids blue/green issues entirely. # - # During significant upgrades, consider working with stakeholders to schedule a maintenance window during - # low-traffic periods where advanced search may become inaccessible during the update. During development, - # we found that if a blue/green deployment became stuck, the search endpoints were still able to serve data, - # but the CloudFormation deployment would fail waiting for the domain to become active. In such cases you - # may have to work with AWS support to get it out of that state. Worst case scenario, both the search API - # and search persistent stacks will need to be destroyed, redeployed, and re-indexed, hence why we recommend - # you create an entirely different domain and avoid the blue/green deployment altogether. + # During these upgrades, consider working with stakeholders to schedule a maintenance window during + # low-traffic periods where advanced search may become inaccessible during the update. This will allow you + # to perform the search api cut-over to the new domain within one deployment. + # During development, we found that if a blue/green deployment became stuck, the search endpoints were still + # able to serve data, but the CloudFormation deployment would fail waiting for the domain to become active. + # In such cases you may have to work with AWS support to get it out of that state. Worst case scenario, + # both the search API and search persistent stacks will need to be destroyed, redeployed, and re-indexed, + # hence why we recommend you create an entirely different domain and avoid the blue/green deployment + # altogether. version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, enable_auto_software_update=True, From 1c863ffd42cde88911fb7c023cf246214f896e75 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 12:08:59 -0600 Subject: [PATCH 117/137] Check compact fields to prevent multi-index reaching --- .../lambdas/python/search/handlers/search.py | 89 ++++++++++++------- .../tests/function/test_search_privileges.py | 82 +++++++++++++++++ .../tests/function/test_search_providers.py | 44 +++++++++ .../tests/unit/test_opensearch_client.py | 4 +- 4 files changed, 182 insertions(+), 37 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 8c2191e2f..b28ba6f43 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -122,12 +122,24 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus source = hit.get('_source', {}) try: sanitized_provider = general_schema.load(source) + # Verify compact matches path parameter + if sanitized_provider.get('compact') != compact: + logger.error( + 'Provider compact field does not match path parameter', + # This case is most likely the result of abuse or misconfiguration. + # We log the request body for triaging purposes + request_body=body, + provider_id=source.get('providerId'), + provider_compact=sanitized_provider.get('compact'), + path_compact=compact, + ) + raise CCInvalidRequestException('Invalid request body') sanitized_providers.append(sanitized_provider) # Track the sort values from the last hit for search_after pagination last_sort = hit.get('sort') except ValidationError as e: # Log the error but continue processing other records - logger.warning( + logger.error( 'Failed to sanitize provider record', provider_id=source.get('providerId'), errors=e.messages, @@ -210,42 +222,51 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu for hit in hits: provider = hit.get('_source', {}) - try: - # Check if inner_hits are present for privileges - # If so, use only the matched privileges; otherwise, use all privileges - inner_hits = hit.get('inner_hits', {}) - privileges_inner_hits = inner_hits.get('privileges', {}).get('hits', {}).get('hits', []) - - if privileges_inner_hits: - # Use only the privileges that matched the nested query - matched_privileges = [ih.get('_source', {}) for ih in privileges_inner_hits] - provider_privileges = _extract_flattened_privileges_from_list( - privileges=matched_privileges, - licenses=provider.get('licenses', []), - provider=provider, - ) - else: - # No inner_hits, return all privileges for this provider - provider_privileges = _extract_flattened_privileges(provider) - - for flattened_privilege in provider_privileges: - try: - # Sanitize using StatePrivilegeGeneralResponseSchema - sanitized_privilege = privilege_schema.load(flattened_privilege) - flattened_privileges.append(sanitized_privilege) - except ValidationError as e: - logger.warning( - 'Failed to sanitize flattened privilege record', + # Check if inner_hits are present for privileges + # If so, use only the matched privileges; otherwise, use all privileges + inner_hits = hit.get('inner_hits', {}) + privileges_inner_hits = inner_hits.get('privileges', {}).get('hits', {}).get('hits', []) + + if privileges_inner_hits: + # Use only the privileges that matched the nested query + matched_privileges = [ih.get('_source', {}) for ih in privileges_inner_hits] + provider_privileges = _extract_flattened_privileges_from_list( + privileges=matched_privileges, + licenses=provider.get('licenses', []), + provider=provider, + ) + else: + # No inner_hits, return all privileges for this provider + provider_privileges = _extract_flattened_privileges(provider) + + for flattened_privilege in provider_privileges: + try: + # Sanitize using StatePrivilegeGeneralResponseSchema + sanitized_privilege = privilege_schema.load(flattened_privilege) + # Verify compact matches path parameter + if sanitized_privilege.get('compact') != compact: + logger.error( + 'Privilege compact field does not match path parameter', + # This case is most likely the result of abuse or misconfiguration. + # We log the request body for triaging purposes + request_body=body, provider_id=provider.get('providerId'), privilege_id=flattened_privilege.get('privilegeId'), - errors=e.messages, + privilege_compact=sanitized_privilege.get('compact'), + path_compact=compact, ) - except Exception as e: # noqa: BLE001 broad-exception-caught - logger.warning( - 'Failed to process provider privileges', - provider_id=provider.get('providerId'), - error=str(e), - ) + raise CCInvalidRequestException('Invalid request body') + flattened_privileges.append(sanitized_privilege) + except ValidationError as e: + logger.error( + 'Failed to sanitize flattened privilege record', + provider_id=provider.get('providerId'), + privilege_id=flattened_privilege.get('privilegeId'), + errors=e.messages, + ) + # We don't want to return partial privilege reports + # If we experience a failure here we need to exit + raise CCInternalException('Failed to process privilege results') from e logger.info('Found privileges to export', count=len(flattened_privileges)) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 91f6970c9..5eaa34328 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -739,3 +739,85 @@ def test_export_query_with_nested_index_key_returns_400(self): body = json.loads(response['body']) self.assertIn('Cross-index queries are not allowed', body['message']) self.assertIn("'index'", body['message']) + + @patch('handlers.search.OpenSearchClient') + def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_client): + """Test that a privilege with a compact field that doesn't match the path parameter returns 400.""" + from handlers.search import search_api_handler + + provider_id = '00000000-0000-0000-0000-000000000001' + # Create a provider hit with a privilege that has a different compact than the path parameter + hit = { + '_index': 'compact_aslp_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': 'aslp', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license-home', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': 'aslp', + 'jurisdiction': 'oh', + 'licenseType': 'audiologist', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2020-01-01', + 'dateOfRenewal': '2024-01-01', + 'dateOfExpiration': '2025-12-31', + 'npi': '1234567890', + 'licenseNumber': 'AUD-12345', + } + ], + 'privileges': [ + { + 'type': 'privilege', + 'providerId': provider_id, + 'compact': 'octp', # Different from path parameter 'aslp' + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'audiologist', + 'dateOfIssuance': '2024-01-15', + 'dateOfRenewal': '2024-01-15', + 'dateOfExpiration': '2025-01-15', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'administratorSetStatus': 'active', + 'privilegeId': 'PRIV-001', + 'status': 'active', + } + ], + }, + } + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + # Request for 'aslp' compact but privilege has 'octp' compact + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual('Invalid request body', body['message']) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index b465e23e4..ae5e42d63 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -482,3 +482,47 @@ def test_opensearch_request_error_returns_400_with_error_message(self, mock_open f'Invalid search query: {error_reason}', body['message'], ) + + @patch('handlers.search.OpenSearchClient') + def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_client): + """Test that a provider with a compact field that doesn't match the path parameter returns 400.""" + from handlers.search import search_api_handler + + # Create a provider hit with a different compact than the path parameter + provider_id = '00000000-0000-0000-0000-000000000001' + hit = { + '_index': 'compact_aslp_providers', + '_id': provider_id, + '_score': 1.0, + '_source': { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'compact': 'octp', # Different from path parameter 'aslp' + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2025-12-31', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-15', + }, + } + search_response = { + 'hits': { + 'total': {'value': 1, 'relation': 'eq'}, + 'hits': [hit], + } + } + self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + + # Request for 'aslp' compact but provider has 'octp' compact + event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertEqual('Invalid request body', body['message']) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index 47fceee9f..f71a9847a 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -189,9 +189,7 @@ def test_search_raises_cc_invalid_request_exception_on_timeout(self): query_body = {'query': {'match_all': {}}} # Simulate OpenSearch timing out - mock_internal_client.search.side_effect = ConnectionTimeout( - 'Connection timed out', 503, 'Read timed out' - ) + mock_internal_client.search.side_effect = ConnectionTimeout('Connection timed out', 503, 'Read timed out') with self.assertRaises(CCInvalidRequestException) as context: client.search(index_name=index_name, body=query_body) From 8b582b96d9f8ef049637fe3ce7deed97924f9222 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 12:34:28 -0600 Subject: [PATCH 118/137] Add error alarms for search and ingest --- .../provider_update_ingest_handler.py | 29 ++++++++++++++++- .../search_persistent_stack/search_handler.py | 31 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py index 7cb572247..b63a881e5 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_update_ingest_handler.py @@ -6,7 +6,7 @@ from aws_cdk.aws_ec2 import SubnetSelection from aws_cdk.aws_iam import IRole from aws_cdk.aws_kms import IKey -from aws_cdk.aws_logs import RetentionDays +from aws_cdk.aws_logs import FilterPattern, MetricFilter, RetentionDays from aws_cdk.aws_opensearchservice import Domain from aws_cdk.aws_sns import ITopic from cdk_nag import NagSuppressions @@ -154,6 +154,33 @@ def __init__( treat_missing_data=TreatMissingData.NOT_BREACHING, ).add_alarm_action(SnsAction(alarm_topic)) + # Create a metric filter to capture ERROR level logs from the provider update ingest Lambda + error_log_metric = MetricFilter( + self, + 'ProviderUpdateIngestErrorLogMetric', + log_group=self.handler.log_group, + metric_namespace='CompactConnect/Search', + metric_name='ProviderUpdateIngestErrors', + filter_pattern=FilterPattern.string_value(json_field='$.level', comparison='=', value='ERROR'), + metric_value='1', + default_value=0, + ) + + # Create an alarm that triggers when ERROR logs are detected + error_log_alarm = Alarm( + self, + 'ProviderUpdateIngestErrorLogAlarm', + metric=error_log_metric.metric(statistic='Sum'), + evaluation_periods=1, + threshold=1, + actions_enabled=True, + alarm_description=f'The Provider Update Ingest Lambda logged an ERROR level message. Investigate ' + f'the logs for the {self.handler.function_name} lambda to determine the cause.', + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + ) + error_log_alarm.add_alarm_action(SnsAction(alarm_topic)) + # Add CDK Nag suppressions for the Lambda function's IAM role NagSuppressions.add_resource_suppressions_by_path( stack, diff --git a/backend/compact-connect/stacks/search_persistent_stack/search_handler.py b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py index d2872e553..798083649 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/search_handler.py +++ b/backend/compact-connect/stacks/search_persistent_stack/search_handler.py @@ -1,9 +1,11 @@ import os from aws_cdk import Duration +from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, TreatMissingData +from aws_cdk.aws_cloudwatch_actions import SnsAction from aws_cdk.aws_ec2 import SubnetSelection from aws_cdk.aws_iam import IRole -from aws_cdk.aws_logs import RetentionDays +from aws_cdk.aws_logs import FilterPattern, MetricFilter, RetentionDays from aws_cdk.aws_opensearchservice import Domain from aws_cdk.aws_s3 import IBucket from aws_cdk.aws_sns import ITopic @@ -98,3 +100,30 @@ def __init__( }, ], ) + + # Create a metric filter to capture ERROR level logs from the search handler Lambda + error_log_metric = MetricFilter( + self, + 'SearchHandlerErrorLogMetric', + log_group=self.handler.log_group, + metric_namespace='CompactConnect/Search', + metric_name='SearchHandlerErrors', + filter_pattern=FilterPattern.string_value(json_field='$.level', comparison='=', value='ERROR'), + metric_value='1', + default_value=0, + ) + + # Create an alarm that triggers when ERROR logs are detected + error_log_alarm = Alarm( + self, + 'SearchHandlerErrorLogAlarm', + metric=error_log_metric.metric(statistic='Sum'), + evaluation_periods=1, + threshold=1, + actions_enabled=True, + alarm_description=f'The Search Handler Lambda logged an ERROR level message. Investigate ' + f'the logs for the {self.handler.function_name} lambda to determine the cause.', + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + ) + error_log_alarm.add_alarm_action(SnsAction(alarm_topic)) From 34cf9e3d85415a4252bb688d40835220f41fbc28 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 13:48:01 -0600 Subject: [PATCH 119/137] Cleanup/update docs with latest information --- backend/compact-connect/docs/design/README.md | 161 +++--------------- .../docs/design/advanced-provider-search.pdf | Bin 0 -> 122181 bytes 2 files changed, 24 insertions(+), 137 deletions(-) create mode 100644 backend/compact-connect/docs/design/advanced-provider-search.pdf diff --git a/backend/compact-connect/docs/design/README.md b/backend/compact-connect/docs/design/README.md index 8189d8da3..2513a6c72 100644 --- a/backend/compact-connect/docs/design/README.md +++ b/backend/compact-connect/docs/design/README.md @@ -687,59 +687,29 @@ Provider data from the provider DynamoDB table is indexed within an OpenSearch D queryable by staff users through the Search API (search.compactconnect.org). The OpenSearch resources are deployed within a Virtual Private Cloud (VPC) to provide a layer of network security. ### Architecture Overview +![Advanced Search Diagram](./advanced-provider-search.pdf) The search infrastructure consists of several key components: 1. **OpenSearch Domain**: A managed OpenSearch cluster deployed within a VPC -2. **Search API**: API Gateway endpoints backed by Lambda functions for querying the domain -3. **Index Manager**: A CloudFormation custom resource that creates and manages indices +2. **Index Manager**: A CloudFormation custom resource that creates and manages domain indices +3. **Search API**: API Gateway endpoints backed by Lambda functions for querying the domain 4. **Populate Handler**: A Lambda function for bulk indexing all provider data from DynamoDB - -``` -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ API Gateway │────▶│ Search Lambda │────▶│ OpenSearch │ -│ (Search API) │ │ (in VPC) │ │ Domain (VPC) │ -└─────────────────┘ └─────────────────┘ └─────────────────┘ - ▲ - │ -┌─────────────────┐ ┌─────────────────┐ │ -│ DynamoDB │────▶│ Populate Lambda │──────────────┘ -│ (Provider Table)│ │ (in VPC) │ -└─────────────────┘ └─────────────────┘ -``` - -### OpenSearch Domain Configuration - -The OpenSearch domain is configured differently based on environment: - -| Environment | Instance Type | Data Nodes | Master Nodes | Replicas | EBS Size | -|-------------|---------------|------------|--------------|----------|----------| -| Non-prod (sandbox/test/beta) | t3.small.search | 1 | None | 0 | 10 GB | -| Production | m7g.medium.search | 3 | 3 (with standby) | 1 | 25 GB | +5. **Provider Update Ingest Handler**: A Lambda function for updating documents in OpenSearch whenever provider records are updated in DynamoDB. ### Index Structure -Provider documents are stored in compact-specific indices with the naming convention: `compact_{compact}_providers` -(e.g., `compact_aslp_providers`). +Provider documents are stored in compact-specific indices with the naming convention: `compact_{compact}_providers_{version}` +(e.g., `compact_aslp_providers_v1`). We use index aliases to provide a stable reference to the current version of each index (e.g., `compact_aslp_providers`), allowing read and write operations to be transparently redirected during planned index migrations or upgrades. This enables seamless index schema changes without requiring app code changes, as applications and APIs can continue to reference the alias rather than a specific index name. See https://docs.opensearch.org/latest/im-plugin/index-alias/ for more information. -#### Index Mapping +#### Index Management -Each provider document contains all information you would see from the provider detail api endpoint with `readGeneral` permission: +The `IndexManagerCustomResource` is a CloudFormation custom resource that creates compact-specific indices when the +domain is first created. It ensures the indices/aliases exist with the correct mapping before any indexing operations begin. -**Top-Level Provider Fields:** -- `providerId` (keyword): Unique provider identifier -- `givenName`, `middleName`, `familyName` (text with ASCII folding): Provider name fields -- `licenseJurisdiction` (keyword): Home license jurisdiction -- `licenseStatus` (keyword): Current license status (active/inactive) -- `compactEligibility` (keyword): Compact eligibility status -- `dateOfExpiration` (date): License expiration date -- `npi` (keyword): National Provider Identifier -- `privilegeJurisdictions` (keyword array): Jurisdictions where provider has privileges +#### Index Mapping -**Nested Objects:** -- `licenses`: Array of license records with full license details -- `privileges`: Array of privilege records with privilege details -- `militaryAffiliations`: Array of military affiliation records +Each provider document contains all information you would see from the provider detail api endpoint with `readGeneral` permission. See the [application code](../../lambdas/python/search/handlers/manage_opensearch_indices.py) for the current mapping definition. The index uses a custom ASCII-folding analyzer for name fields, which allows searching for names with international characters using their ASCII equivalents (e.g., searching "Jose" matches "José"). @@ -758,96 +728,14 @@ and military affiliations. #### Privilege Search ``` -POST /v1/compacts/{compact}/privileges/search +POST /v1/compacts/{compact}/privileges/export ``` Returns flattened privilege records. This endpoint queries the same provider index but extracts and flattens privileges, combining privilege data with license data to provide a denormalized view suitable for privilege-focused reports and exports. -### Request/Response Format - -Both endpoints accept OpenSearch DSL query bodies with pagination support: - -**Request Body:** -```json -{ - "query": { - "bool": { - "must": [ - { "match": { "givenName": "John" }}, - { "term": { "licenseStatus": "active" }} - ] - } - }, - "size": 100, - "sort": [{ "providerId": "asc" }], - "search_after": ["previous-provider-id"] -} -``` - -**Provider Search Response:** -```json -{ - "providers": [ - { - "providerId": "...", - "givenName": "John", - "familyName": "Doe", - "licenses": [...], - "privileges": [...] - } - ], - "total": { "value": 150, "relation": "eq" }, - "lastSort": ["current-provider-id"] -} -``` - -**Privilege Search Response:** -```json -{ - "privileges": [ - { - "type": "statePrivilege", - "providerId": "...", - "jurisdiction": "ky", - "licenseJurisdiction": "oh", - "givenName": "John", - "familyName": "Doe", - "privilegeId": "PRIV-001", - "status": "active" - } - ], - "total": { "value": 50, "relation": "eq" }, - "lastSort": ["current-provider-id"] -} -``` - -### Pagination - -The API supports two pagination strategies following OpenSearch DSL conventions: - -1. **Offset Pagination** (`from`/`size`): Simple but limited to 10,000 results - ```json - { "from": 0, "size": 100 } - ``` - -2. **Cursor-Based Pagination** (`search_after`): Recommended for large result sets - ```json - { - "sort": [{ "providerId": "asc" }], - "search_after": ["last-provider-id-from-previous-page"] - } - ``` - -**Note**: When using `search_after`, a `sort` field is required. The `lastSort` value in the response can be passed -as `search_after` in the next request. - -**Limits:** -- Maximum page size: 100 records per request -- Default page size: 10 records - -### Data Indexing +### Document Indexing #### Initial Population / Re-indexing @@ -860,7 +748,7 @@ The function: 1. Scans the provider table using the `providerDateOfUpdate` GSI 2. Retrieves complete provider records for each provider 3. Sanitizes data using `ProviderGeneralResponseSchema` -4. Bulk indexes documents in batches of 1,000 +4. Bulk indexes documents **Resumable Processing**: If the function approaches the 15-minute Lambda timeout, it returns pagination information in the `resumeFrom` field that can be passed as lambda input to continue processing: @@ -872,10 +760,17 @@ The function: } ``` -#### Index Management +#### Updates via DynamoDB Streams -The `IndexManagerCustomResource` is a CloudFormation custom resource that creates compact-specific indices when the -stack is deployed. It ensures indices exist with the correct mapping before any indexing operations begin. +To keep the OpenSearch index synchronized with changes in the provider DynamoDB table, the system uses DynamoDB Streams to capture all modifications made to provide records (see [AWS documentation](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html)). This ensures that provider documents in OpenSearch are updated automatically whenever records are created, modified, or deleted in the provider table. + +**Architecture Flow:** + +1. **DynamoDB Stream**: The provider table has a DynamoDB stream enabled with `NEW_AND_OLD_IMAGES` view type, which captures both the before and after state of any record modification. + +2. **EventBridge Pipe**: An EventBridge Pipe reads events from the DynamoDB stream and forwards them to an SQS queue. + +3. **Provider Update Ingest Lambda**: The Lambda function processes SQS message batches, determines the providers that were modified, and upserts their latest information into the appropriate OpenSearch index. ### Monitoring and Alarms @@ -887,14 +782,6 @@ usage metrics to determine if the Domain needs to be scaled up: - **Storage Space**: Alerts on low disk space - **Cluster Health**: Monitors yellow/red cluster status -### Important OpenSearch Domain Maintenance Note -**WARNING**: Updating the OpenSearch domain may require a blue/green deployment, which has been known to get stuck -and require AWS support intervention. If you need to update any field that triggers a blue/green deployment (see -[AWS documentation](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html)), -be prepared for worst-case scenario of deleting the entire search stack, re-deploying it, and re-indexing all data -from the provider table. You should work with stakeholders to schedule a maintanence window during low traffic periods -for major updates where advanced search may be temporarily unavailable. - ## CI/CD Pipelines This project leverages AWS CodePipeline to deploy the backend and frontend infrastructure. See the diff --git a/backend/compact-connect/docs/design/advanced-provider-search.pdf b/backend/compact-connect/docs/design/advanced-provider-search.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9c0841b2fe54ab5b84bc6706f2f7d025c7aa1da4 GIT binary patch literal 122181 zcmV*BKyJS!P((&8F)lX>CBlHhfx?2qg93N#=vAa7!73Oqb7Ol59obZ8(oGB`0gFGgu> zbY*fcMr>hpWkh9TZ)9aYJ_>Vma%Ev{3V59Cy~~a?xso2ZKTi=GE&n*qi#ti6(ZHyW z(2#&$NWGZFKoF#mx=TXj42~p7s<$7v?=xOJE}0RG%#6$|qEPu8!`JG}g9?`Qm;|6PAFdTai7{Yn0L*|}ux3LFWTWil%~W#G3z{S}Yq|Nr|x z@BZme|5ks@{rw7Qk0>fBwsV{a?Gk`I|rg!{7buzwD;J{oCEY{LA017ykVB z|2X}J=@VcLnoInVI~9* z=(r1UBJtv1B8PC8<(_;;0D18_!bA~a!4aYS+})G)Swp&e;?A)@1HE%zZWq?}_DAQI z{kpcpJ_N9bX)(c)LsamLN&G7OnP29EPXOG=o2I;Bp1`{z?Hiv9WeN3znUt$4xEp z%UDvg+u-##eb`vn;*d=fF-l`>)6juw(>h!;iI|_t&^YisHj_!9dD%_a_OTD!PASm2 z&9}Su+m~?O%6Fnw9=UO_bKhVub$s6W<+UE-bJx0wGtd@w2}V+v6Z9>*ph2&s>PUhO zbEPXw+96$8p#RqrmH6d$GvN;iQf~FUpJeFx4m^ujI7%l1&RpTVe_1fEV9meSWuj!=lgEu8$9sT? zU*I(_yEzn$kvn4)iNEsG*f3nA+SnHkj&GD7eFlDU^VfzFczWN0KhIrncqTxM6l&-|QGq!ljU6ENB;WVnC+? zfxbv4SXv-j#bNyNyJy5>I-OXCWJTPDv}H)g zxy26fX?PqTLypi)%DOanco{MLq7BgD!k5V3SiyC~x~H=*?$4=Cf+B!xE6L}uPoTbe z*G*hu?xVZ3)QMf$Let^^3_f1RMLg)pmNKpQN_M1(&O#We3O9j;DsGi0>!i&=XJ9U< zBZ9zZGLJcs4}jfpRw~UWaxC41Fo|UT7VF4x4>I|+i!Lq*ExO}_2>Ic^IO0Pdxp9;R zjEWI6ZdC~lPqaw9X|jUtWD;4TA&Ez2QF#3ln}!Z~&hoYOEr2C8mzG^j^XRrEi)k*c zq^r$s7_A9Q+f>dm>AkVY_Wo5V>L#VtT~=D{6{Qtu7LFV*Dy`egmo&sB6%BDxy>4nW zM8MvQ_?7SQt8iiUIl|Ns35$kEC_hIHai4rz#kDlsRUb^t6y8!@KWku*tF90y+`7Kv zjjF4VoO`A?`y9Q&Onp4TG+XP-x~b_}7bw|`D*`%_< zKrwv7?_48Gk<$X4^ZRUmo!tmz^R}hT8?p}34mPb^Xr?ZpXvSTtb3hT^AmJts?E0`h zgy>HZqJ8IDXeG;4idI7dCCk|E`Vm8qq$=u$kt>-EENiu4aB)(9%W|FSY;uLmgccBp z(UkEHSq}ACz>I|yC?WgOFR2!W9sG#FxhmeP;AQj+KryEIw5>q>+m0W(%DHOJQHdS; zL%Ub&qVp8m^dT%~Rj+LWtFFE2`=-fUdKsIN7eB%kaU|Ojpa67n88F1mWgt8&X4P6{ z>s_2fstS2T5Y-Q*xvqV%&vv{wAtspAhnT_fJ2-yo#P(3khhqMSVs4B{_miC}Y%ljK zYUYyvQ@nqy516j)Eb8TOQw=v&3k)~ak91Q_X6^WyZYrPm=@*KSP`kKWQAHqD zsk#~0(hSunY+%;t=CLT%^BVrK$_WftpQ;JEJXC&r^8ZvDhh2L~ zg>Ohg3hQv$0^5olAD5w05kH9n2U=uJG|Q3;{ZjE)TS0YCwh}#Orm+N<76qZ&RI;?Q zyU>iLZ8_zbwo#5!QI5)d-R2kTqv#>v4FT^V0k4zo4^vX}x{os@b?$S+l=Kx%Nuh5t zX(FMJp9M{$lIFtQDQ1y<&N9?5vmyFTi&hq^yP$FHHRHpk8Dn#=)wkw_wQRb*$xGlm zw8^o@jXmx$BZloFyU=U^FFVM=ju_#*;g5WO1kbaX3=f3WumRX}Vn9k@HV7906(}F# zbM1O^n_Ubz$ewZIH$LmlNN-z+zQ?Tdi2AhZ)lZG`gA$FU`oh_8v;pihi%?C0beDez zdr^G`ButrR(dkmfN@qGYH>$OmOraVR4*5dJ>j5*$jsiucI?`LMUi+S1MfLSS5ssRo z2FzHaya0=nfq|4@gpcJdrkaN`j@R;8? z_f3**Xjc}s(wuI3HX9C;Yk_h@+R%DDZ|D+<+vQ^~HOp$UZ8w_%-Y5Wv4n#b#@A`G3 zpK7)oE{bc6eYhxwi{i>)Riu2`spe$*2U#?Hd z+oNo6e`#m%JutX5U7r!e+heOANIbp^=B@-Zulug7k=t`|=VhPFGbM;tXL%`G$VTz7 zDi=)VQq%ZNbJ{3Bo7UVkZr(N37kSz&Tx71pKUKhPkuW;ddxqLLZX??AYQ}vt3@6{I zEwy^JeTuMf66KTT=B^Sax16{1#;zD2oBC9VA0LkCP3S_*@DzPPPtnphnLo%=6vNU* z&cjtSTtyGLil(r1?zT^ecb(_2JJy@@zi?Uqo8L$O1AS#isKEB00|C|LDtzhBVOMTE z?WSDk#rhv}*IPh$`u~Zd@yb+y53hfske&RK zw}}E@f?#H_Vk`Hwy(oMMIy65Vuipmf(7fn){q=wjuT^FG?w_@p@X$`U*?Z$3nsRwd z*e_zkTK{rP89MzJ`|Khys|kWdhA%;f=7;0;+W;NPTaFjs8*tAn6a2kCVl$obeWx=D zl}(^@V^?CcVQ=68$?zozfe=aH$a@vO1g#smP_w~z7VsQ6?8;D%6+8!Co!4*rHJb@e z?>oWuV_)jKkJrW%$(!=G9It&}B3~-VIbKH@phJW7ciANH9$l z$gyizivQznLH2d zB<{AJuQ?DQ7|L-Ad%SfaZm`6q%digWKyX>XBS+GJ;S3Zik>^1q(qsMlw|Lq>QbE#| z8tuDNBUZ=DuEIx-h3gwD-R5|ci@V>Wca-eF1sx4)@_oPf&n;6c^u7F1_ za~T*GzKjdY6yrK@&0ubh8yJg9rU}c8>n-GcXUX6Zpzbc=3K%&*Sol7315D3>L&20B zH!xU=%nq0gxlPR54R$8>o-z-6rMUNB2VzdB&c7=^^!$*8Rzrn#vsvL;pt(Sf*AESh zdm&Z?feg{Yd{OFF6+RXi%mX+G3Tc34fw562y4Hz?n1cnrFz)`)z~ZCW!=vv7GH~ki zm3dbyF!sMmr)A;9C4LH?I`Csx*cE8u=dDoWb%oJaVn*)zUMG|=bNK5`hVNBTZT$ga zn3JX@bZFos-4ndS(?pTL(gyn#>^gG8u&hVTPeIJcZ}X03d4a(~ zEVng`2+PPRsV$t9ko}i{kL4o9knaVzQSqw4z;G;olK!DrhZqJOM$W)euH*TI*A+f; zHq6W!z(-lRKFg}odlkmLRw*51XY(m*gKV`Fhh}Jw3gg+kv}SsC@f*VoRwK1UG`QLB=9xM1sb?J7@^b_;1hRx0=9wrzY^Kzdl|$c4xWluxBLf)pK`FE?)}R_9b9@*4X5^ z3D}-XZQy!`M;Hm8^UWbQz8rGn^^hCicka$-4e8rC7M6#5i;h=*|a8H(ZH}C`S+3rw!QpiQf5sYIkO{Y z_O-EnV$Q6EtjoV6XZEx;a%M-)?6ey>vm}GFxsB;A) zsao8m2;!rkYAvI-BgrQzxYL^;5M_l8-|VM17m0L=RuC|LyoD> zurwU;&FenS5pSlR<1mLr@)ac#W4q>ZoV7f4oV9Fx53&bW`y8$|fA%52DKUJ01Q(9r z!m;{CaN%%qV1WBSl8a*m7mnb<+W4`-h5oW<$xhciOL3(cch$4hUcKbTao{r8h4g?l z{!2FwVXKiy+QQWGIl>xRXtNu~>g4meyFE*vHKcd&EX8@n?j4>brYjH6(jUaL^y8fo zoq1+w1}+Yl^rfm6Q~F}N{i*FsZB%))q%eUS%BITA(P`p+)5Bab=3A@Usp&~m2`7#( z--_-qR*I8^OW!KY#kxweL8cKq4_UlKE6X6Uz}lP2ARrK`2PS`O7QQ~vSu~OFzU*o| zR!fB2V{v(uo23(>2)lXb+-#)7zw$}`mDs2pFIiYrT30C#VoRQ`e44J`pi1SQ+6T-aqvm))VQQjK^YDxB>mj)i_i@CEfE!uF0I~WUSShze~WjL05z=3)oIv zrmn)}#gk4y7rIKUvgV3ecGcTR>-rB-A>S{`H za=X2acKz$!Z0qN~I*wfPY+@nt(kcDZd~tAfJoIFwrjbX~HXa%hh<;Yo%(J3KBll1@ ze5p9lpX)lLy2HrnWJkHy%b#%=YuRC~~4H$Wczr8%+Z*XYk|5 z(UAZl+9_dEk&_3+2Ta>o&KnOShkRSGLkP=CtCZsKh1e{tT)nhDc+z+rMR`12P};3F zY{gO@mOf)Qt#G0B$jB^J$ZT}4olj@~9xg3;qUa)c*l>;tUxM`Ba^rOJkDlUZ@Qr?+ z(rzEDneX^UH4SonuBAQ35MKlj7Z>tWZaM`I8-`HfOOW21jg1Aq1U>%wQ0w#OFD*@> zaTz>ZT&fjEr{H0;w^OXu)myP%uKWl-I$TZ@lo8bDH>-x48e~K0LKX=5YU41Lbi4 z5BL9757Kb|Q(74A|Ka{0?*HNbAMXF*{{Qao|CORSwMoC!QJb(# zx)vxmC`;0i=6K%Fka#w7RYy(BI%(!2kJ_~D5bmk+NQxX@y5Xh!T;ILxq&u^fiXxbObJQ1@DE;V& zQyopa3u=C5)?sQZ>oC5;r#cKZ{ZJQ$_Leh%mUe}3M6Q0cnp4<-?Tn+gO;2|yF@1CW zoZ5EF+$FCM(d}!9Znwn(q${xi_DU=O%~@Bg|7|Z{$%%5A2y^-^|J9r*Ru+#IXZhup z9K*=x2s0;2;6yVie&ur!IZ^-j?*IMMpZ~A_{y+cgKmEtU|195YYrXsD4ehTO++Y8c z-vK6=Z<2pk)cyHioBxy+H~nY-UzY!}%u5W(E&tK|=f(cVI350J{!h#9FaO_WZu7Uk zJ?Yk5nRIt~(j$F{e&W#1^!n{(a;ha;TUbaj`yvXE$yPA2WHb~&5b%$`)saoR|MA z*q6iU=PjJ-_s{_>#~RLhR3Usza-b-VKm;s>^990Th%=!B4@S(tx^kz<&kLr!2fE~i zUQRBQ6D398$pGXpLH(lld7i48c&e=3rF<^tfqgSG7O$2s#mG_~I1fX|*CZYxIkSjG zh8$Qr#*8<<5cbqPGK{bC9Plh>SmG;?46O)3=wcx3S9HXi$n>2HRaBNTo?Gf z6G*24kmm`kggI$Wh@*vdl5x)lR%2+$q~hevQjm|6lUJRN4TM5sb8?QQ5}})U&y;^x zqQ(76%;?9KcY2aO-iYOpp_IQt7Z1kW!n)XS&jz0A_ZY=Wz;ptAoMK>C1_9(Nko4j_ z@xWo7QYEB3>2ovvI7)Ql(Y^^*$$ZZ1k+9bl3Brg#610%WyIutQo=+bM>s`1Qhow(Myiw1HgI_Y z{erCY9hrAXzb&k!r|-}w0B7Qpep@&{@u#a<)`R&D<~x}0V7`O-ekAkV%FXl5a`WqY z$CXO+wmI%u4618;UX^pt>SRV6?TJiv46l~CSe0s1yYlOI%#MqLjV6WTyq{*zxuc(Z zM;OiKV{Pe-4SmhqBfPzzJ4g8XjJhi`dgb`I#Ph48rTyrv0POa1^w5b*oa+qo)DTUN z55hmj7GJ(CjdDwQ5dUzY8XU_WE!G_yjtm69&1ACj3qkp-HwYMcbUl9I)B0d;{6^D| z5q|O_@WQj{JFvB}VkXntFlZ$1c51b9xP^`^&kYm5 z(a1WZa7sQpYOGggrc9Hy!0d|(7vJy(?2?cHofc?aKBl*sx;fWz1Sg*C1_>!cY z7Zx~0MG$tQSCZ~Lpwh|Yjs%R_J3zejpp8IDj*XZ^=w&&L>!NjT3wmOrKxvuqT z(=yI2H?{nvKDudzwpT5KV(on1GFsZ91n-r#^^rLVO>ELQj$nUd3`*x|R3i-gTZdsU zy=!bc?O|$2FQ>OOjgu%djGHC)_ZOnfR-772UVssf~#1D%u zV`!$n7}WT@=(6}uhX#%mrv2p={a^}{nS96e{M2+Hv&zI{)8Bv68APrJ*`q#K^Sbw} z4|bcoY`)@LxppwKjp@2?WqUP=D|6kf#f%i-x?f;m+Q4&$gV;5bG*`$9CKehlC$Uo4 zY^=Bhi#HyyLqQ?2ViFR|<$NY6$Pqpg z4)nxEx#tobG8_e+L43neJ}Lx^YL2wJdXoRLft|_z?aMH1u^I$|U&9W2s~G&Djo*IC1iX<)uotzlTD{3+r7hkTQkTC9)j zQ%$2ObY=LbxoEN~#^#KOrvgGoTma6DhGRotB4C;r8b?=d)LsD|c##eMZGOlDt&O8L z&>QY7?-Jn6J$0b>arAD9-`-99r{q7UyWEVc;9#fWde}6F>*0@Vg&3}f{ct_pzQ%^@ zVYnWK>tVPahU?)wZ~DXa@TFW2CzIoQCg*{8ELkn_nM#pKK^YDqInf$xQi;ZpuVj^^ zZRwr&#Sbp-yk*k7QJF{`c?2g-y;qwx+|yiW<{?-u=A0k9RT~-y)r=BYPbh)ad3@E_ zkoP?s^3sRxp@n~IEj;$k`g2N;B{xcsedE$&r`nbqRb%f6`w5rBe%zJnT&@(Snl8ot zoLpAX*Jjc6&GV6L>lb?c?_pQ(Kku>L!|U}wb%yzME8Fd@oc6S82X=1VVPALH+c~;Y zVGPSa|9{29dNNl*iZZ0YxEip<8Lr|DEI}1X{sakviBfq55jJsQqM{^KM$N!$;YBhg zr6We>$gAr65MLqzDobGEW`Xgf6*=0={R?Y#!1wJfqHr)eWqKP}+}A!aI0adO3nk7a zFfK%vXAsF-{E8Jm%2KtWnmfjkSlw@(to4QbJ4*aOB>OJ#ECaIgu%1iI^u>jq^uuda zh6d_C9`)OmbEU>e-^y7e!8RBmu) z3E*1fG$0^&Vey(*DCrqP* zWiVBsI*F`y#FI9#jpS^Jm7?ZY%CV&qhjInu&mPtmD;3ZT0q?C29Aty zL&3EmE;}h&#PVnAnRHSJLY2r0!;&abXsrAuT(G|HRg6D-SR0J{$HHFExr(9XSp^mi zlG(traxG%8vBe_>8+9XCDG$nyWht}`Eb}n_{cIQ*i_uYYB5$ICtoK^ufj)#Q4Lb|qoCFQoCs!trgD!Ic2?F~uwod%^P z5%`aCo`(ijDb?!_DwLC#1_nE~sDZ^1X*$~ zVPFIu;!aoNVJmzj94GE2LBu1Ca!t$;sW9BxR_~A(C8Tc{idT^;U_2Gq{55f{z(=`= zib{!{vzs!I17Qpg1Fgdc^tuK#$a9eA7a-3q2X^nufgMY%&N5{M*3$_9&lq5&WFI*{ z2ZgcgOcaPzqX%>#5&=VC!kvWpe1$JT?R6EoFaq?7QPr^?8u$`)U?HT$F}yS|^5jDE z6;3j42yqmIW%uTtF@L4wwhM1(9U1Qo6-yO9DvUQQ86$Npuk)cXNndD^wyW`7?dITi zi-Ren!U)Z>gyBhrp$*WyP?Q90NduO}^6nH)M;8{m8qdaA-Zy_v&vvDWiVe{KGp>0teH1SSCI?4hc8~75G@AWw;jbB{G z`nv2tCgBq&VY`~%)s`Ltcexpg;uIBswHz!e@h<@-4S-UB7Q8J3>rxr+c zOsHV&M~-zxtXC?bGd3rFXmA{2yJNYNU{ZZX<31UGX*cJ}GMuEe80|8r+FN}aVNs8! zdL=Plgn25OnBk+|w>`4fX=sTCQ8i!rk@30DsrBjl2Kzv4`KtPvr^c4+8N`fIl7LhV z8Utkn&7@_(WU7rr4!lK6!>eKgYl;I~$m)uwlAo{eAqTdhWv5*Ac7<7gHt3l=%o_>Mri#re#|BxB;J#AAmRT)1k`A0^VyvL6aCz~h zF=&i7Ocj3@>j>v1+vZK*QkmEEMO-F+_)AGnGy9TCMZ}P*?UAT<6#hXZfb}+g6hUon zUyfRseOlSzx>uR?uUmOxLB`vIgtGBMHw|C|&ceb#MqK>PLFLTJYf1Hjr}*lW4Z|*3dgA-=z_IJh^G9-~=xZt)0>z{f4NhBHA03PSgm%;a^S) zq)>Y*pLTd0MX~~v;y5}Kvf(&0ZZMOma-(J>%Q09AiI_>ixXtd$qwLKxkUAgAtOZ0u z^t-*YX5Q`XV#7USh6(3Iyj5z>kuv693O4z6@(wyh6Z4SDpUxKdca&LUGgK<2MMsgxXLHqn-dI8sP}5tCHtkfDWR92d*eKa55oo zCZuN#3XG0%D_&NI(EAZM0$E-Q2S|>RxAge?rkq(U7dDMo-g@c1e$n^dlD)W((GJCE zC`R8)F%oO#Qx800ment#l?13|sDzu(^T1&w+R(HnwwNHwOcj(yi*aG9!hoO<*x%-0oozv7Ym!rT-^SV#3E$_?~ z(^kBaWD0D~Syeg{X}6aV>#=Z=V#zGZuv%E+v=*hmAZt;h*^mi!5SxGouPGzD1fv|V zz&JhLiU*C5-Ip`vjUJbaBKYo6RhTBE4VVCWo??%X8#%8MZ<-Us5IiNQ zbP@I$US?+RS%nO=Wx$K2+2IU!M3>^VVi?(bvSb&YBEHxhd2l2=TiC@=()%)O2=k~% z!5k_VBk2rr&Qb-IStU{;yoD_fz;u>znP;$;YYwc?4&#^8Jw6oU&h;9&gNz0l-AG1T zF7{wD)0UfAF)k??V!u|>+rVO0rme~`aLy@YVfxE(zAo{rFiI~5#9}29+9_GJFidLl z=@^C63d%E~LjxZ-u$!0%?bidXK2yu%uRL^zs~R-Ej)4Y zVHAs+oMI&|u|8ztvIRz@GD@uL7kE~Bc(bJZn3QKt%4Vg_4<@l~R~lN8i2@ZC6(pzj zPCSZZ+NpFm@f!ZT*q?T`)dB<{fm|WCSK^l$5>qj7q!Q+MSBZNoqc^__kppguz1?PfP45wCG!8Vs&72gR?ZNf~?y-3lm;5t%ZhNf8 zOTnMt;4jBUnM+L#9m-8juhd63t+{F3Ofs1Po;33rA=zU{iQQ2<%Jw63jsSDGEKuf)wPX}J z{TGLcEaKHkbO$bFSiy7P^$NA$^J}&&NSJ0qAw^{7pY+JICRD=h(r!o=|rPu%$eTGC-GM z%eWb4f9avosieat=DaObUkglcyQk~ih&Lw+WZCb5PWJo`fek}kanEHKsS%bv8>!oh z_ZgR+`Ep)1x4s5unw-|-wW9tcexG@>B=|tpIjXj_8Q)b#lo6W=Ijokcj9gwVELcJS z7Yc<7ZUrxqd2j+UyA&gCz(|JOfG7(h-!9lm+;9C((^P>F4CT0mJ>I$yH(27*B`cf) zWT_yGqyfVjC{%)aWHZLG?CRIQ#nT3o3X-3(J&neFu(6`GRA3q0xXQlX1F%4L73}R<2`| z6M-iQQQ348O%7CTP_?1VWqMD_jEuyTU8#UaE^`@}hPMVTQ$5E6YsRv&Q43>H$uwb^ zalM7S?<^TS0@U3lTmd8J2Mgb4ZeW<9jG-8fA!=Z-6q%jKXeNs<((ML26MIjYhh5DY zrbz!sN6eP$%)9bK&ktE>Mpsxj8-o(>5{#H3$Log%J~DRO0DBVh3A8X2h{{!kj|B$v zNQ{6alrxhvjEzFkWi7=V!&<>c6D$iXnNTuzu#Sk@50sTm+0_bs)Qnb43-j#>Eqv<0 zk42Dqh8BLFF`V53YcQWCf64IrN@@#dCF~9jd@L6+ zhI}u$jfz(V28Lt#qrsx2IK?pNFmeW#avjevysq$3i-OFY0eqBYU2bFZy$a)AtCSA1 zar|tiXWxxF;g;Yd4K>jiYC$IeMNa`h}%V{dKn>)+X@^>3O?JEGc|D0g0+ z-PC*7aw(P8Tv~9xpLN`(-aOE8{)YWD-|`J}6sl|yBIz#l|FPu~Im?@)zOSo0asB$- z&sXN&w{!1G!7iB28D@g&WgT-bkI%I;RKoBYL{nRgmz$c_t5|#s!}m|GekG2+kzoq{ zXYZNd7g2jpTdxi*cSCEM3&H>tb!#`W$aCwr^(ta^yvIJ4pQMl4F{#IsDBh%~k9+N% zXndXemMhdZUn%s-gFvo?3vI6=qjKKsVLk-4w3>hm&K=2ls55vvP9VdM7*ssdj61^v z@si<=m=IX&xEUF4GmP!fhHmD(2_owxKRJRp!ayU{Cm}D!|3T=)86XD2+`>}`47Tuv zBCv#v1&E;)Jmdxw-7hpE#9eGK(V~CZBcsM~-?8#bMgEw0 zF!M13$lJm3G4d*&Ewymg%?{b~L2Yp})q=!~#Ki1Dk$hgVs|f^^_^y0!$*S@Lw5DC2__&8KY-oO(5|_toaF4ZKUi@ zR1yyo*f}FweC7enLV_qMX_W*6I`Be*Mh^N;ndqoWH4RYUtTk^*fm|ijkA1FXlHmwl2 zEj=F#h|$x9XEF&1h>{Kn1eGO%&N6glxr)6Z^-OuPm2%QDtu!;UXsa=sqOF%`$K?8- zw`9q#v-8w~h9gV%J7md@nD@_#dG~$uLvkqTPWk&0G0ybSU##A-Fi}E0u_fr;ZGzt2 zHt1cOkZz171tPsE&%jC0JBo(XMK|a7y=TxnSHMK7(&Q={@mzZ!^FEccMB9u`6h8E!%g<+*@SE2gvym{fl)_azhd)AFb-;l zdS30dl~=ppUtR4yAV6OXveCM=IyK8&TRJ`@duYOus~^i0;7pFSr`wl%>YQWlU-%RW zj@9J>RAKGxAvKo-Lm5@Ak!Wgy;jCG)L^du2GB`a)VXBQoVJqaNuHlhd%2N4dXfdc& zvE|8G)AqEk0AH$Qc{Q&NYjtjtS56pxPQu}}prS+DP-@g>WHoMW<%-74&!h!AXoB57 zA4ZaMJ+-CBaO4&{z^CDH3MDL6StkYcSz^KPi#CuM$Xdvtkyyb)&b*}PO-%b#CqWUw zY)4io=k`-`T6H#WO0AkZQmaNz?K5&}*RDF|YNXUQulu-^+I69JOwP4*yj%3ORS^$3 z+QJWgZRl%5U;CE&+DKBaYv`U@AOqW1<;I&qQ@x$u z84{(R?gApq0z)V)OUDPqg67tnBfhVz_Y55LR}QjS!d1FhPuIzHx=&tkoqNge zyXZYaz>cfxX#w%y2vQ=fWtP}jbtW_GV}yUl&7N7nlm0m2+!MKY9(Rq5bjWr3|EBKO zml$KXlABEAzFIXusg=ykugHR@^6kv{{17Q_1{JcXG#bpNrG?(5XAO9PKB3e%bA74bXySTm1Baq*@5_fO zhn8p)_1(tecYt6sWdo}vy66!~F%RFTGXB({OIsOTHrvz4BXrn!WjjN6lSN2V7#Y6~om6l2TmtM3jh4(`2be&XyY-D5$xOaWkYfPylL0-sTY-L!Zu zQcWR66uK(LJ~c;2hC8<*M=b)tG)!d^CFDq%28Pe@cV!xCD$Y!UY)g$}WEu?zDGn>N z=E?-xciuhoE2cxw#B(lm-qmM-1%n+6;eC}UV`k}H;1?8d&hw4mO=W@X)2Q7r=fA49 z>}mc~Am4&o1BxZPv0*sX9xPOh7sL*j_a+ zskGr$2yYp9AoZ?WU7E(3$W(=uoO=Wx~ii*AC-SaiO<^K74>m4_G z4OW$(+4n<@+YdEvsBu4AjXROoNu!zMNPi${Q}hz5cno5xT9=Zi#)h?1Q4}?_=(quP zK15Ml>RlT2p>o-YSbiUJmlOtaKMJ|YEM3T@yqTTLs;Y5wkiS*+Q2qgiyPr1kIUA)* zKNc*f)#zIFu7-JNSdxl<5RbS)#id!}k^Vqk@N>Z_86EehELGa7%_>G2i=x3ToP^(M zZiQN2vU-T~$T*QyUU{2SnY}R+=of%ATHWff?fZVY#h4DcWk~TiV%K$HcS!NilH$Fa zpC-k-WqGbMhAR?W`6Vw;mrI&W*?@D*Jn*EMZ)Ho5F>Si+U`<`p?0d}yYHd7oHuM0u zm#ix-Bxy2RZ7h7`xXXs;vQqH57$*;O~bIsKXVGGZX?`(^VxOk0;tU;W2q7C^y~InR>`@0 ztp!crS))#+*_=08Y~zeVq5-rGTUADepk^I^s%mtr4pk}SCA(iJg_IEX^3_FFQX@RY zzeeZD>p5?huC`PibyYRmszZ=C8zc5cI{GAa$VOc!OQz|~ezuLP=lakCxr!M$EAN|- z(|Z+;UTtm|;pN&e!uPD1I7|p1r{$U{4G+i1=|;8-yTgRA|7s=#-!qXzbo)V~+qw@t z&-*n4^-t8)@+sNMlQdf#b7obULYkH4rRU@@<}i@UlmBqjrF@g~AxLi2twW~na`LtE z^_<|*3`8`$EJi8I&;TY{Q1Z2@X=rkVHkpyYEFR7I*9%iSww!6zzU+SeFQ2U2C&$cO zO*#1vq|Ga?#sO(HA2$3#j20o}9%%%9sx*Qb^K}%t_(7IN3^BHmMl#D$k#}2+YwprM zSogKV)Hqy!yC=B*=G5n@{UW+<=o`M}W^GmLV9V2(Y&RQuxYW`+L(@xnE@wvMg`B%j zb8}Z&#}`_9<2ggPS0n2~%@Buvk~m~cN1>ph+I&6L#&u2PkV-QCd{61QchUyq8PKoMRQR7XGYX3!zAg^@tBTX@8;=79MEKo|jYT!IELJf3**=D3#%?;1ROTZKZcM#8pjh6VqvDvp%%ta>e9=}$wYoY>(lfp# zCq47aiH2AUX?%h@F*xx4i*aDzs~8TiN?m)yt8%9+Z&)K9+xJ7KHDb6yhj<#4rN+Bz9VDl`4Up*>Q#D9qYu zt6<3IYn_I(Q8j7(=iRLz`CxZ-Kp5T9-`(rDx^fsTfd zLW9!^4X&xs;CQ}xEH}tuq+TB*%nA)Sq*N+2@JGnWxOiTn!QEX&;`7JoAFU#>u4H2U zmh__-^qf;ITIQ{fs%RDPBNZ(zqxJH787+6-5>7wwPU?Z_=CExUjc$yY50%ng_o^mS zAMYNg0N~m<1>o8#0QV?WoisQ~>rR(S>)xYT^kF_WOW-9}G|j~eR6kdH+rH=lhwS+( z?f`Y)mS!lEQz3RhqkCwmvnp%XyaQPnKl2W>mBV#SH)eX*cm(1`F8r<3`J_TMH?0dL zX{j{(s!8&GmZYt!L{W99Sn$K<<|9+KTc{5aD>7?G+#>5Z?vL2(TzjL)`#TkRzbgQ6 z8f@|zY|_{+#E*)jAS-P!onAAxQ~NYapv<+WoDrx(5s-xi==PiVg?kYm zdk{VjYQ(DfUI#Vaqsp6}>hNSQ^|(e(sE&h%ivxVKj0;|BFjkN(lhPpHWTcNDm96rx2oYN$#cKhw)93iqPfk~-tMN^gc zeG@wjEkE1PvQkrzVrRBq)1}QaRQvDT_|Nh((#TOv`BGM*sqRfY3k8_*?`4WqW5=x0 zz_OY3tyTJ@Wr)$dosc4w?Z#rgEEek%N2yUefA+rBR`$O63ZHu4Xg}Rpr$akiOKSz; zh+O?>4P<2lGoLB6_SETqb6wLuxcnFA@@DQ^&dOu?Jx+s5;dcNY!05YtoUjyTj%zBF z&#Fq;dRI{G1H`Q(N}BYd%@DNWVKd(QakR`GMK#_@DV-+?@PDFGYB`H9`NY}w_*qrr zY@Jh(CPBBZd)mhIwC$d@HEr9rZQK2|ZQHhO+qP}*{`WpR&bc@jSr=825fv4Yt5)WE zpLfNvTgZM>B{S^~Fgs{gTA5GJTq=013PK+BL981`p--qd#N_<5NsQ^OgE=5imx0EZ z@VIKw17f@nV`?!fi!2EX$;tqU)t{Q>vRdgJ-z7>QG8I%YXdlBBgc$f8*&e+S!N%;~ zX1k;_a=2h41rXooDlcahqh>_9Se%DvJyLuhd5gJY*yQR(3xTPD2%=U(eRO}p1ed|B z;lq1YWu`;STyQ)YBK}vvFUWR=W-KhGB+V)_QizFIt?Vpy#4C|qs|3Fw3s&i(OQC%G zWUQHu0zI<)4>Pg^_>Gfd2SwoVg9Q*U3lw4n?r>;!GKy$DK{>d1lDlDBiJ*h^jH=~Y zrsQILGri6{RG@AGBMqA^IiBHZjucBt(4>+G&&iID`)kD>mi1gNm}r zpT~`tQ=KtzNpnT0I6#)_{snw{$I4xtX#mq}izYK>~Uw+)G$J~qNN|*qV1zy~(TzZnz9&9) z0}7y~6iG=DY1<=2%3y3^-ko1yeeC%6Bw1?Qt6bku*B)5ntO0eCtdwM`;H((Jb zOfoBKdE?0P_vAa+AHb^G!NoUU9NZX}49D9#&TQyf!7zrmU#Y6vf99GYFT_J4m+h!B zsK&b-A6$c@aHptyV6I|{flg0;ou5FBr{De&Rv4ztPZow{o5JxD?jkl2;Wi|7uOTME zAv%OWjmeLKe%aS*Ddwz&7UpB9%p9O0$%Q@?7C!M-TBOboi?h^(!A-r9CN0HH+@T;| zc1@OZV4&8LXx%R`O1C9OCTm`=##$ZYN<#chii{HCM7>oV(D6dDak-u`j#FXx`}ZhD z`uYf`IkJA_4$i>(b_M&R=!YKPR=qMl$Lxrkcq{r4{Lz+lBomF!oOTDppojV+d7Jic zde5O1{X6<-KzyL^5Ps{jF2kvrtB)UkqRj23XcE1JQ6Ge6+uL@&#`$i_-F^;td!MRZ zu*n&40mq57oH>dQ<<&-!e4S`yq-jMns#|L^V*KNz zjJ#wb+$d}%&!W|~@yfknL1v5e4NSD~mmb&nAt!Hr(^GGq%p85!msGRSTRb8N*na;J3$&C>g$)&tsx2PgI#e5THf|F}-rXn$7Kf89Pu z&v=2*XJQx-5oc;#k_8uZ)2B)t8^R;^)6sJp>YwDd>*>Ehkpj~Dii{oD6G)}gYf91g z+8VKjUPXx{uL`P~VP63~S2=__59Zs$f|>(?KB|8Ac+CRZq#3}}LN)WS`fp{iRDy&F5n z9Jo#_j?7=RB*CLPjHLJ6hIs66UHt8okTtuoBL{SLko>#%mHU3j^P0{TtIlF$i+*n> zsQL`)()=0o@hYR)@zAE{K+b119gv6-;LAF}z`?v_oe2pxzeQ0B-%=w;(NoE=AwigzA*vD808(2{+EM@gw%u0ddP`sIHy_<3g# z8|Rl+_Rv)B^6t+yh<$hBBK~qluwSkxT)C-zTMN6sQ*Ne+>_zi1;ZR6K@0D-?&>Vy67S`Ut%_{4@UWag2eu)(hS))DI;`!`UmT<+Sxf$$|Y3B}c53 zahlqUzy6HxVcR&pYpzwS!kH60jL{m@-`ju(UsU_nZ8mavgANxxON5)>8hr&YIKYTP zN>Vt{{oNqiT>>R#QE#`D^xt|078tF(Rgt?QGa*z3YLhI-##Xp|=E^3!i421|kNi~= zH?H-0JjjxJ761H;#GMk7qC`DCc5TS0HBAy@m9FZ36nxXqC+JKRwGe4-IPv9gR(k2? zr_}v8P!Y^zruT+0Tj>PD)2BpCoMi5TiB8~kpN5jF4fLt1FZPSE$l}d}^A6STd3j&5 zY+FETHQmUi<=Z$3p`1ghNytN1tcBi{wR-Ku)0GFR*)@&Vc>ca z_W&R?&PE*x-u6n)AN7%@$F;3b`9V{{K>Kokrso7_Fp=D!4Jkoc4skJ>-)~R)9;0)C zzb>NTJ}WUO<{1SH8oJQaW`gKWzJ5jq8u78!3GAw}#BBaBrj*UdYAkbH3Ym{`zWGGP;T`RQr9x%;*N z)z3JL)sNs>w@Gdt#0(92; za-ktC*W!3!Mb|I<6NkGJqER+C5dd(thg*IApW{%0@HzE*M1kKClP$??O$98vC`{mK za^!b!Z$c#s%=OlBiu=GyZly_$UA=!s?zc#$1p$xR5gmI!-y&pP2#ze+ttkw&mta?I zUqtN!riL8BmuQ@0vR%~tH>UiIu`Xdx87Ght0eS*vypkp8KI&}~SQd;SB*B|%HCAtM zs$9F6yZ8l-ksAlnqjNGF2hp?MC5Q8|SwH_zl^9gWMBQP0#HBT;`)o~zK;SLmL~X7Z z`MsA4aR?(s>GL;jd_eU*K1%SHN}al4$c}zUPc!k=Kq^H2taz#RnEkCvOeVo4@Lh-X zfR^qERwQhSftKvkB$QlAP&_#KSTiZcV_ze>?k_o5Q72~G4o&XRw?NKA*JJ)IQ~m%6 zEiz?Wc8N}`!D~KJ?K7@X2)UrxKdg$5OKjyW>4+ipm{;@ zJ2yBS(fl6Tq%2LbgD-MN5jrGaQW+qvapQx63!I^_LD}SyT)*gQA&clv%L^j|S}c#% zmGL3%C#$ErN^{oj%C$^#m-4PEl5?B!kJPNQ1UpB)u99`orR+~d5Y)$wbbG4bI0WW? z%lW1cd4;{jfgD>q1v5mt!q|wNz(Y?t`^UrQZYJ)Cj2UX#dlPBz#$-q6$!NmiqpTJu zOgK4J-V+4K2T7b->(EyHi=03;RNh|BoTgpuAm_{-V9wsx>n>?EZlzFea`kPdSbY3G zf+VDuqovtWnZPT&JB~5Vd_9r$@mi|5H2u2_o+CynZzorV-yQwQ2Wk0s*PDEkh8#n0 zyo!@`3@Z1Ezpf5e_H!(JUx_idmYAfYxQu^_ek?VCt+xJsnIlxSD=!fYhOg{Q`f>oQ zCNEN@5}1Ta&y8Bo>0ulf$)f_=A&!rp{iy9Y7mw(tk5k|*Mk0b7ArT#3j?UAY51BWH zd3Z|=S}uET_K7+*V>uHdl`TS0Ko4QPk=IC`Qbtp`&UL$gztu~U`TBw zcO3+l4g)S2gU*qjUm{vWjKMvTBkP|}GVH8-jx(@k)cRe}$*visol%!3U#;389d(Gg z&PoVb{Q>3(ZXa$a7KV0KfV)o=j^RKHy4+8O$?7^$vmx`}c1F!S8k*kqEjzxJ49;uU z18e3MLPSTk<+zsng&{#Eld`WS(O6)tQC7RQTm@DPk z<{4Kj>KgI6|FQ%Rmji@(CxEPa>GATWmv4b`Ha4I>70&+F6zFY=Ih-=Ks@JnJ}};|vWgu=`0I)I6WaGL_!5 z-1^jII$C}kh+v$Bmr1chZ-GGeXW%%v(CTdbTEaEydhM$cU z#|(|5_ty+$>Q?ULBpL&V4jL!2CP(hXl9u>8B8XL>}e^8r8mP}d};Yqc;W*V8}y&lL)Y#QAu(3D7_- z$g)F0J$&-zb|wiQmV0q(n-1f8LpjD4a2tqLDT4%?;Hr@1WB)=wZ@N@eq`m z{3KT7itbJ=?y(cqgsuZw*1P@#8P$8?Ev_&kV-d-JhI7X^I*T2OvQw4ViLBo&kd4x8 z`}|;vAw68Wo-1nMTXh8d1ZiGmIN6)1{cuLIjt|N4d0?>5slEQSE)_liTWJrqk#v%cSjZ0)~ATc(6iI$~O z7IoF(N382KF15LYT_U|W_H`2b(UQstF1{A@{!>48)_G+{Vl5a z!&zuC!^9Zce2^*_i>4JrFl`7>Y|ebm%J0hpla>$o{i9X$ZWN!7!2)d19RLVeVO#Y^ z-~_DE?fX+VYM7M={zae^A3>A7OXjuifp2fRmwTB%U~}bG6aYDxI7`15?TY&geYva+ zpqKD@V2SWzc|`qsrA}9m=YU!iw0s3mgL$N4B{i>C052%8Y^&7rH1zBXfqQeS<#RYL z9q*@S#dbhAJb#|W!E;|Vui6dcqiRfb=m`gxA1#y^Py!12h+Re}3S$pt; zu5toEM!zwNL+F%!P0Y3H4ufyh{u-9HjNUPsnL7sMsYb`Tu=-|PNw?E-V25Ss$G;EZ zGbUN6l3H9I{fKp&IPhX$49btgQY{O$EgfJ;AwjKfZMt9b4>UG6ij#7vtoOfLb|vEkld-cfE-Y!`+x#K{}n@-Y%u`& z^eqF!L+KzQm3#k!$B6Qu@&&p~bwiBZ!<4QVT%CgqvDO`?)ANBttpLg*`@%e_8K9p` z>%sI=T}1CTXXu?yUx?t73`n^mDuUyYnBOd3);6(ZbJQ@KT@&}gQU!-VKmGz94@6hX z(8~bg6axqudOGaIPH{qHtsR3mNgB`}o?NFMYk~YP5Q)Av12QLO8juzg5gTZ~(PuBq z3JqaIe(OF%{+7zr@FRb=YyqYZQxw)T9XAID24#-lCAz@6!~F#AkGy|D4M2~-s}NhI zVY-ikf#YwNn-`exfx0S9K<|kt4WSJJw?*=K5DB^I2%Tn%rd2~AkJNnhO|=-YEk$!+ z57=HwfbuNGXDS23kGh?L^9uD93)Ij=ocH+uB9Uo!JI`1W#{UP3-&jhI^a8ouxpvt{kd4`ZtLt z|30bVxT}}DW=Z9hRBC#HQi0IJEpY_ZWs8AE(SDMEe}hRz9RP4{VhQw8FkGEM0C?ZL zzWH6}wWYVxQQ%7+nkpnERr*C!tI6EiP#SDng5EIV-7sRkZ+7F* zw&8##IDeI`Q@+{9!$}Y^5;6}diY4S!A~63oIM1*t!b*@k(zb#AuaSHcjFtCr5X)Yn zFxwiM$aT_7GDQEZCFFDBU+rtyamG)Vex=33va?9a_gex2YeOfW(~7HB7w?l8kJX?K zPbrB&bNP8xoSUT5s6n6tPPIwawT~A5ZXRYBrnIVp6+P##BV!ZfZNJJWwj)}^VN1(p z%`ewgy+Z+<*GmM!g^*v=+|2G#tLUK$V_rg?*rgSa>4%t7HfS+F_PtP~G=e|poSEeA zDp7;<>8PRTd%RuM9n@$%O==XhL}U6XV>sIP(i@~%GUGNAQfN98I3@xa^n*h%xVY8xa2X8eSS<<7ZPlo zO-XeD`(?{k>CMvaE9neI9AroL!Z~7A`6}y?>Qc)3I3jrv^p+ZcV4kV;DJzLgO4da;cf9=B8S5^8=lZdPY>L5ZO-?}Z&>#=VuU<#t_Go)ug<%5lD&+Mx z1>(eIc6?kb19Cru@IZWh3Y{IC7`Pn7Ua)=}{Y_OGk(^|vp2Y7xA9 zz33dJi_TQEc?i?x`Eh8au+7GW%_0FI1);v&#!A< ziH)iM<@O=mYx&c>=e)|`Ukd4i%+cydHIF1=-Kq6;gYF-XKN{}pL7{-G z3Kxvz=jb9wW`R+Ar>K>g)D@;JL2aAj=c&d_gK+RvBID1R+OywZMXg<(*)Eyqu0~b! z0GAfF#a_4ehiRu%=<`~ZsI?J^8mo@Tjw$KuOpEL3nH-0}p-BfX51wm|<bV}{ zy5qscS!xAO-8u)Alw`MBC+(S$d;fLotc(9lx)`*ci8&+AwYxXm-kSE1c5boXiA}&6~v^BQTl_fTP(nMEEjly;21S##yvsFW>b0fOCdIY>zm4m^Z zyhkbM1{}1S=|L0pTn|XGdvLC72{GmBG=3im(%iQRv$YD0=iI6CZO z=*es8uE1P*1ck)??7{P?wat8tgxhP{{|&%0Zqe(C$1*1N^h^(omOh>i88+2^w!1~o z6k$&Yy6fA)e>IL2+qvbM^e4}C)JSi!(8#sY^`lw*j4cx>&nV%2I#@895&T$`?VXg6 z8qp@G3b(NCCXJapEh?xI=0KITx8zZhw%hj`%JjTCb|aP!NbMK3+4@GTPQOeW|2ewg zwu`3pbI$qxg)>{R{jt!~W|tKMxQ=yiQRV*NB$!9*WuHQB0~xMD9u~mEdwU^zb3NW= zA)>PP8B(>L?c}KCvL6?=UY%?;%7j7_7<`MPSQ>K%gf;r4g&iirKk)1U(h0-4d8l2? z;n>_Cs)5upLjSF@U}A@l}AY(zY!Yg$A+%n#)jUyUNJ;A z^x%CKkX&-Bzb^Xu$j>h@?=v>93q(1R6AJ4D8~xcD#kX0817*MTX298byc9FEGFvsy zax~s}7%hCU9P+3xC^WS>Pw$GDDzTUx^`Dxk{jr%A-hx`q)MZnbGWQ|? z2jHywG*>jXQ`?jtkFoAlMfqn<6bbjO&h$1CN{k;A8X})_+U?znjn|w8UMo`W7o-wo zO~Fo%sv>w4q$_GM-jD+C*`%;e#wqZPhYky<`|d?1As%RU?8Q-M7ZEOs@w?&R0{>6l zX~ROS!5?aHcUK9!%&=d{npxb=3{%)@p|upkf;FK9;~^JU_+Sx2w;^8m$^f_|Ie!OLV8`?*{KVkKYy*`F(ts^zK3F4baL(wKJuXeafcW9uRy3GBnGy z4E(M|xG8Q93pko2h$@A~b)bzjBA5wEIR=~sa$=GHuu}?(%xni4AY;KT>7EpeR4Kz{ zhh|@p0CX@VegqvpxqkAj){di|NTjS4P~jm->yWxsFY#Q_;(p zXCxP(DkO=(@*7m|eIXl-!*Y%1vBb{cxG-2R!Zaz$H0kl&J%+1kjkaa6imKIwM#OvQ z-=T@%vfo!QSG>-!e9mE?Fa4&=<(zdNrfjJcc$O)6I;ACVG_Y>}a1@bg*zpF1*m^0; z4{5;{#TE!w#xN+OY9cQPYTR?xu?i_Fe5F>*j6vz zI@A^m3x}N?I%cr8-{N^#IC!dL6punAU zESQ^>W`^VRC3NmtW;36EIf|>P#HLedwM)&%@Usf1ZD-|V=lwmW#YP-}x$3oN z1`U`&G1I2xpRU#!2IT`p=K$q3sVJ00VzTZOjJU zvn>bIg*;639W&2%hwu-k=XrK>rz5-1Em?;3U^GrUtF<_uO9cy8ITesa-QYV^&^^7X z|87w(ZKG~3sH{=8nsUg3vK!sX60v3RSeY7?B-OkhGx@o5suN%Qj9P>)H+qenxS>{c zhg8++y{)iGyYD&@;T!Gb3rn`#Y-j)4#g3bFWMgG^m9P>#!D`qwdSK)+g&UKv5pTv*46nDo~Ar<7*VyX8W zg^kCo-6fdJK_c34!Gz|mYZbZ4aiBayomE+2ynG~iHJ4|ok)Eb%xftJNB|P7R{me2* z@SLN_rkq;6?s#jbeKftA?zL?QPye$RAe1rAHO>3!KDhy>HJ(lX>~I>>v)bIEBjVop zvc0oTZDBSJQBPojIYRSkVim9^Z{<^*yXq<9)*cHHxJ_!jub_dv$=Mh#`xV%?ZBf1O zOlC(yFfruRMzJvDh=|M$1>roNg5OF-gclI%+#@4Os^gFvt6m-ybA6QixjOk$UGhdx zjBgsK2C>=B730R`2Q;ZOKKc@55VL+CKR&wc)}OcGl_jj%V;%{ETgIIl-7N??E+~?~jqp zksaa!(zYDmQr{x_R!>dF!ue4u*JC%0rLPp#k?}nKy8qa`u~+UFh4o9AYj9Tk>T5w= zQlMqF&QQS%n>;wV={5&FJbDsz^#p$+3CH6y60+uQ ze9`mh2UQ|cZF(_snl}y}hJ&s(n`kvj>em1FjW~|ix=E_@+#VF_17O}$K@!kz0VS}K zGx#gJ^;LQ`8A<=m93bS_nCppt4n`w&Vie2RyxejOqW2lbQMX?4?Nt?)d*Ll)|FU9uLGlv=y;iiI`ZrnOM(`f5{)AA%oaKp+G-#m*}bp_kKj!m1j?L0<^1xVdWes7nI-31xNO|3Nt zP{LN-q!i7GxJUJ{pgBvBBk~${hA&VK4&L6tHH^+{y|7Gox=dyJFS;{iI>XiCh7-)nMnAiWb z;T`tNmN7~JFA5K8U$xuF?I({xy=O#qu(_KEaev7ufmmkeAp$>%nD^0SF4x;I``OTl z96MG8>1|*PWd=jr{RBiAr6mr5h~cD$>(#gk3MyxOj|COa?6!rE71GNN~^b`XdK1bZYOK$7ejmi_cKt<^8BH`?M6B8yj$a zK*y9#^lkEsd_sEIt!Z>Ud+BoBedHi|ZFQo~r+%)*J&KWy68ud81!C9}==X@p(Xrq<$vSpLSQ`3TXvJ-l0{b6~5)UPRBWaDrzN z{6WuR#pti@1OI+Cq2bEeW#_G4yLU_+!Wo10jO-?=^Rx-M<{>y_} zOJoqLVm=c24z+4K^b(2hi5tlT7IjM(zr1vVlB|~Ezlw4+@gAv1VocOOaP_+0S2b>0RxhHu`X<-%w0Yq7NPq*5IAa-BeA@;C`p~p4`uW~j>UeH!cf2rn zI{bqVUZm&gYsauw{H+_APON-=0A;W-zs*7c1R|7As`w2mYg=UErd8#|8mL&ZSC67P zS7Y7yrMb~?R{HjHma_9AU{rfC6E*!XlXW6T1I5+eJmSw1ONBj-hqJhDZrobUgysKo zd`o|0(|MC0C!1PRbJzboBi*&m%HYX9aci~TB9!R!*j^oRf9f7!Fjv3#aOO*2S{dT) zt&MPBaLwh%UwL;JaZEa(>8pdB!5BOyUdHEzxBsFDisj*3&YE2L_xp)h&67=7!HTAg z`mPl&dV$dff?Q>>i(!nKb@pIrYK%tB2q zq#~zU#qEjE>3`+(2{Dkjs296kzZINi{`Si|{0$;vHXA{0TYy0=$df>Aw@* zJTNp>_awmYO}4)I#~of9SMmsk0mH!K=x()dFGWV!+*-ffSfcb^n8*%;KE&7Fz zv^!d6(Gvjv1;%_M?W8YSl1-TFj0?<|&}l;Mk$DzqO+#C4MYCIx+f2O*rP5RqeLrt2 zx?rsBY#ySwB4IfrTt>(<2SX?Amj!luB2s?523w^Qa|g$f_}@J@CLqnCs+*lO+f>jp z&k0$qH|YDf!YXIDDvP4zU-->PnW+IZm7rN^!du@flB(T61o5;e6QX$)SL2%$N2`%D z$Kwqr?dx({YG}_G2HC;75~rMYWJOPvO@oSoKdhWNos`wrH-rBCG^L-+;vxFES?Z@F-*7#2pYW>)bX)aQ8W)F-v3`Y}KDv z$>#b&UpU=7GDNpMXNt2$c^LEIZ4a4!d^M}>LM{;)z@vWdj}bIc#(gLRc@t(@ zw7>!aaB*!htF`L7&ZE91^Q}|H#3HD0B*2U=y5hUnAm3zXelAJUuS$nDDi64b}9F zuCxvkGt;|NU|oF;=oZ2umO=r)Z(%*mFl0fJo6;=8?M+{8A(dPP&HGh zInAsJF2Bjsil2Y~*fzSP+>nN6E=i|iun+;+{MlT!0CYPF>R)g*yfaj!bD@#hib?1{ z=DYt%8IhwaY~Gd+(J<4GEYBFBEnF;jg>u7|E=R1x;jjO1;`nDgx&aZD_jzPvO_(v^ z1wjs?Ue5{v=5)P)H`p9*94*P{-AAYIrw zN9a%+c1ofATM=NabiRC*dymUX+fGeA{hSKx(_yk2*(aovKKM8_ecw07uOqxHlUSUO zp=aj9iMKdL?@pKsNW7L%e{DOu8y<(J#Cy9aVFHCOC|=xsEA?2YdG+*4pGJi2YUJJ7 z9!+8*_NT>1qRNavUG-co9K%^84-1|1IbD_Mz&smHtoXEqSoDRlrc9MX9kg<&9%8g8 zwqgog?XHut{C-#YzHw>yseS)X?>?Wm`!|zkR-gN0pOkzbX6KvZ z?(gAK9?9;T?Qi<{_0RinRiDrE(nUVc9W$k;eEcI$kCx6cx+^rz!yKRO)daaj#>m%d z&2KHe_9{Jz4EzyX2pqde^)VOQB!YqOX)S4;=)rn>-cWA6UfoU)Lya zHtkBJ(B7Z_-W$H8;#WW0ySG0Lh*UE5S>I6F8EZWP7D%vOxHJ}BdGy2zd2dHP&?;j6} z?KxbkSoDW{s+u@R1!G77FzbK2DmY! zSLo~YT9Zv^kWqq?yJ_X-F_vl#CT=&|DgiTLCyfMayjZT{-;$zQ-a)yt-a^wKr&rj@HKPGqeQuQ7t4dw%^&iVC4bNQZ&L$W&dZ!;tNmH+xbN53-4`3`s zTMH-GCa0XCYbO#b?!*nYNm1EmQW#E8-rkv=y@uWR@t2Y*U$01{nabHBEjQu^yQ!M4 zhQ0B{-qxs|Hq3=c;GgVzGnv7B(I0LTyU}CW*O8s*bC=Jt*84=a%-z`ejCZFfx>@*) ze)xNTy~az*V9n0H5~COJ>)gQcFfs_zFr6M`SD@WWEnE%7QD~w}^kX*ljHcSkOUILF zA155=9>jJGtS@ZHBD#pJcGkYPx0W9`qOdHk?$vuUMWN9r2pN0ve;MBzC!wLK zy#S|z4u|9O#(h20s>#UZd0tp&$62kOjo}xEqYB;Tm#OMd)p295`*vY}gAc#ht9S~#x*l4ZuT1&x1b zmNuiYngh}VWV7cH7UamRCg{2y%FhF`Q6bqGYgL6e<)V){Ni|yfAIg7aQ36(b1T7~S z{e%P@GcY$9y+r)y!ucFa-Z`+7+PRb`0ei-Bkeaq?CvMT-AQpxW zRtuy=6l40Y-&@)nh3+3FY7!>nhC$xT46GyX8A`#xUy5@&fDjTrUmw&A?hVcI9vFUZ z%)Ut_rleH3-KCdC=ZGbrI;2gsXrU3*_Jk&hc`C*t71iBv%!JMuH$5K$n9vG2;_?2l|JBNV zBqR$;>s2NqGgufIwV@R?iK%RRV{bQka5(Z(4GMPn+TQKWjZc-GEaUIaTFd}{xT#Om zKtYPwx~b2cFtu8f}=Kw)N`iM z!Fl3VUPJZR)1|SnN-7&Gt=rwZ26Owe(asMz@IFf%jm9a^)CO-%JVeaSxX}~@x~xb9 z-!7)Obu&+(poN(k(qy+tgc5`LDxAw?lWiu+Ypo1kvtw^NU?vD}XLXZCwqLo%O;g5< zDdhZ;o0eI&_4GrWOUd%qfT!}YVRPxWNcK}@6-078S72`Zj%r?md}bDRIm-q*q@gEj zl|EY*spC3f4z$xpgiXuFGdZsZIZ6&Hj(gH>CaI|zw@WuD zbzga*lM`+o2j6wUW)v8a8SDUwN2etFtf$#EQ~o*ZSz6qAtnev9;Q>WMv;5n+4H3z5 z7_z3)9;ii_UhIa{1&Ew%gf0}@ASOJ6k=+%|S}!8$W2gzZTRkEoJT@-r9B8JWaVX|W z9vEE42{<;~BS2p){$5FBcPhBJN{Xj0u*xec8V-?TQ*>XSzR?V(F zS^e)zj&WaF@Qg&2#a6bM8GCP$P$efq7_s@;>nm;TFjYI&9vxqf=!?%*`zsL|>tB9ny51UJ5%_WVFoOowS~8G2$wb6ObX)8dA#YWJ<>1 z>jpJ{C(hsMSCF)qe%0H{M!6GKKM9&wM6|gxDFZOw;9-jCl!c??yC3ZCh&3KbkO(lU z`_uI2lq<>lZvHZ7iAIn~FmzZWJ_oWJE(&dXUU}2F>lLL`5?Wm68FDB-$}Sqjo*@~z zHb+r2rQYyo;=f2%~ z=$)7Bx{+LNapy5`$D2N|;GiAxfQpX`@WAo=K4|~TGt(p{pA6B+mkgXt#y~87f0L?Q zQr_i+NvARNtkB!oL`9>CLO_VIML6xbHmp?H?oQgN=vB#hiaAr2Hv|LLP!3_{f`t&b zFjkRuK=wckW5cU5{vx3<1hHC1i|{o;*`b~{jr;^6KC2jiO_!MAW!AS)s`1=8Cu;Qw ztkEfpuFgAZq-drjzN9d{&hyyu8Uxz7bJ5yNulLiN>q9(yD4s-w3Pz7{!4Ij7i~}G2 z`KPs`L{;%Yk$+HQ(osx;;O`4$RBImfaGprP*QJLG4wJ+oVjI!;1nQzAqQ0$<6;HuF ziyfw-+}Djyeat(LIFHQ)YHk7zCgr*$a~%(}clE5%VRu_89ZYS7zbe;bWycPVCf*}c zmNp0P@|9jbS&_60Yn@rXtp(=e6H=HJeA7&-0+?tN1OKE3azfFoqzC9AV;eq_rlSfKx;Tx5OI&BujD%fg-XXwAJk0(csXsweoWBkZky}Abo3bWSrXmLnA9;^BV_* z#3!OoaXps~#s_b*y79Z6Qll*Ekdl@;W@>}5VH^y@UwIsqvl7*We=-1<^9ukUsdH)L z0Y)W>SW}9GU^*vqc^J#SSUM4LD8u)KSl=RuwR+%eJQFKpd()qFU@4Q78@m)(;(d6C zq&-&@tl;H?zv*5J19DdP$PZg#3ol4G`ye)vOxJNDaxw;W%RRpq z1jq*GR187sNQ8Kq&$2S3U@+!<86d6h><`n*dqMOomUaHSD9X~ty6s4&Ghogxc{L1u z6-@Cr#Ys+{*Q8&MRV@hhVeye<#FUdv%KEbYiL&$CVoe+xkf{tnpdhlCN#VnP1uUpMKs;H0biG)UPDyA_ziMd0m64$qPWp6PTy zMb9FO3+YQZLmLS3l+WRL<~9`nt-rnvI=4@lO0_Dm_)6kk2qZ8D{7!hDBit*rbU>Cf zY-=!5us9a^x}+K1L83{?860wn(8^#b#OHi{;<>7K>IX_kDkg=X8HK=lSyp~;DbqEc zu!XOMVLP&Tx$e50hJ+>kRouU6?#`x258FZvb}wvR_}_;ucttMb-cU?(mA8|S;@4>}C!!bInOnjW%{6iGjN z#8OIO_JDg7oeW~!w^qs^J+c?G!+cfXxhz*=39FaSr#a-xI(*dgIZI6|mB^oh#QG-8 zM1t#Acm~V7#f0OPs#~WY6~8<;--tnH#ijj9Y}HtXE%FcDLVN70VeSv#LR;i&UX{NN zpL@?^fAXty=TWN`GlnGV4C}) zGXe#YHPup=?RE)3o;VO<0-;6l*O_G;)QV${G+K{ zcKF(>h@G&j_uIzUZIWiusD7DdJv9}tXP;fBmD9Aw{uwPH^2+=3xYiv&a2kcGY9bwe#wt}^VO^|=pyJH_lT z_*ge5bgLn{ouKeiq_i0?W_|_KT#P{1X198agv`BJ-V|nFJ@8YwGxMc(_o#d0rrZ9vz4pqrmK3s9b}X7vF4ozF5Yj;_(3|8M!f#O&{z*$4w&;%~tJAaU%IChVI^Sne zSI@H#^BpHSUsmIIhTTXunUoewsFpmWxAupYh~8pgs1!LXYwPkkFWII1&Tdey_T`b~ zwCw@3>l3#!%GJzD)n>s``4UWgcZ1XsJa`I_x40WMYo8w4sKWBd`1B`W;Y0aV6;Ee$&Tf5G_S)k-u)BLCi;)fH~Vv|(UVM< zP#P3fqau1JzsspawEyme=;T(X!n7Mi)Cg$*J&%xAoqX2*3*x`*JoHLQf=AFdI}EOl zkd9!IB&*hnYN}^SR&4_lY2#mqJ9CORk@-iwO&xoe-e$r=vcH_4WYUv0+?)DXt(1F9 ziS2S-w9@gP$T+<#8i|1>#1{|-1FF6l{7V_|GB5&=)+v&tu2#flA)mP)i#2AM2{k`V zT0~TGv#V*uSR=^@_5usX9{qXHfroYp6;uLKxku6YQa%=fD2Z4Bvs8ej5LWVo9hC}iH%jPafljMDszv$lnHB0 zZ9fWw&5JuFld_dr-;8=KY#nmuq1f?iUbmTc;}%@4VNw89R10c|#_KtTsCfY%SG>7m zJ^E{%lmMl{xSUx>uR0ji%Hz|zh>a(@V`~u2nHRa=L)Hv8RjA8M5a47^_CxDNo{0RS z7w0P)u4`{_`@Y(Mkv#FOr{A7EogHy3r_Ye*-oeOUyrrTc{y*+6j0`8yl-d};vsLE8 z>UsbX?gmx}GG7wrQ>$eNdaKy!h?gPi?da(OTW0du5{ho2hspH*Pp{mQ*FKd~4hq&I zvN@$bVN5JWY@quFe`c51OH?!Um`iffkeN#=*uS19nGzB)mJ6MkJhZYS;B1;q6==3CMOBK|qxE;-cqhZyRw{n;x~m@+;gdu2+&kqZ0N?;noYLeWFW zYeLG${EH0R>$;#9M;~u{dtIDdEo+ieSmrLvA`C5^0MlJy28kPFy}4NDL%gQ90+AdF z=M2A7fdN(phh*$^*DG2zM#5s;Y>is>PMo(lJG({UJSCwepFCX~iTF`z^Bym~qc&z# z(bMv_iKAG~IkrzdJ0M|Ip{od|IOqXwU|#)QGPP85ZS8DJ$AM2fz^YX5 zExg-Wj%QrpA%RUyRo0?(Y2zpvoP^`1c^F5{4`k^jNYNw@S&diGL&gKW$3ZK!EkNFV3swroX@a* z;1n0GOM|&L*1;`!c+3;ARiwhKch_&;IdD=+6!m5=upIaAqwKLMxBK3J*SY9y$(aq+ z*h{75^`hl5Ol7+fS)7fgj_x)DfD;;E6Kuiw0;i@&J-X_|4c2>Y5 z4h7D~Sw-Ud&@cMGXwLa8)?EElVX3WG_0#R}ErGg#fy}9Qa=Ac)&y{vwiD-n*Wkg~Y3?KKEWeFPwUy-4Vjd{illEL~z z$mGo98OHk%TfT5h5GS;Bpxph604rC_v%B#SuHda6gZbr8hh@nxbX$5d)YN|+kO zr2`0MIQ#y8RZ#wM@TUVv4O<8D;z48*kFsKqfp&kH`CeE6+SL~&;gU-C_NYf2=G9rH zsh;P}hXxT4#=ml6KO&D@fh_tBEZicgJB3qce~TB!bX=}A*;}s51D1tudsb9sG6UBF4WpBGs|ijlm#`5hVZtT)u7^=1i|K@O7OUD@ zBUVW~)vM+(%}Eb?t_5{Zw0o{qQ1~qTjx|otz5_1|ZJ}iQbT;9QlxOABc2HUT>AUt1 zA|6zX^B3ajigPD2WagcA-bx(_X0H4!8|g3q<+vG+HAwu_uA@@Z@qbBQD^T+_6c1?;ipRFfUq;!ZCDx;)mY~oS zjVZ&WP>dU@#tk0YI7uNAtESIj8fyItc3;UQyi|8B@;8jZ)J0Q9>Ys+Fnj%t*S;c=R zX60rbjkWvs^PS!DU$DUv`}LZ|es_##ok&Xwrs4RwOdN{I{8t?t#AEm&e8#QZW%UZQhh8l7;FiVS4(laP)ibhR$=9BH`58J0jbt$-yqB$<0WFV!BBzwi_< zCsSrq-qT7Xp%a^xacvm6G?A}o%h;NVM>@1g=8h77o7&bthvh<(`G5m|uN&yy6C5UFX4^JzGH_N!031Ft$p- zqx49Zm+B)ur+dVibh8{*zQ#jmpRT`OW^K`B<;{d$Dbwp*?vLy0n|r2^s>K(0r^|t^ zn|!DjzL>6+th$n@!Ai%CKJ`|Q&0$sLw-#Nh8lSWN}R~$O(+4UVp;HGmv*m za9~3^yUQO+D}f5z1qXK??>jCGA?a4;furXMi-P8B6@=1=64M2PU5b>}N68Ne7rC~k zX&4d=@QlP`mI>^en;7=on_Oz*>b{se@w7-SSB+;+52~J%^a%RoIP}vYB!sYic)on+@9|MI zL(=1pHvxlctCoN*I0v#h=b!0WIhl6lQYME#nVcz*nuSGmOp~9?ioRs6U{YUJdzi`^ zdg*qYdU|DI3Eg~0oZOoyZ|r{y<#yG&Pcayt-tvHbK{{Vll1cNK)7|t2TbYdW7|s@~ z*7hsyQiU~J|MWr7hkt2@gnt#@<6v6%qluT{qpu5;r4fX_!1+BWeQ-!lFF?<-zPbju zo0Q7ZT#o;v)Dk%!A72n?50LaC{Y7U<|CxqABczjXYG9qd<+bWjZ+mVOIY{A}$=~{S z|M~C5vFQs?u0_dF!j3KYw&dt(QIMZNTF`qSIFPR~|16l6k}m(bQFVK7!XuESN>;zE ztx%!&2uK^Z7{R`-i&rY?KicwPDm`3-b3a8UK#&v;fNDQEYw!~oqrpQT13fjVbHf1J zhI^^j!tR=p;`)Jk^G#w$5>=iq&6H;^a2U3YW1I$&UO?erxbYd=Jf9&J`X&?nS>-SN zoS5m*#j5!3U^^2y9QclM|LL6KeJP}y_tBa-zR{cJMI12j%h2rbDL$@RHb3iksQb*Mzt5Phh^T%#+1y+AW-iD3MtfAl0YTdj;i+{ zMH3BF-IAlWVpX8U1OeGxC!MF0l%O^tI!a=KkU>P6N)xf^HD$Zvb=e+C8l2_~W9dDB zt7FJAI!6mu09+ldPecY8ipeENqJpgm)bIpIq9B2k3>(r4!J||_kc56>Hg3N)S^`dl zGwtsUiCqIMNuYsM&X1@4h*VQVir^^(pj>u^1!_an34~7ZR{9T_o(|nN4K5=N284Hh zND-vZw0OJX-Y>$+1?n>8h3Ee`IFFI9cc#TP0f*Jy_z7}y{uL9y=-t;f1<93 zOc)BzkOp&yg^6mYSs4rC<^6?Zz@>T4n&Uq4K}56)i28kWG-OkhNM zy(wc7ytD5JesJbjpacG*W^l%yLK=5!E}@TNGxg0It_B>C^2BrmV5A}5h7PR2OPju? zqEFfenF3^DE+Z)7kPJ~W#u1o7K8mN#oXxG8&qkju%kkdEz;G3lf}xX=(bop}AwJW} z;m&b^#goiZmQI1LHn#F@rjkevnC`C<4+dV5oi`NE{j26Ya3)e$61>XBUz9Opn4Ju^ z@);l?^pj%WC@CB0Mbi8JpvrJQ6FEx{K&lvG+V9%c7ox3V&kAB5? zpZ4&uW{Dgjt)F1X!ckF%I+rJErZt!uWvcGnVP2~;pT-6PbMiy2#v6Ojj81x2L%OzSlT)1%4{ejq z)TT;Jouf`}ZKx{C$f?^YNgFu^Mk~f44&cFzhU~b!ObJX}!dQVp@ z{y*i7qB5#vtB>^dd{2v=KwNsSW1x_8V{a3>i=<-%*D5NEidp~U^w3(dx?O&<% z=mKq49VWDo*o?>-mX1}e%X+=M#I|_PwxM2`)@ddc-Q}pYU{S4>S$kb{HwiACFgi2Cc2Z|#-_u7HUc$9 z9ztL1{~Xok_RZv)c3i~;qT#!(Z)4u(rrO_g^ky0|0vt@v&y)M|$G3RhZ;WR$8MpK0 zE%S{%X>RfuJmka`$4Y9_7-Pn$PI%~mDgHR=XKe6VVYr}c?4h;=?t>7{gBXp zuQ+Z9H!u;5o~Zfm&d2xx&lHVu17e{Cl!@QBtVLi>8rAM;nKO-VlLps7<(g$CNM)iI za#r0aY`>WFj7U1v83RZ@voW3uMt=~K$-Vwkraiu(2fRNbXuLBk3T#0<)YLLF<^{df z?I~vHZ1i!Mfd`97gCYhVNfchyP;J6+B)Jwd3s>Z%^LjwVIvxtrWT{qp3P>-ASfd6_ z9`U+U0??hDWQZ#WZ5{w7W*007aw8{=L>{xs48J)!{1K>E;ioi`Tj?~nOsW*)Rj4_F z+wYnbGMDjTzcK4yr?;dB0?<7h&Z{8f@{bCZbq&Q-tHzt0mKxP^zZBY7Ml{-3N9`2h z2TcBl>0%a7iIp#@7mn8KYa)JvJr49ua3fJxe;g>oq&wkvuzcCI96@^(;O;l_UD_1Z z&*cMw?0cx*`xL6jRtWM%NVg2#%HW+^%`^wHiMrmrUH~lIwN>vnm9XrEC*kIxnZbGt zq)lgXxgyx>0xfzabP2|8n?k2xcQA^<6wUBgS3%|RlsOcO;K?Afd4#JI-}nOix#esy z{;v8X3vf7cSf^m9zHUM7ZoQHL&1$V&8SAh9Ou&eH= zl`uw%;1&_-RSWCxwm=lCK%h+|2?31C*v*t`ffoh`8ebOSy}<;up?)p)A7JH;NDCbO z-l<#`w?I_e8jbIS<}s?nI1dlu&`MffQb~OMw`~-Cw{_~FZ33_iN+EC6_AC=8n-;5q zGSj=(__tYTto0F&h>D+fL(MK5x>7#q>oJ;PsP;#afx%@M23^W>3u20TL!C}CQ2ymbCgerPL z&6e`D1zzakd!qn{N=elfedG5d4`|*O6YzpY#zijX0q`Yv>sYz^tD^bGKA8# zjUx+6AfU4g9?>X+N&+fkkZZR()zD!?GVkUhJ(%DImpIY6&h*xC8Lsj~Kpyo{4MJ~M zqB@g@hG&5VvS$*_^s|oT{j=pI@veli%JHsHQGd68Lx~TrO(W}n(7gBA8gJj51=@?Y zuQHmsUvaBjtnsNgZWY;yozMPAKbq%|HT0otvH1}OmGz_HUduhTT!KI4yr#d`M%(V| zm@1D*q<1KskKJ~6Kw}E6MD26Gd@=m^w;Mr>_4nhZJSmnWK0w{8+UEhz&xB~2*moWF zV`RXlSOA}fm9>K!%vTw+nK($hi~dJ6_l&VwI|Dl?#b=W$h=8% zrBqM+0A4uhIRQFt|I_R65Fku7$S~x<<)XNUuAVHvg}Rq>UJmj_8!GL3GZY^CssU)4 zurcS~>+X~+V|m6S!^uScW*(@-xF{QMjf9y}1_v1~6=S2o@zHQef#o$4ofZ9^mx^&@ zixh8xj!HId8KgD4OhPR9sao>-JX`ubs5ltrQo{)TF66~KXH9wp!zK{LP8ptsb^>WXB(;*Z#_NGAGq$6c)BoR zQ|K1=9==fC&Oo6U9E}^+UO9|1=J=m^LIubQsE6ZT`X0#GEvngDFhYaIb^F96MEr*m)RW~m9q1)h4O_*A?JBc=uvUW|&=Hf2|%BjnCJlt|>%ocxB+;o1<+Yb|Y@XzQ8iGZLwuNSE$&+!dYDKfe^( zMdTj`5XBpJz*K>cjZ_TW%L}%|L=)a9m7m>3)DfKffjuPC?GtKIsd^oytrgfqXyMRZM z$@kNsLzGeMqYXW>*hss0Gz~@&LW{R{FmOBj#lP^6ED~<^ zQ~a$~eCc(UZ0(E0kjnq+jSQ4{!X?4`zWRq4w@#=!`=(gKvuaJ-BKFQsk5lIk_pNyv zG36H(&u7*8YnU2$9C;}7@McM{xI{QYF!DX{Pt=i$v(ThdC)X)`MziGcZLp+{{1sc0 z$?IZ3#iDXR#W#rbS0rrB#Cqw^^UTVMN$0wv1|6W z&`Fpe3xA!Rxu)Kh(n^_)`xd<{12lwL^-yqrtCJ4R<7+DuYi-8H=o@%xJdgI}*9h-? zq_iDbTU6XxMaspGp#LZ&NL!SdMmXC2hKl1bHRdk|6Fr`&GOuA=*l1RjgxQ7{!Shj{ zUPvzJ5a^o?lw&X{KtJ~dPMr2_zYwkmpwdpWtkHZaYhpnx3SPLjU-Jh9uIH$5{&Koo z?|ZSg$$i}#mj`3lfZZ1D5qBi*LoMr#mJ%wdCu zpB7$0RZJrLdhq$wJxH}V)bN0}ZLhON#aPl9i%#cC^%++{fE9U;#jzRh0viu5Kf=~P z@V1Su_O;u34N7P6ug9r*?d1+C)5GK!;5ick5B)ovQyo%S|!h@5$h8^Zyp#s zB%0#ZpB8%{Xq^*ZZPutH3lS_QV{}Hb>c6}YWDwEpv1#l#oTdZfelbJ~+%RMW{_!F# z!fK0l@f=JS>wP}%_tFI!-B45%SDSQ|z{T-t_WDh=@zhgdHzV$?;(RVHk{~gr$ZYpI z52ulmAz!uS6oP99-$$zjk6ha!RsRG?ydln#qg*QDt0UGbbiOO<{C+<@G$`_?;j(BQ z>qkkeM}eNgVhn3gVp+y@(UOLq^-Hj|OS-sXhPmqV1sqv09D>fov_r-UaL9~2DPB!t z+P41)HTJ#qXGdN(NybJhAYoW^CwKm&(>&`Y`kDAj{w1b}Jm+;r+vBTSfFO;X zoaHhrBctHT@JZ^nL=6%N=H+W=5K}Vqm1!BgMI@cv3T>0oty1wgW^P_ex3{keXl)nS z&qg)!eV8bf@_mB>%#^Q0rFHtAp9cj2jLl>%l!7W{hRU9u=PWa&8)3HItL-*K+@!4r z^^^Eq($zt4K|vIYS57G%OSCIb0q8Bl08l19<$i^#Z{3f|-I3Q-3Zk7H`$aKs7~AG< zG7oJZ!6%R7D2Y^+y(s$ww)C`#C@YCMi3Qw21>v@AHkH>SbLTw5K3etdv3*2e$a{*` z3-&4&R#)x_>Yl_L$Kiyu3@uHuuwcKa7%UhQG{plHY$)fpLS;wDHKwEsOkQJt<_ke4 zE0>ono#Wriwi4T?+GuyA}yb3Cc(=8YtLXt65xJBhWQxqayN7 zA6fU1`zR7FL656~V|wCw0wa$k?2zGBw0WFpAy=iq6C_M7#}Bw=4@Q(><|Z8&9dn+S zApJlxEkN)p)jZb{T-|%}KO|B(7ho*d`lEaCyMz{Q(w-?H3uYm!FTqvzI#ci!=jA40 z)}v<`qw?h#1}m9gC=7!pb@+2rSLp;UB^UQzjFl_p`{60}5G&fl?T0tAbmmZ2{AXV6z=KjdAUw6zCs^hOq>p_+8HMA^m5^=GQu4ny z1ody_T1eWhrNnOqE#`1c1)m=3^hWGMU}|b9R0XG28_Bgr(uVPyTdH}7gme1$5{Z)*(Ww{d41LU9=TJUiF3w5!GF>qSA0c?&{2NTPsOJ|TxWX%ubVn4{om91clHf5X+a>2}J}JN3|)B5|rqFD;45v9h|{2TwijO@;3I&8f`Ho>s{$<9-il;^tVf>=`B?GBMus z1n^$9e%@c(G_l?eVW_o`!}<{-HIP=!;G#MT0)hSrA!0KuCh-DGR-gFF$ehbzK%=G!Sf&?+6@;49;(Nr&TYe3f!0!U^47u@^{GMT;nyJD%k zg3cbhphTM{z-K9gC9-wnLyd?2>KbN?bUlxV+nFTsI*+CGoFkF;6()-6K;4xvOaja2G zO8Cf0fj+rhK@FCD)x)baow|}yg(p_by00uQg1_iFaaaAOHo}I`{#gYXru~xy zj_&X0RC?xNw$EZ59LOTt8;0YpyBAzY`8I{~i^84KKOGVn<^Mk_QhFnkXq+onb%= zFZq+-5nfHkFMsJwAYdqBDLCP*&UirB@686oGe)K-T(>95la&d1sj*9ipg7Dx*0`m_ z1i_H~Td^pQ3SaU?Z(S2yzM=z`R=z6w$JT z^Udk=RJuCb1^hcHZ>#^U7EdP0G;>(s$$yM8e1t*YEPgeIe9&kRB>1s;8U-pCpn}v< zFoT?w^iTGG=xf@4&nGH6>RC84ide1-ZrCO{I%%E&w7|`rRf%o*hZ|Zi$a+xe4R}!H9NSPK^_h+eU-sB)5d*u@(5s+tK0a^IcG1zx*J}(=w$zf) zZv@x5MOTtHre6e4tS~y#WgJw2D!%f&=mVR`9JP!>6FA=8q6CxA-8iR7q8rdoaDu zrt>XEqb-&wqD-QVlL$T&S5~#=apl!!ri|Qyjl^R@x@|XUiwc8NRds-dwzE#qW^gm)G|!r5_C-PPF6= zvosagj(-$v&~}NpN|9k$ll;`j_zf5G>v7W;1rt2X;x&jVNSnMrSYV}5ak(qU0~ZI2 zNUKq#v?z(MNCe_jrd%xKzSOMMqu#87@?mmibm*lB8*V9J1Jfhywf;c}0M)ih`%HO! zP`5<3?seWN5`S7?Lfx|r;!r#xy@E2GU+lH_|3VH()wlw6D&+{WzyHP3?CxUM9O- zo2~|5_>g8)o2({rj4p1x1#QdZ{a6DmP%vVw7;{jc=>xE&x{r!I5Ksm4-~~(65MHT$ z4I0#cN_m*=1W^Tx#PdXLmr{zWo**+7Y8VxT>1HLxbB5zJ(8%F6*uNTL;`b;WxuGZe zL7>CMS>w2H_l^jomY60u6akkNHex71dUTu@?RIQnBP&z;0#f7sf(6}|GW!!OGIo_p z4XJwBc$q_k7-R01M-H;1NB5QB%L)e~x*Rr4J~x)nDOvgXUk${@AY=1P%HlJK+gp|gf%b(D^m+-%=>v37nqJVeb6E=Vs;Z3@+8 zYZ?qjr}bzEwxj?D(hDoLpN<_dR~~7Gnad78gW?rn$yA(~p{DG|rcW%On1|G- zfEEXHv~s}2juet4Tj=l}8Q%EdZm(?|{Zvy&Eh2<5nnvuJbn4llnC>fiH6$xdt1_^c`jobPZNL%~ zegsQ*%0|SvMJ$Q=HmVz3VH74$35=I)?U-Qdg#8WId>G>GrCP)&L->ETAHE4dfWB#F?Jp(T8dsc zpEf42O-pA8?o3HsFUiF=tY9n=DEU!&4(_pSm@n!H-BSATj#R|q%J~LrS%uTixj!ep zc;DJS@m)`H+xE$+l&7?O#KpTEh6d|yluDCKMXcFM;byrOgt?ryqvW*i2StV~ASQ2k z%CnKgfGs+f!;gIV2s$a+qJI0BS5dV9-nUZMz>vtg=d^aA!Jhmd#ew=$j?+*#(myQ| zwM|KkGzs!8T7a~SO{(m~=wE*JfeiFxXFs>aJd5znNp>NjSCWh4jJ6F`^0DI#)WoPm zR#{i@L9u^Z%m4b?bhI$Nv^;a)hD7pk1jaV2x2jHOl~e>g)3#3-!NH3j?Jya<&LIE- z2+IZ18L9!0qpm4__sMlW6m_Yzxv=^JTcl8yMv@!!;z&ll{&J)mY5qs)ePpI=qU5Y~jNTig zc7}fGvf~rAOV-STuL8s6H6E^rRM7jYn~pIwo%?;|V8;LVmrcz47y0}l+oXC@AvcXm z?)v@J$LYx%v4@JO8;_dJi7$&$ zM1J0#WBSuV*;&a8*=rT4f@wOIS`B1J-f4+9fxmw#&+KR@L!bKfI)zxPn}*-xIoyAgmj zwkz@9kN^ll_J4?w`*2e8!5WUfM7u(^!jE+WAA?v#-BNPAJTnE%Zae##ve`s4a^~SV zXbYbGmVYkD1-|-$Q@Tx2Rs*we02a#G*6AdOTzaSFO_q|QNoJa;RAR#3S^!eBa$z4J z)kB{gkCyZ7m8<$RIL(+INdf>h6`+sT3rw!v2S57cXR1VJfT9%qKrdyE*q_q3`%A}j z%)LTEi{mypE=KN)=`8moCtxoGi4K$-j2-SOhTf0UwTCkE*Y>V={*>b7yx`t&-r_No zknlZv`&^GJ&1nM5ODg`r!f1J9Lwbsl;GU^r<8|YMJ1?Of|K^RT)Wx+(Tbkt?ARQ}q$nP0c|@ z2;N1vl=ev4DVZkmNK$jwJ>3W}M_h7UwS`BHUO(Ko}$pIgwx`8k9#*5#Oj^YQ<1jB0dj5aVQxZ9E{{)9E_$%dGE;|Br1 zC-xs3uKVyg_vO@f!A(_*WYSJ9_)9)+&@8a>_8EL~!L%%7D4jf*4i4wOTXYap5MxdY z-!lY&>gAVE@W=-5p%YO?v-BruHj668xucmt#$+T1`p!GNBShiqFlp1hResn=uVDLw zqsgVfQ#X1pL#C(ReVbNlOPth`pV(87LDkyrPx{0Aw79j0BuDmtz zGV7J59(xH$o`P+$EBf)VT);-s$2ya9*D3kLDA%>2=pjFO!dL(d@VZG4hmu7m6#0M5 zbG{GL)Kx^-vLzzsPUTsN`!^^CCe@*K9Rn0oJZAew#}5brlzvR!thb7_w=|ktdo;5t ze*}w6zY3GJ2^00UZ$^i&T;hj6@!}E;8w3~&Bz+pR;vn?M2{nF}?6-a$3KRdu+EI^k>8Ii$u0>(@hQ-Da4bnyKYX?0uMH5JBt@PKxfs(nG0W zWCi01#}tIysm{c+Od(Bpe?4IXh*k7~Efs9Pvs8knkBJ5E+I z*y4>*GlZv!KiKwHQNIs4$WRW{4_brngA>Cd2yT0fTi2%w+KLqc$IhieS%t(9{ig{v z#d8^Q_TX~9qQCO&AYVVTsx1a~)-HL5-$6*OF13dNyT&y|RG>~Ro7d#Ctq($mQWWp( z{++g0|Ls#LpS=Cx>Bp;R-AiIlnESqcLU$Ce-XQs8Wv4z?aqpI$_Q5k_^BHh!cOS3? zCIQO`40FCvB^&szLo6U3_|9^OH=oM`bFLWw20zBdRcd=07i3PuE*}0RIH+^<7>n4k z{pOaziI>H&aYetOL!QHwJ!J^fx(~n@PR2;%S7wiU32V3jqHD!~QUL|M>K0z5msAV8 zNy~+*c&kdGfVWo0MSI3MgWO@!mQkq8GFx@GMA$?CT<;eK>!%>w2uelznB z7ffqTrkMkhm{T^X2zarpf>`5R<8jm_L3v%#`WCcboI+C^e=C__2VdR3E-aGRJ`7H2 zl$f4y>g<4M{QRwUW=-$eQvO&>2GQ&q-wT^BNYakyWyX73udYMptBxnk|824KD@4Fs zA@i?lS|wI<1QVhre%XYiiobdI!fs|VHQ*24lF)q+DWV~XW!oU%#!Yn50=3Eko(^Ai z0;Z7ji+k3V=vAR-Q-Y54nn!W;EF@t!LqykZFN`hci( z-gw5#hdQg(jo-Ay?*yC9ED98(){~$z?4^mq=xpfRYJ}z2QYHy!%cF|_3IpIr9{R(Y zE2|5A_0}AKp?MPzIlN+W>hBi ztHHcnU|ugtz5~#mMI%0x$#nL_%1~?NOidUHOlbmXrO#~9^^%OMn1)Ap?qn#pcn?*E z>Lt@VSyYTi#f`0;8*OJNtW{vxaGJ2`KSz;xIBr)(olkw(s@kfM{amoU6?t@kGWUPx z^GDa`iqvrnRGp}*&z$^dcLx2{Y7d&r>YD4Erd!U?26=lFW*h5zpU zMdrV8T@=4wXk?skz24y(nq~;Acdh~%X`x6cR@r*W?$kfL>qVL?<-0cOCV8)22C>Lq z*mZP@`!22PP+lE$yVgqjaj~T!ip7Mx`ILy5XY;<8kxR9Vn3|l;cJmiZ0Ctj=-w|~f z6O%jc%l_e+&Z9>#&!G~ucC!EC_3xdv;gye}{T>E+ydmV0vNJvHn@kpnK`#I)*-u^{ z3ZXBacGKOPx#7B@n-^LX$iH5=O5Fd$yQM3P)6lKF!OA5h2ot{DQU8`Si91-`(g zohP6Vxd-*ICVp)?V1<`&YKnpTS;)+gQO02P#ao!qE=XP}jeDzrx9gQebO z_efRhZa+6umMDTujOc!^smucFeac9p%9(Rb=GsNAF!DkFFdv&eyK`9=xZk6UzZ1rtJZc=W#+@%^8ch(c?kq z3PAec-~E&cq+#@#PV)ASUJjzVqT5_^Q0fhJvHPR_gwao zSq=88D{)8t1sx0_{fpbr>)Cu7?Sfb{LCZd>CDUHUwO4E~i9br)#s@fsy*QjdjM)$a zEI~*DktjNu9f%*UjNX?{9OYYrq8}Y+1+9n6d`CmvFm9u$7{@B3X5A~l#(w%g=?NIl zrEMG+McEo{2?%W)&am|{BOsWhRlQOnx{a1a%1V_zV-@{0yACkb<0&8AgM=1jA!`n@ zG3zSWVXz@N&EMz>+LT-<2hq|{lIH=1#J&lnCx9ju1UJ$QYk+`6u9N+{K2p)Y z1dqgknF+;(+Q&f|hz?|DRPJwvW2;Zss&zrw#wVs_fsf*8&0 z#UO&CDknk1gY{SR{gt8*fphFKIK?h%(aj(7k;O~-kX9m*M59%~CPelFkM5jLzrKf_ z?)=kjwAOOxv_&xYX!D?J(>JpP18I%?4JGbIOp!+vi`6rnw~^w*)edgIai9JKsUU4% zp6m_4w^S1iQvaaZ?e>Xd===vm`WiL??`#L`vF^p%6T^Y&balbLJ*N1>_>Wr0OorJq z;pD%a|1W~~+CPPyjlch6fE(}@i;#yM0eHm*MyD}k3rBVap`ym@p$FfO{q+oWg%D`B&`jy;RDjMbwnzpD?v3x?b zbY6}Kv}^HxRUfEqe3{u;p;G4a6bFG$QR2%<>U5F^vC1iqhqZHPZjX1XJXS3-wS6ry zQ7D2`)KIQw=ob5_rDJ6bsg)#|*^GlqOG)&yf7F!4{{pQ2Jc0x%l+XgOeTZzj#d4ua zAXm(!V?UJ=zXn5sTI=vMil9o{^%CDoZds21g0Tw?ax7iklS-mH1C-Ry1z&BCYbX7i zNtgEV|7T5i0{=f*)6^cuk}{ldyFxr#N1PhsM;30HIXH zgvrQZpGM?FF4+D;{aulpZG%-vVtCwk{vHp*!Af9b+$E^yTwvdba7?fV$ejdJgvB9) z%u;HZ@mB~1a8ItXW6#hIlHn-jTw^3b`8V3tMs+eEnV$L{RkABuc5nb!m1;Ye{V`vCAcxVUd3RGMJIA(%pbXtqujcImo8MazCjUHb>4z;{ze46dm6IzV&y4nBX$KSxf_%eE`-Xbu8I8-2f9O zH}R;_n-9ui8gQg|M|L^Zr-SFQ2T8ek)yGJ4#%G@=)-jKT!=&@$Drdj$>a#bxc-)pH zgHvc7TqE`k7P`ssf^@OJ{$krtSVcXo~9pS3n2I--h5Z?vrDt}Y8xr2e9g~6vdq`p8) zNiUH*8;BEHf=~+4^YT{&!PMs+O`JE}Z}Q1P+Nmu1K;2}7<4S`IV6&uLB!t$l(j6<7kT~g8ydtkvAc8&&13KjjZ~GOVXw7G+XYUq;^IiL1#UVQ1dJnL>aIK=ET*5nkyXQuST!%_rH!xExlm zHBL!aq!42k{#+jQ{kEE*939S7a-&0O$9{eQ#pl@C9Ynmw)@eHAZ#Ox_H&~sP;K^Iw zx(_-qanEA}FOgEVg+ZOkiXPv-7bQ8BPJ$WxbP7n#Z8;(8HX~3=U-?cFsIVlAHjOA& zvOv{R;1h1h;-mlA>%`)71(#Nwxe7eZ#GML^P{31UT{al^CD+hX!X%8bYTSkDFP~s8 zz-^>pPua;~oH+w$xK!=vfAKe7w$OKkm}3vZF#K^z@5wq7$+@ z?_SK}#X-v>efu7!wIiW9F2#GUAlEpKISo_VyR3mr$B}g(L$tLj;zM}x)X@W1_$)-&FgB`W);j4E8+u!Ryf80hZ%e49WWd@;Zy;lPj|I9Rhny>Zh5y5&A}afx*{u z##)gJ5MP+?Kj_~OOV!q@YTL);CAh>UXi*pzXV(e_>YK9ekarK1iDZ0lW6Saj{p?*i z&hnfwg_2lr+h_1o4mm*ct(@}Zc)$ulh%}#1C`Q3ICtqTcbGfJja@Rl zBM_9K3X0nm;53sg#bx`puhncR+@WZTFKT9E+1cjF?<|56CJz$(;pzq798Y>On(+J`@uVq)TlQk9Lq?4;s>Bv<}!^v zD@bE1hMj*Uf3W&+bz&^3=aNGc=0_7&{v+@C$HgRKGZ*yYQ8H3?t_uPpkG=XuR5RM9 zBCNy$D686HP`3WkNJWHlkdR>`6-tVVHSBcY%z)b1Dp?}iqSp)m!-_Labim_s zz&w%14Y2xXez@7ebDwY2^KmeL=)m%^uleB=>%)hhT?p=xV_t>f)D4C?CYrpS?{;sE zycVAsMcF`yNv2?La5cXW92l=YwF|iM)$#DzsW`fFKp$%RbP=@5fC%r=?F!S%%VOg_ zvlg}}*2*xT<8jH3pF8+^Gx#G{o8?CtTxE-GEDP`hi(S5@o-&s?TGM%$Q18ch?tjP~ zO;^)pV@InK7R|)ZL)`bIk$vIec_yvsh&S?mp!POh#qo37;qA@!Bz^f(S5y8rzyG^*WAo^_W95372CF$*Ff%W*}u{!LL z%GLk1t3NpZEbTGz!sk(ci->y<^=C;nGcwdP3w#Dfn6hOJGy}P;b$9HB;1P^fuJ2_& zR}KZ7*mMJApS1$r!C4j)CXq5Ul|AqY(-xesUywAUNt(y#m!4G^>&EB4S)KDY%$}|? zRdvxl9VyV~u(Ewe>^wb{$n8sN5$|OBRX*E2^J`sV(CbpGN=wZ=J^vt8q%BtLXRr6l z)JoEZy?;I=5(!EnSCD3#!wXsf$800Fe8BUr>GBhogpv)EF!`4#ojn7_Iw11a&<_z6 z6INdPgVZGEp)x$TMF*|{j24&DT_wWr=iVA4_~Uxgw{=C~eY+?a^UI!0Sp=csn@IU{ zuqlz?|}Rar+XCYe;EGEV*C(UVtNtPDQffIx7eX_mjEKGF2cu za~QP(bmWK|N7AKt4*^BUpliZsLb}gEwq+l6kndf$N1BO|AgA0KlESC7LiF1*-ScVq zGIJ7WZd*t5o#8y}PhTJwb<=)T&CtLO_+XrJj;`3>nZB&q0ZyUBE5L_NlWj|?!>%C~ z3_mF{XegO~SCEli3Z4>my+I^r-|MfZJ`@$N+3!A}ak5Zqidx7|SMF;Vie>uV&t{kD z8XR_@fZy_XJO2m^othesU|^wdYiDDi`{Cz1OOPo7D=~oh=PM5nqqKpAk)1IyI|~aZ zqo|3woq-LbsJX74frx>grM>|lAHsjPbo#unsctzZUgj}eywv}kNuWTt+!-1hagD9K zJVDBEzqe`4FLEn^8zVb5SD$H`$n!?%JGPSMgo8vd0V?)1a$HsY&TgHfeI=aGc^b3s zgHfH+$;wI)62qii6R&ox%u#i-`|Vgikrwo5_3-BY*rufi+|ZoO%`Lm zBWZE$@pT&u!q5tJl74IFN^kXIkib26uy;s5)r}E@p848M9t}-I{>{$jf$D5=-Bp<~#FaNk!WvIv4DmI8TS+qMO zsLDQlPpA&3P?O1$gcW>y21KlePH%6)R5SVFu2{%^o>Q2F4X;`rgJsipbDR;6%%I(I zf!;iL+TZY{=u!$#hXZ?fZ0&h&70IbXQi{s4IwSvNxhX*d?HooNZ{?OMEl!yj+!O}v z5}DM6;T6$lUMDf{j$*)wiVIc>lvAYz&Y`awj{{VOFNFdm;;w|R4V5`XqmWa@3ZKF16&4o z4Xe~9We~fDNmGoUoq$N2jBiZgx3{L1&Utz#9wkP^*HnBN$;ci2JSk&>Mz9&#T@3|s zj*c;&C-a@p5#42)8`=i6qC21Tpi@HPBBN8Mk@ndjG z4#XFGIApUU)Ue2gr|8bc*7-TGGt&C9F>s(WT9XEHtZsgSEB`Lwi|Wg?N8(nR@}T8~ z&4ZpU_|~vnbhK!kz9IxfinuRoMXI#~qwo6Bc>X#O?PRD&jSU8+v8pFEq=Cm1)2;eL z&WA>G0o8aeA)bCR{{-}2v_1dzTv=mqmvHig2+fpQOvw9?J18;yL#OrH6U^r7kIR(Rf$!U7h5Hr=#liwgDt&JYGT?_b72E;{2stD= zf=i6AdzS;Ox{k^DFh2W!!AOHQvnA$Wg(HB$`CK)AyR&IK?WzycCr#WM67NYiFqx>C zr`~fauek88;7fGi5}gy@tqR+iDK-*o+-bgsT=A0dHsIZ;Do3Oy1HIDHazyN_w3w4M zy?*P{au5JFZy{_r@^g<>mdmF_2Cv~`atX1rm8%~Ip6_X_lfppCVS=!&S7%aK-Qb#3 z?nSa*pSqrjQbl-^b&dZH=kAg>7D;3|JxYuMnd{h?}F8LUuf$$Dl6J1nL~= z%hF!H(g`!ixZE&7c0-|(ByGq_ZmC*$ERdVXCc1@kf?DTw!Xw3LsENRG+T?J8^7A!D zM?%Z6pE&+ni*t2q^SX-RL~mvw4*VdaLwdcw_mZdJQGU|4w$0Nu#8c~;r$#mz zuW8!HVT9um)XL%8v~mf&!*hw;`CT^4;!sS*TfDtubh(U1)-!FJf^@qyl~vDMGjRjV z4tfcBL{4$}3qRuJ7ftBk|B7y8fs2&-YrGG09YorPp0bp>QJ-2B(OQ>!>9SfdIpAp? zt`iq8Ou6u}uxZ}t`bMLzqy;U?x*)vxd8!%KO1Woij)QnRXFL9e%crJF71CtWz(xB{G*qW7)HT5!aNGIkytf7vc?iEIz>PN{@by zer<&b0w#Qz=ov;JC z=$lvp@da>xj`(BDUwZ8TIy}1s15;jAI$z|ui35djDYLv9U!qXg>OwZ+NAM20Hc>zN z4%=j-QNo8Fxb=rcpm-niTX#8Ask_THZH$%$uA^V%9>gn6)S&9I<^aAuW6DN*8K1deuur%r74QymkBF}<&07g^1npg@>k zv-=9b(NZpjRSF@;k4)pd<8re^KVrWVEe;neM=jx%7P%sJ%%q%~<)&5ePZy^@)Tym> z+FV?_MAoS5cVsI?#INVi7k!ffG}2ZUC6r+gxI|&&@028 z&jIu8I-%?uz)wW@im}R1`4!=GQHIl)9|%W^DAs;6`8eGBJ!P!D#l33cH6{l57ovna zWE0KZYQIUViV8SBhC?G`@;b7+N3pKVdW7@5@pnQX*IYQqVz*W3QCutey7m1czi=|7 z?kyh(Ze>@sq@e@q?jW&Y>H~PyFRFtO^JKVw$6j1pj~z<^73xYXjR$ug@=O0%%*Rh+ z4@=VHT!81!f@lI?a}=3v5@zCH##SJNFEsaiZ0 z+@0#2mP`8bAeS`i;F%d!K7c<^WG#|B;VYhIV5oPC1=_GU7o0S-;L_A$Kr8}IvPKsao0$Dwhovh!O|O&AaN%qf z>D_jIsuuK^!C0JqIlSb0($vJ~;Hy*aZg-JsNw{I%x{$(y4S{#B;JvxMpJ`Dc$#FT^ zBP@R8#sJtktDub#fY``zhmDzZZpbNW8YGDY7vV7=>d;7vI~P7+n=83$%Ea#`22aFm z(&z+EE%FvF+q#+IWmyIA^7uOEixtQSI3?@15ChogqT2k$VVr&-&%V|?OO!Iialg#K zOE3N+xVGuCz#{peC-=QOzz7Z$BgKVGjCy%Or^77RHTcR zyZmF9xx(J3O}P5O5HTnB4Bye^6gr+$MNYF(5OIjO5X1~);v2N)!Ce;49r(JqZCAcpQSa6@4TMH%F`WOM4{t*IR;?E?$RuzhP zoVH~OLxa<_pgqH}yIgrD@p9C$0D5T{!S#S!UA7vLcPhhT*5D(r$Vjs9Gyx4-vM?Ns zZZdo}iT7f(v~dYG84QH{Yj!w0F>oDaSAfA93^6|{h2VqUp}`p3`ADtOQ0mO&V8IRK zSjhmv{ZWT6Y$p-7LaJ_S_-$e7xbs)xq;5>Vz#l(Qt2Kn(zmm5??(O1vxi8J*o@lzKXKi|uA8Dz%I38Hh?SZ4D*v?QgOs)@Q*P>6V4JDazlmb{!u zr|%a23XMp!Tl+JXDw_8|v@NIe?OQ;H7*6NMgkc45XDty~5m>QC;EuhEy&>^d>2ZiO z204>N4?Mc2e$)~ssYn7BY4n~Uudc09k&)p1_=m!_nBigX%XJtFgaa8Up(SO7cUxbe zgAs5}eWOm*Hl5tVxZgkra!2YzIW{z|p1YzoyN?HAY+=(-TK{;oZYDGJVT5zjwW18k8$ME`zeW7Q=RB# z#l{+nPn_7|&@qmP06CNQvo?q$i_K#TXQaO3gpalT)CVZvo%ANR;ROo9Q{?99=59`G zOXi`{it@Y22;=j(jaTQ5H@9Klrkt7G91h4Lq?qYLoi=pc?)rovQ3}8XKU)YhU%*{B zU-k;Qqoe3~ydis_BPbC(`_87K>?v(OKvbc;Yie8hd3b5=d7}dML4?1u)8d!_ilyIS5Eqs{D{FFv9Nx7Eds4}@Iy)|Bo16!sw=qyB@`4a7irKv>Z>r=Gs(fKqt zXE-D3;NDN&jbojGpht2!5@RkM1aN%@_APSD6JsV=7`HC(*YHGA+yC_qy?x_7KZrO} zpR@_YIU()9bMQuxrt?8!?cLaOMd}Lv4jS^VUIKexSWC4GBeZWooRMP%Z)!Vsh1V%3 z=2?^c608a#aC?1GQ9H_fudsW!WlhmJ>&i&yVzVb2Obx#o%vU%ZNHoGP8ENVZojaw8 ztgzzh)EAN`r#MIQqSKbgPM)D_|k zJSVtG;fr|&;iU_G9q0A$8+4;@muX1nn8P%1HevuM|&m9=`FM7LC+fIVs)*nDUq6-FT5`=&xK8 zGD?(8zAQ6TK}w^az*~yfU~1VpzRO!iu3a@?79${InTB9~5w{~8YEHEB)xP%oRIXI4 zB=yVMqIYN2)*20a`h?rcWC+|1r&OeNJiYGP-{b`twdwL0B($7*`;Af7^s_P}f#|?i zp2LW(+;n9qOZ&ss;Wm}Zh{pZd`*3r)f*hpW!b!1|mFWSAZJrcy-d2u&gY(X~=y=VM2pkZ#T z0W0d`lxRjmwJ#hgdX4eAb=3qMK;W-3x?boH zCpnEfiN5Izmwt>Qq9u~jJi(+w74l$7td1c{HgZ!g{wBBXhT>8r#7q>RqTcEF%-tr+ zX(4aTFRfwKw74I`2*w8mLQa46$Z%Mcp9dBNACVW9u6L}sw!8;0bIfsa;P*+&`hsXz znsB(a5U=&Qx}y>aw!6`DI{!qZA%*Ok?;FA`1uj2=Q{i{CZT%W{Gdi zR0DzI&o}lpnB%M=DeDl^J{TpTji-+qLGC0WB1i8FTB+v+%lxOL(dy|NFXiMf=J??Q z+xV1!|=Q#7mO1{>tY*o?6fjRJ(EpTgQINaF${@5 zdge_x*x5n^(%!qy?&VmDXv2wG^YYi9QY}^3(C6Q4%+PMVdeBPI&Qla8q2Z&N)?kSO1ZV_f5{l)rM zQaePUPN_<7YD-_Dx%-iCalg^_BFX%U8p#Q~nn^bpnIz|*u&OTq&c_VJg4kR#6?R5E zME9e5N>z5`Qkk3Z;B^_kC>wjUHbDPSkaa{JK1y2zD9ATD;1X;EjTFm!9k?)SW`Foi zM^p3tZhqpdF3J9jX;^Y5%;3J2A;$5mSnmNYheX~&BzOoXFDUR2(#KLb!T31B?3x^~ zHW44y){gZ}J2$&c6+mLh!wI5Ujhi6)Xu$(V0=8m*)W#_M6GX;4a(8%olKSX#rpjh~@L(j-?}C^Oq4O)+TN z6TiSUwin&BJF2?RMGgxvnI6Z{U&6{($-&}+Eqh@~0t-)5EBKW<9*2ubN1;#U#QdR_ z=H-NyGRH@5bJ9Gs2-Q!jAP_A4Hc5eBG2=UZhj*VYM6t;s!_T=6o@p&Z3^$B@^l?mh zt>}{O1tnrr|Ls7z-~LraWz=HW%|Q{#8iegTF?ZNd$C-V@*e`mZ2G$!|mLZ~HWkOLP zYa$AaT~@~|u~xGxa~5tVtDlE6I9E)nE3Zik6GZlr(2dX}VbAG@Yc_e-wC7PaUHe)K zAE@@EXv0j)xieoCLEimd8w45Rw7v*?1p3TuWb^wuSZ zOcdMXwB~KGN+<^;$k39Zk#Nr4L$kC%T{Ngo>ON}9O=`X$>DcY3#idiZ?K}smxsy>i z9F&BT=>aTDn5aYtd&#@(jjOvRA!J_YRJd2h*md(Q-9KE`rW_vq9RbRxHajkN(#`ov zUm)27k1tSl7+Is?Su92Ry!9VT+5ENm%eEmHhY%5Xw1PW!$vDRmKXkI*TJ)<<^W96W z_nqTs+g?jwpE}fAuu)sBJE zTH9j}8{dHw*QJv{a2|zD?`(K}(gAawA7Pgsy(LG{U(PVq@ArBs zOn9Gq6i6oKeiE09uFcPy*YFmi%LcypqjCi(?X!d>>$j{Dsu0;kpZKig@q+pj3#)|_P^J8*}%vC_d2hTuC0M6SQDeTfw_Z$or#_< zqoO^??q}nl5`h0D=bt0qSb&#)O)QKUl}#)JEo@Ex-{=38%l@C_(hFId>;Hvtj{hiJ z4rFd(ZEx@w@;U#b{GZ1Ey$;~dQTlJs|1&^uob1FD?ZE5mj6yPuvX(XPx}6a=>OF2 zxBmGTYX8q|{{_nbbK8Fb?mxBtcONnRAwx`m!4T75FvRqS3^6^Cq2G-u(;srg^hAz+ z8Ivb6^h>+H;D_mn{QS;8%ui(Kw{8D|BjzV^^zVLRej-D^wfBh}{i5>^8DjoJhM515 zA?Ckei1{xVVtFD%|L#|oC-U=4yFX-yl4}eoxQA2jtA@(OS^zS}m{|kQD|BxT{C-U<O$Pve1aK!N!9C196qkm_C z1(}fPg3R^!xt)1yn%56RG;G=06W%Kar|mzySU!QNSk>^{d7w()4S? zCzAAg&;AWfz`vmh_(Yn1?eE`^^wdE1Z=Llw(QM#T!`Z*|_o=b$-<$kIs(#=2Z^(LT zSo`;y|As84KPBs_(d}P`=x;*XK;}QaM*V4E8~8VoZ6Na#iTc$^Po(MBhEKdk{o3$} z6#d%pPYHTzZ2K3De;U{Z{%K?z$nvLmxTnUpf9>O+lJlpLZQ!4Vwt-KLZU5TGKPBf+ zquRiyhP8h&^>1R@K-MSH^V_TYQzP5In*1*W{dchdCa@_0aCV43xNrcFnH9h&WMXG4 zZ(t*A`N7K4;%9;j2RO~d#K_o=m>s9qD1I~NVv$M1z2C{-PWy~#YekQ~GMjv>e{Ra#1POv?EeE)T*qLr?m0a&1e ziJpO&jjl5|SK|LwGtk%bfS3u&b<^ zRu`#Yl5y=Ih@b4x4M!hMVsTygq|GZU7WCe%9q$I6Ymb9e4K}skGnlacQ{ekAoJXT0 z!(NSIB9>Dp+auwi63C#fLbf=}QB=JW)S#`fB$UgyYP>#;;v$xMPU_p3y6hOE^*CA> z_u{+`iEpj+{;`Wev94{EOP_Qg4n-@%1GH@%ze51K{d#|9i6CK?hm-$k8 zGi6NPe%Qdrx2fs5Z{Y(QN~L`1NTw(nbph=?vMU4kI?SYY6m(4D!-zb2v$yhMnkoT z2v%ulFfIrf5*TQ~a`=+>)c`4JMsnDS<|UODHU(+$+q;9PRfD&r891GgnFJeX=%b9C zGAO zs^gLgQx9tg-d^0wbVGxot<44EL2rcX-wF`_kUnZ+93vZ`yIVo zh;rNPU9!8IJu|94sv_DS4eZuCdUM;i)CNe-ckZ<gLZEKHi1%BlwMR>hskVfFwgz?jd}(<2v?KcglSgK$~xISJj^tu9wuJ&Obl#! z5(h3Tj;4xj0yDZhh$Ne_m8s5@;3BA7*Ge^Pyh@>4(e(y~F4JsMC;kdAl)*&tOW=d! zhO6QhB3Sv%^ZJf4T`LKPneB$nj?J{KVQ@Er+q{{qL=H^lmUMn98a$&!?M!tb`isDB zkQMNi2?|fZ>I;RNITcZ5=>)9FTPF9Jye*|+OxK$+711dvm)TyB70vgL3Hv`l%p#tF z6;n7!Iq?`4aZ^d57Wh@~#6#$}S#v5Uc?k;V6D8C54V>C1j!P*YmGw7x=JxKwS=@+2 zbIpD{JkpyQdPGPMKJIU_?rB_}4B_+Llp{VCw=}gF8R2RcB3}f6?ZH*wM-)G)oQ~^q zSnS-nnIV{kS^YL!p}FM}z`fRJxcdn5R=>8JZp`}35_!S+wP*uo9>*1_n%im~alyEJ z=FQ?;2eoOw{w-4p_Jo~1tGOQH*BPCY7)kfQS}ykq-oP5=gyWM>f*RzzD8@{;5jZR?HHMUQaX4Pp5gzUpt1&nksZh;?5 znd3mZH;Fnl+P(~iF!t?dada*wZFzCYg2RoU!1Ai{?6x+^eb1M4DQhP5+F_gQB+L-N zT9fw}%?Ao-O@(&H4xkP71?m8#HE_;G|M#X2INF@!}G3^|}*|CBMk3ggLyQ=iaD zsW8-j32TWW%c{d|j!BosefRy~I$^UGdY~XfUW^?vxX)OSBw-0S^-=Y7JM!}`xer8n z3+$jTPwup&FGC^tI{E49=n!aE+F)(XwU=bwv?4{j&jI-&xkZCO$(7I0xyaLU-mx--@iW0AWEXcKDhEdosgo-w= zkBWy^Lv^k66tyfIC~AIhCm%lAq$n3ZJaxFLrprSPI3_%eo+`J_i#^&0I;VfGBI-l= ziv4n!=?16Wq#|yQMn2;&K>HF%b3&`oEEnS zw9C`kPF9$Ij;cA!uY*R-zgMhg4y)@e<|Z#>#Lhx;9~U=iF9^ih1419pZtoZeA(lQz znTwpjk=@UQjvJLD+3QStp3)A26r7z~n9tXR?9sgq`T@m%=+(objeV z)_J+N=$UFy^ODlObxw)`W;3~~$XVXR*`PbJA{F-N&S63`l7^;~Wan~ydv=TeQ~TjZ z7~yymt7{ShLkb>Ky|TN>`}k)-diM^~+$Bq8)~z?Rcj zfN#wtRW+4FLtxVll@9?l+fDC69>asRnh-hIr!!!EczGYXU2!$eVM7Z{0}3-0P*r3C zrvlO>C#wTa*Yx7eyO!GWK3~15B=cv;Obm5M$Xa^X%x=bLT5=VV{Qd?yRFg5_+r!65 zrRZMkVl+0ew;F6_Tb1&^2Pi*1M4O@Z*O!p9{PO24(L< z9HSM>U_-I)wFA0gXRDm~514HZX1k9K^kRrIZwCc%5**5$oW_be-q@lo>h=lqn_O9B zUMHf4s48OHf*zD@PkT&5&ur~l-wuH1IcznhL@r30hT;8bXvGW+nXke&r^Bf%xRH>9 z(}F-Kb)pBS1Z{%&f^FDS!`Y&Sq@lGuGN0xu~0uB z2kRsqb;4EjF>aA>Sl(S>F64#`G%cvW#WhMS8k<@EM4|{!G_}~VJd#XqKfl|3lr*fMBPDdOni8}?Q%sTNI zo!RlVF>)f;`q_TWm95buS?14cJ||h+>PYC~gRj7|hd|V=ZTMb{XfNhb9Z4WE_&15@ zbqG5h?LmiCEj57cM%wND%}7EGafYo>Tt}Rj?CusuLZ=>_0TPNw(~21`{2y-L5@Ose zi#f*goxcw}7F69SrimLp4@ynzxX4M25*`MLaz=CZ`@U;Q#VU5JPO$aS6(leWBkx~~ zzFQKq5d_!m4?&o>PL(&8l94&GoH9f4YhLS?RYxL{uxQx=962+CYMUv+vme;oTFXuK1S@$ zB0b+A#Xgiw_hD>3yXgMz(5sj&#tx=Bu44>6pj{*+1EJ~wHI=HT+CZSx!7Vm(5rFnf zJ3jfrRqDGJGJGx@fMu9=Qni5)Neih7`_v2U{F(>=HqmuZz`Lk+nwjU0m$=qPf>iMR zTRVFtM1(d0*>tRsk{X}Xgl4si)CMB5F7A))urhP7xT7m>ozw=B108QBa91q>bOoI( zf~M*@sPsO&hP^4H_dQ}4Fg|Hf?Goy54(;DLDh^PELHe|wAsjQ90PKQb;y-bL?PPT= zNg(D1@s%V1TE5T>Hcja%^gJ(zz(^)BilC?)milAJlhvBD~n$Z#c zk(0)Jw0dj)e)XNKo4n2%DLn%L?f^9783w#JJNpGQ35dUjH)HLNX zCVD7BX#%;_2dVkYj%3Av$81t*vDNO3HQz6tNWJgRpFan8S!9y}h3qA|=$=wI{JP!r zWJ3O4XwTw0%Ha);kBc;%R0R)Qya3kz#p{q|WtK6%UG4p=xLT~JFZXj{9!{PLzce3ZEU0{O&iKzJkek{ z(*;1C;SiN}cEJ14po3VgaNPagDte)&#!SbR--t(XK8!9mg_ zt-KXOedzB!U0U)R$m;0c2JoqRtA&OK4TEcdQg+}2ZJ{!Y0mf}0e^DPP++AQ3c{(Lt zXk&D>x;~*cPXE?yLvHAS)%KT#Evdb1`mSBOn$M9fg3pD!&N?+LI95;LUaT>dhh8Ln ziM|rIHLOuP9wnNK$deL5rW|i!s$)A=)C$!Q*|};Z#0(DdY!M7!mJC`k2k^c1=0XaO z;+M_+DAtqqrS<4J42cgs*GbYq@(L~dn^DkSnA=(JRu4s1W3LXNQJ_H#U=!JqC*L^~ zKfJAuG!^w#ehkqC2Uyj9M6LV4vIXFiZQw8kxO^0B(QSxppeOK;_YLD!MQW5lS_4n| z0}m@AQ#1QAUd|ARGOSn~KU)n!-*^@Ox%U-V5EatPiz%rEQ=Nl^Cc=JA&_7aieoEO5 z3nSVl<3ws~Eh$B5-_^m@Ox_w=16xA)lAn#9JGMOh6A8mcPBLz*(P zRwav8C4mWD)4=`YFx{eCAu#*0otq3MtP-b)%2Fz18Cq9e4n?@QBaH1<(v`u49(+Yb z6uFfAhZ4b=r2P<&rdDtGcPpl?YmmAvfvv_~S6-BZ2pzKqKW!nq2|&(S`;f@2EH{JF z{C0L0)_k{DcPG}(gvIETk)*((W;2&*&kqcpl)3eG3rGk!{XMV~TnC159e@WZ3Q!@9 z8^z9qhKY8E$%MsW`L1h+Vy%9Glkyd4Xjy10+whfb*@{#7gbB%CxeYw*(}<|iW8Wz{ zG%unY-zUE*T2x0GF73_^(_wj-k7uR9?l&HS#klRzIeX?KiX9e@9na=HDygo=J+<0a zfzMz;ESEftqlE}|l@22hPN;^^4+BQOSEoUa3314`FfZA^3xMTp62B*pdy5noxNG@O z4{<)f(TG3l3#{4~{2_~g63-dYZQc6~6>NBAtCkZ0r`DGTT(Ej08s^7$UB=WUT)3|0 zTe^LSB+Fg%Z&QU#IJ|CDDm78!vgs!iXIj8Zq=~P(K8=6Y@gG>x$hvm@f_Yb>U79} zenRo{*JY!o5%1e|0242ZEs^~h&@DH*bcARWMpb9?SrVw~`5xv2D3rlgnyeaWDX4lI*cq&}za)DZ z#BQqxNXIfahVo$!BI^5J5!gO&684#Kcv^_%!b=l7sA%y;<;=~)C8Rjb^@yuh0CjkA zqp?Bp5TP363j<-Tz6vEI+LN}TBsBtQ{ei4#uhooj?QxP3UvAmrAhA`AL%$7ge-DGh zV$-r^3cgrUYAu3KBqwHe7);4le>W5#+hquM2C_+k0^<||J8Sq>=$o}w%~(>d{g&5A zc%?X+$o*S=9lNhY88E<(0}0VgM+}3@rkY&tlVCp{F#n^j5+_-zAuz19DP(O^P186^1W1AfUpz=~S==E6 zSR8@}hsE6;LI}YLB+{wZJW3xVdhK6F8BqWA|Ax(di$sFvI*ji*XSv9>(Y}0AZE}+@}J<9Kt zC>?>imlB5z3+h?&{;-ea|m^Y5OIE0>)?DG0tG`hfzK+-A-U zzrsRr$@P4W6FvqH<9E}wF@x7r$p{Vim;9SVg`*zgjvW~ZQiD=te8PJE=F;+hjx8{c zY5>|Xe?*!eWdFO_UC0xZPs}2*c~1+;_V^3Xy%(DRUjgvBcwovw)m1Dus@L*lkC*3t z|7uyd-3+HHyJe#_{A6&_wUV25WEOO}0(NRR^payaP$6y*R2HJA8EK1;9? zBk^DK^QHO^{jiKVL(Q3nEz>|EOE#rGx6Oa(g1-Uc#wHTGAwxZYuwzlPe|n%R97~SZ zKhJkJ*_Vn;7r#(hPoBG{`@j8CPN(zlc>16O9Jy2rrMFqE2EuO*!MH}>c=0xy>mW+CBNuUo6dcOW>Fvw=LbjH|l(3rfbpJ|H0( z7QuRR+K>bc!i%IRe+$nZgVf@^(g;t$9-$u6SAz7z%ZX6}Hjy*Tb1so-j$IT&hVsG|J5D4T=u%x8W{Wi-qsbOg$Dm&k zLi3;E8jY4?v5T)-0LfHA*w_=1&y$iCi&EB)DO~IJuR!<+0ly=rh&~ z>r)9!#kF}qrY6KGd51x2hDXU$yCJVOlPDO`j+EYH*rnZL5e(<|0ya1la|nO%V)^iM z(+pkk4^0qv<;I(6%7$TxS0rP2dJY$;CSv4-7e!U}`8vN`4%Pt%uLV2lKS-=2PphP@ zki8J_BHP+ZQSJHJljotP??&JCVpk49f~7}Yh8rA*-#IqevmbkI4MD5Zbkj-DJN`kT z-gfeJxd-uc{Ugvfa{h>V0mF&TXmJ@#qlzODQT!pmXDbJ0$wHT?MCQg;2Lrr(M%76^ z{@*(semo{Zk1wi>I0b#DwU-m3>FO{R#8iB~Rn;=Vbdjg)lO77VCx5w(XivpY_ht+g z9|%`+K*74!QF6qaV4;RYwnU~MQyj$o@Xyw&>A0Voo;LH>v`iGDw6uFglmwW%xMrBP z11Zu_NMKH|%*sz`ie0%zW%eABywjt{m!yz$bQHNU9SB*wb=UOI0E_$}6Gc>B*q$TE zhq}btqZXge_94R8htK}(=*-5Ub}(d@vUC<8xo#A^Aa&Tw!_AfWrlSd3N^s=ML#DaZ7xk z!)#H<(L8$Stc!jCkVIO&75b0){&=|w7dN7IA6)rNWBdMhPy%`jZM)H*PfdJaXQcHn zr!@C-k#HXraYCz7|My1(Wn6YG-HVv+xT1=>eZ*%&d<~BiLMZ`dj*}nu_Fq|~#e{wP zG7%rsO;6UujM9@O$&cO;XEeR2ozlZx~|&= zRHIf0^*-FrtKxmk)TrjcS|+mgG=ZHUDb^@rz_3)}lIvfy{578lOOdv6btOe#7cHQP z>W5`8+&%xNUl)1f1>pbFdBr<5ceZ)iP|;LHMy}y?bm=F+Ty!5ErN&|=AQ8)-GgKdy z)I>nSmDH@9c=>Y)$D0jHTMbV`TK*V759)gb>$~t8pA$|Q0@JsfpXVOZ_~psY;hs1k zm2bFL5-0j6%A@iTA_gEmqOs!X~MjK8l z*dXP44_=P1bdr2p_2LN+o87vE=MB5@A;98=To#MPzm_g+1I*S=Q)RjlA6NdMA4D2= z9cUIzgcTYbgi7F?Q6o2}e2p@FG}@T+igE%=3G^$ai8fiDA|)?A_w zSnGDd7rfIg`$!S~M$D15$;V1cFtyBtb5J`UFj}F>G`FM~@rz(9>N$|MR)p?#FM)tvYA|6lV^zKP?6OSoz4? zF>d)^9Ee*KY^u8OeDp`)KR}~=92Y+gXisjC&U+~Bd&Tc%yov=>hFl3>*1_(+BxLVN3X>oty;p$mI+-zC>~@R-f(E#Dx&aTMGM$er-erTIk>%TZH3wq)?q+&$Ru zIkNdrUeFdGGGE3z|J9~ht~K^@zQ|+}=BF4Cl#hD~JhllxP%wu(DnV?u_78qtx=u*z z`mAw|UETc-e4|X_rxS$?F@7*LT$9~yTI^N0oA--7;sj99K88f8lCVj+)Akvh z2+LhZHRYj{dFZQ8zg`7@l6cCQ&Zqk%4qsH-4Wup6&H%^Rwbt2f@B3s_iqLrll>gCr z1s@ZA!+&t|+2D`zF}hDktF_l%K1dKV)tXtrS~OQADAa?0uaPNC#EllhOeSfvKk#aceKbQZ+iGT=n2E)P_jCec^4et=WuO66N*hu17(DxX3kR zMz~F!b{45l$zzhF<~gbDmBu{@2z`dZ`msAKA%XF1%TND}Q~;BHNhEDS-rEmA6%B1j zkKM$`SS-(LO3dk$0*)jBo0rgbS}>v|K1pfxjY-CFcxU8W(-nh^K4GH|h`vqv{zrlu zj+09`o>V2+CA}_AUDg>kN`9aYQ4KEsSOH5gb0AWScZjFa&(hAYA`@jVEo^R|=7KvA?%foJcFs?R!o1_pX;Yrm5wC(Vg5oF? z!i*0UPEUENA2SxUDcp`2*@#J&=4f`3g(iVk`%l=ZI+W$`Xawn7{hii{v2mGT@M)sZ|y+{w%LJ z&6OaEi)R-<&Up8}Y2|S^+Sj3v#szH?8DW^s;ZCF?`hpFxLsZV_m(sRvrY@eQJpw0M z6WW5z)FPKN)$B5tyxAkIJ<7N&IX%p4s+NK?Yf8fH@fC5;#J?Yr6H-+28?G@=t>L|t zmYdT4mmgoG|AH{r>2q9QuuMh^c!`G6t3bk2CvxXuSci}wZ7z(8d>NG>owrQosdNX; zj_IfMF*p}}@zgMcuxYsx#cio1ZJmWf4!c9v&vFfy3%4N{*{y#1>;YR6R?OCq-uC7! zee{%w`jKsE5S;ckokV|g?gM%5#bPC@Jg`O(UsM;d_0EK0#U`C4zpqZ}g8prG_r^wW z+}fsdZiXfcE5wjJF>HDXat1ek`pc<@b@K3 zb*-ndHWM|=V!?6xQ}Z2gLDi&}8>eEkL_hP%XCIzcj|!s%Sa+TW28ZswD`uKLu4=yzu)#w3!2r&lub73WP9n4p4q_9WKag{a*A&uQ;T zbJNopCzgWF_|U&w7T(@SoA>N|Y}rrEp5CcVo6i(0oYWyXcyiB-n#-UM5S&wR55 z-%c;^06uWTiJ3R!MfoaXmMkBgGrUUdbCmqWo6W(n z4Y^&%^*<@w3P_ur3!ncGq0?TfkZfA5Te4u+!iO1?W{o!*_V13EAZ!&Tl|RG#|| zJbjFKN`DfDGX~!ep#BXICgfeT`}PH{`#cKjd!TFBH(C2*M+b1WfSY}QP7;qU)RhRw zfXmtYOHHyp=DZg4%a+nJKpTW^SjBj3g}NRW8m{fpdTUMWZ^(fQek@p;;W%B3*DJ`i+WVbrRLtz^IMWKz0wCIxeH+zVtmGarf z@7#G{5`NF!Y7yV?9I8rWO(2Nb*l$uiL-abm&6oCT3(ci-=>P&73mV_rTPMdl`0=<= z!aV|SFWrOH=p-^r`nGqHgnu>v6BS^d0$*2{67?VQpC{>TO}O+k4z_~H zs4{^HRYoX*Ty`&!E)q68f%B5&RMWQy^)NKVXF7#MA;7t#_BjwD*N2T}s?unWh&D8^ zi857bI#@{y*`PdoxcO;0B{pQkZn+e)m4b;vC@FEAx|T9APkM$z2mvA%f9_9QrOa|y zzoN=uE8{nG>~(glt{I=qQvYL&}MFYLFx3CgA49&;Dm;q(%0 zd3~9GI+5_Q*KhY>eRpDJ%)YS_2Y-p@EM#Zy#2QS&!})>W+T9 zsH~ej_9xzHUnRB)&?vecpQKlEUZ@XLGA_kU;G94yCeF>WE}K01wLJhWMN2Ryy5U+E z4@i~XZ{t<*iAA}x=T95)dvOc8oN|!86e~TF{mrw1%NKAXZC|;nJ~XgUx-PpQPLduP z=;G@}#erJDe^dC-c%xC1t*qbB0kv+C-dcBZd8N>$kOq0=PF7 z7rWX8`ySiV(FuK9X(i9Gi$~jGH+Kcv3%XIbDpZ$tITdo@40Kr%$0tdtdWVq#xbxRQ z-0M-;_sWNiS+DlYP zJ)uexE0bo8s;TZfp&<|++qKwOLg~Vj7d1U3C5aa5-`{?4nK(hN%Wef;a z1!rDE5%rWm0->$#0*%Z%lne-GLL@Z=aR{(X!)^BbF;nL`m!@l{Q=k=-40>j&8vo8WKVtvGkCE!kdaJY&s|X zh47IZP;-X%X7tV6cRDH@`YnrKe&Fj_9#1X9fx?Q+nDV>Yv=HgoqReXzATX9*OG>c! z1r?RS&&WM65dP|uFW_bOSAqAlfIQ;K;RRgS>^b%qHZ7@(5nSvqc`|jm9}S)X!L!-= zA@=P4At4aAw|pHQRX{K<<$<_6=5lC=1@pnVMfib)1Xpym-hd__F7`_ryKxFIpdE^t zzj=hEWt@LF2jN)P!cQN+l`FNIhI8bHq zYMT<>SUMs@&ckr1w7-L`fP-TXqaT1x=^Q$>7dh5c_qgI_w(t1aBvIZF>T1| zuva)-LSi477%--dIRhh52&_IbHg0!8x$1|hUqE1`viirVDASl@|tnN7xbAL5;f|T!;?2jZ{y7Q*3VO zgg2>a8E4;0^Ho&yWi&!(P0QQx5BSh?3VQ28H-;y9s^v38jb=^NfR1X;=yysBe;ebj z*3A$#Ts0k8Nb(Rs&(ZG2gFirHxDjJw%LM$50ik*bb;JCN1k)qk_~$?=X{% zi@3!83=~%)U7;!pB=1dTBrs*n9B1w|v(xhnMThdzq>wFh$R(CH(P|n`INgu$NaDk7 z^k{aL9S1gu^)`rOyq0CtxL6l5=*-Zk!{_RSmPg~<*9E{A4cvL)h{(! zfqGTyT35M@+t7JPhoED34f14e{pt-m@x$ut{f1s*{MLtly*nhjNWX3E;!sa9htsre zGKB&Pk6Ozv1+-@dxm_l#WSynebKgq}(z7ymzE+_oJxlMOG+0+r@FbtASDPcqoS1g61#*L4F+-LXlwRC4=ueVfETc1;5M?wM`OvQaS8<2Su-3-H?2v@6`y{8pz{8e=Rjbh$zil`S8AWTS5hU zA>Uj47rGLk>rw5^^n!=|QkhC33Nq-N%sL)~)4dPyq{B4MZfGY>Av1o1z(Z`Jf~9?k zqs&U|JAuxpWKGQGpN(BnXit7tP5dhJIBC_bcsV0n-6Bfwsvq!6*4J3v&m?rB_vZLP z?q;{4Gy(!ugO(ylHoUdUMU1SHSlw!%uIC1YaSe6SdSiyMpd^#u)a4zo+q)Powe8&D zrJrTJ)hU#)Ij@$GhoqItZJ5cv-KmD3v)cM}TCjCKO&M=NJQ6X4mocr08+Z*F^9wgPtFZaTf=6ggco zKy$b~+zyS3zzrI*SsnP6>%v)*t}WR5wJj$D%-PABdaLJB`=v61__|Sdi`6S+KkcZm zAd_3e#Nm{rNt9sWdUbr*uvW>rk|(fkC3eoR5PNl=ARtt%vbDdqBPTc3RL!GE zlF8lb=Q%wNS7s!^Mo8(HJ@Vb#_eTi^%GX5<%cP$q4w&dx%O@-(D#(@!TGpt*Y@G!B z*IDN8yE(Ko2w5sw3RES{N8s8_4^jEAtsmHpyb?yWId6i}_Zd6%XgK_aP;Y4{yOOn) zzgV>%!Xd2`{>GFkc%7~_BzcpseK2%7RAR+6^l7?DlyIS)MGbepgDhxBNZ3xRVi>*` zY*5KA=>2>iJZF@QT~V0l25g}~|1KjoyRH38K}6cxS;m0TvLiQJo{o*YimfCm*GE4# zq~tsll6TG0Nv39-0!sQEjk6v7OcmoZTeEszDt+PX`8g}IUg|_FlD_#iYo*F6C6G4! zL|vWV?;nzKDSv8-r*KkyMQ=726ZO4qRo8av&CAq?ju5JG;uvik9wg_2pA!NemPETh zPC{5{&+|dSnJT&fpEv1BowIxdkfd z-lwQmOXt}d|NUOmsNsrEa)M-kMOG`A=AsZjcP-4syc1ljsGxGyjyj0zT30Wbht^Y3DUibGk`jf`G9m&x%LaxSu>cwH%Ys=lgbGb zjzeF+=OZh2CCh6IA_xCOunsN-1SI46vww5-Vwx&R&uYhKCb>hzY>L&Z9cQ)fln(rM z)x|>L=uUTd=Wzaprf+6x*}!4%wEbFXF3C4R$G$a9<5WG|6E_LD0=BXVe%mrRJ1Es+ z#{V&D7S(2b^R(JusS1-UN4R1FB7CqrYe;X}H96gu*!23BWNETS>h6kQ9s>&DV$(!< zne8>Q4Ts;a*~>0N2+?k@#nUNo0rz!fjwgf^FVY9++s?Lw$(_@PBNSSUaJ?Q$@M6k< zVdN&@U6!S7D{$=3h11w_m~JH0?{LyVY@k>dlW?JFBKF87_3$0%tWgbJZA1+U;g^f# zhr`#w=lvU@b6G^HHk1MUSYa#JeEqZBU<*ai3sqZ;Me#b7>*`GYHRV{l=QAdeCCpmR z*~S5zSR|vq;iyIi9fKeLESK|3p;|tKX+Z|Ob$`Fu4(1HJksZ}!F$-j{t8j-MF`doQ z!;LrvE(<6IYlnBmP=@CoX>K<`X;wQYMKNd-GD*UDL(AH1y*hlhz*kSjP0zj88lAQ* z0=ks=XYtaXuGO`+Ko7-zWS4)y@XK0y^E-3;X6c7YJsTcSRqkfEh>sgic}tvOgnx+@ z=Ps*ugz3Bq{?rzj>zh`jx;ep`qy-V|eb-qXXbwJOD1^2$iM6NhQm=_vtX*NoEDceY zw#C+3C_Ir-;PAU*sM&>z6huIH-Rf#(9`NRVZJTl8nb6SPqD&Hlz@Mhw=o@885PyV! z5yUn~NKv9uQy!h!?>E{C$id@<;U%z&eJKH|(TP3&?WT?M=6kxgU*m`og z3XbNmadd7!0q_g|BdtdHOi~_UKT62)(vpwP zqKjMXn>K=dZCQE$E%KSvPgoRVCfsoafMszSTKi~?08dyt!()_WI9-+(c8T@~8O2#@ zq_(!DH+SPtdRXu5o{QiO5~m--CF6cmwTnRojHB~c(};vu$87+geB+U zNhJ@=Gc#WwgSN}7SIWE>l!q1DMcu)5Gn71p%Xa&B9O@NEM#k<*Y${qp4 zY$u-MAsH^k)I`wA1$nL6gO4C^&E~}y^!1&Ci>lL&a3vM#I*tqUoO&rAQ}$?%h4zSP zBP+3P;fxNUFO`R_BA|9jZdfRBZ%9)s-EOclsbx9c3vcjejW=y!cC8z;k zvT{fs24Je9sc0llOuDN@$vtT)7eo(N#=}^3q^hJ9z2vZBXGe0TVqViD12ZEztKelm zR~<>!@#QGcy6W*N!23QP&?K!gjN$L!HuAy6{inCQ+)!@{EBwJnBW(ZBp^t8y#&a>1 z5mt(QvB;g!xww{6qy!oGUCcmsgQ^*O8JFHV@&CYOC*KgohN%~-9ewyd5EbR;Au(5O5F*a z!U&QqhY@~|!G@h??a&8%QYIZ8%9)f|U_Hs!(#2jyfzKf6hL|32TcR3|78j8O3+FKG zI8cgV4%>m5n+^3t?6m9ZBCK-Sr%bhaNj5GE>M%7WJedc{Rtey?I02USAGS{n!OW);aXhn{TvNhP$7%swA`}lV z$(?5cmwm;Yer5$$5I?&BZ=+;3d#{u4KhFk^bZ#Z+xv6MrFWVV;_KEiQLD1yGTX z$G4jdY4RD)`sGGrsMSC58^?#)EpdA$tv$z5Lc`gq+|QrHM(7jdZ3;9 zKVp8oq=WAY)5he%H({5PQg8eZa#qsfiKZ@yMi$z&)HgKR02|S&B+4Fr>>_Ts0Indl zVMO3n@Q$drra2EBY$;YS`Z_2WjLo?Heoh`ws>uw2CG zQV0dqMMepg14xF*YbFJ-2jR-a0d34=y7Hf!ikKpvIl~Bm_g!L%N`enLs~t7)Yt(_I z1ai3l24D^(c()~1kjatr#+r=cF|D15M=4 za_~`Logq|oei6;j|Is~9Zi;05A&R>(0kcU|%?G$)Zr1|4FHA<;n zAI`GPu5rOm2@YJjL9#^UB_1GRd&U#5`n0%dv)Ubx_%TfOe zLfN}A5WOY*ScluqmTs3HC88_W*>mYLVuHuM`Imc-ScUj%FMf(Q_Tq2&wSwPZ0> zb6<;S?de6|OI0|g=Jo?>h92^&+}}n{MBQj;oUG1mQlAD8n(v<5 z_l^M)FdN)UrLRX>IS3!h^xG+K=Eb#4?UxNc7m$pWOTa?Ep&L;R8X8$n_9g*o{=hxr z^a4H|fb!DfMqdl0ubqL;y^0H$;kLlK`=a{kgIco0IOpi1w(!WjykE-qk(pyYI(l(U zlm5w1g*l3Fb0jb`AfBK1aG#x+1ATX0GXxhtTn)}UuRda%O#*l(7-f75n2@e^{o>Q! zY|lMx2yCm1#bu>LAKtsKijeFA`@McIV4rRtVq^9nT=(OJdEkxwg_C1QMR9Vkz-G^5 zKXOG;%5_;#`w2TMLLbaCWF{#cMsrH6vU7Nkm<-=@+H(p?6mf>AEw0@~QvOGg)_+8E zh;3drLo?)mWoq@-wp1giowVe95yDZR{bef7deA77oq!W<*tVZwc#cv8ojuyAt;$w< zZx!Ar<4F0nL!-GLgI2bjmlBd4ngnrJOfpe*++0vK^^MKMN~W@3lt9N_K(=YJiJmLnwuZ zvWq_m1n~F<7T~ZYej6~-mYpWKE_r;yAm2WF$$z3z#qTZe9YC(RC06Sz{Z&ATgrBkU zs(t!I@S1vXAE!p{na=Jp&pX8djxbO6gH*WtZ?4Hfqa?9a)w3UNBi9Du!nc}9PR?6n zUvp42H@|hfNfHxv^#qOSVh@so>QUby20h-k{8}|LxrCQJ6Z(PB4AK1sLs`2KlL-aG|tFNTEvTtCycMODW`cUSGdHfRozRvKUA7J zbI;?_*92-&vuOi%^ekxd5Q7f=>`KfgMa)5301JDQ%~2q-Q3(Sr8+`b2F4eS|nDaRg zKiNADp0XS9DLK>%yg!;FxLUF!j*4t~hVC+BjAb+Z3WJO$$o%>Zk%JBRrpv@P9%VQk z%yn<~7&Es5zB@!5R%AYK<~TnCez2b5BQ0!Lm*u-w4Vw@8+EPat?q2I43*qNL&D!6= ztmF=;-_3G@8)Z@-C}Jkn`5pF4TZ2c98c>Z9=rN2x8%`TjgNz#3yh+WpryY03{L+v* zSkkFCzCdNilWF~?d+YA29$G+X3HM|_)_&vMAI+6iiazRebq1!s5@$k~PI)5Q7urnb zOxW4;c}nF=MU#z{#k!WfC#xC8+pT9!6i|G5 zIQ~_%eC{{bRi>NvpF!c_Trl4~KP$DA)0sAb0k39eh~S8v7#+VeDuCyUHUE%0iPEXd zF$byNa3*cZ9E0GQ-!O`*fkn9WQS6zc%yXgv$*y@@<*4>Ab9%_9$bl&1O~Fk@n8P1p z(=mqAG}Iww}@>$H6H;?L+u-meCVsRC}R zHvcf~-uUC$IrMZJV|nYS8_BKXj3u4gmN!Xk@5&iBLW(b!oMQxVoH5a{084z_(Vge! zD%8J6?h4Z_lAfz4dIF$0`|}&$bHqU znMweYA{{`*$JW-j2@P1+$TocU!c8|}i2;#P!;(=GG&&w6`>K!@OYd^mLKcqqzKxAg z^%@-^E#Gry1xJ!Qm7u|;^=UU&j1qC4p%z0&_9%AxO*%N1U~&=VyO__GxR!1AXAP70 z2lrH@KnIvb15%N0k>AQY_O8`(jB4o>**2J0E+D*6Y~v`!lb3=1Y}7PMbJldO<}D4s z7nl+;qCYnbr(%9u0|wvlv;_H(5>g&veJ_u zBxqPhzL=+_w#0gaq>AuvZ-5LpNydAxh}j;kR(6dW$+3sKkp##UzT#WUu4G;H?#tH> z@AD$~BrV*Qh@SHip+~|^%cmx}q|<_Y0R0K2_@wHya!n3F90iOKmti?j#I=myYar9NC0-v@s zY5ay}Qo`^MdQ&ML9$2Lj>o#Has)C)`&-X%RvQZ3$P$SLZrVYxBuWT!QsW?#}^(%pG z&?rU>cJFBaN;aAr27X9%D~>OnF#CK+y((v*Ts>u^j}m~!q?p9a3z9dj%K~0_EXX~j z$!$St0ka@91_evz$^H|!38+K``>>Imk?`6kk*O5}Zz`;h7<7sO*`* zPHA?Zkq3BEO7>V(uo>$g7SP{jsq%+OA-{_ksx)*^4wM;aZ~@6n?m7k}j6%Khh5`eR zHOte%y!YK=%{RH{PNTxPp z`#k&ypY;WzX9jGUP4hDPuWnRNa0AWTwU~yWm0@jdcrW_~ut4&zHgQ*t8e7@s7MfQ3 zy|(AU!0Qfac`KBfN6VCs-bS}^n{_RLJ!YfI;c^W5O=CV75LK48XOmy89S#^sOmC#$ z-0odGVWmbiG|!fKM_73MStcp{XAD=iBJ^iYhNd$)L232p=q!;F36Dq!SGq|ou=|!Ec(nqi0qc_8L=eeWs&mmN)?cw3prHJ7MxUIC8ujIK&)Fd z*8Ez)+Wp*!r~;M?0PWvpE(bpKv6BTUE2=+!$l7xN$yX$r#_c%ui>98j&}WsoqsA-u zs%oi8ue*jhBDz^*wV!@lXFt5CZ2bnf|j#DDOH&<4$Hbfow zS8{iROuGMA^c2um^)8_i{#MAkewW>9_?l(#r_|Hy_7t;`IY%0@xqiD9?BqVH{ zlN2&MZ4>ch%qKGxr#?1rhP!#jsk?vQ!`#9-2|c;LLilI`J~R%fb?RMw5RgTpV~Zt zr`YMPOKf0lpdumUq7;ayU_R*oNZk$;lzd8tSDoy@KtzR5@_9me$tfZanaIT{PGh5l z1kHA--G`(SbRg52PiXS7k4bEZ^aynFF&!dEL#ZI_kX0D_iwTqHR%`9CghYl3{%XJ( zknjxnAD$W#^0n)XcmIF$jDOYzCM_YcRhWUeK}j-;aft>ba8vG7JSp1v+M-59#W8i{iD)Mu zasUOiMhpP0KepB-_62kwW5ds^;_P1>lvkqzk!Pj?t7y_f!ffF<*m|~Dzvzx0AyUs>+LtA> zW0Aep#R0xOIn_&Yga!Q4j7Y|3JX>eEvCxU@MRjRrHlJDwURxL!4c*YO?VGFNuZn@_ z2K2nlNSG$}%C4_Kty&ZyL#sKO4-$04H+HVMzk9Y?{uOqS~ zQO*kFlWMD&@RvAttIiyWt zod0_hUv}>QN}c?7-cum&>wi*+A|WBA&tIj_UZ&4pq|IEWAqto2^LO{hW%^%t=C9M| zfJYbUGZ#quxk!elRf~1#&qz@dq_k~h%jCY42ZS18kD~f5^z7A*eDM;fGtFTg9Ats zjW~f#e}ID>*oO>pasYdlB2Hi*=D*Irq5sw9U-kbR;=fq@)B4{~|Aq56+<&n72fM#o z{R@+S+q%c`Z+!koyZ>VMH}rq{`VZ89u=oqJf73e}_G~9}++IJe-c~@=VShI{hXNG|n3C5E zSqt!$ZkBHbKYTw{$h*&$6p23aRp{AHpgS8|k|J7lo!#Xo(!#-3pPvFMGL5-{C?yh2 zN!l>kyceF7|Cj)s_*wH+4h0_4NnX!>cz6y4J`%Tny@|!ft;MC~gVJS;O!2esvK$H! z5=p-jL&zQx6we|?b3eKNF$_ENmA6o)jwJ>6%>>f25;k)JE{&Gv(%iHbr@c8(GGgjhEXt3@)W20?S&*M^G zQ~JsgvgLO@`rU2q9eBIz&4GtRUdJB1;n&Oi_`TE^!!coQZ|no0WCrJamVs2dR$;3k zq%HBlu*G_~HT!I2?SWx{UgP;CEk#iOA%TO2tz%2=|j3 z)Q~JFUBl9Toez_Xsl2>-!05P(Mu3Yi;n+`e<2e@R%x_KYhKG>Zk4 zt9Bq*TtSGk|JhFMW3FJ(ELz&HW0#>v7*vq6X5?A9SJv2>SFgx9yu{9M25F!!SRMp! zP+H+ZdZ5^;<39v`#uW~-74^Iw(i%=cB^890-?U!H*;|5;i6Ie{X+jg5dWieN(DIYk zW(!s0)S6VO`W*Y`gJ5X+E^ud)(YKK^FQ^568pA=Z$hd?DplK`k1FT&tJHM}1d-2PN ztH{H|AtD4GkBgSs=Cvs}+@q7a_;vc2<0h>jv-1cCOrB@Yn;=kd2N(d>6onXe-% zliz}bSsWga83)bPrko`=IKN_VwWyT|gop2#?_;$;AgbX=dPCFp;{`-VfjMGL-=v%u zgCRtOQreLw1g;TdHf+9Uu%(c}aV#R8KeY^co@q9+OQoq04rso~QrvsR1ndP|d1Rlg(-iG6fS z$~Gea?VZ>35oXEq>6P>~968Xa0DxBGWwg&S;{-@CAV)d-FF#|mPwDUX5JIk6NEy87 zsHD78J=HPNnG3AOEb~$ge>IOT9mwvn(v+smCKmf;t)ens@$(G$xl0L}GNRYlURB-d zVbZlV_O4196dC+}(PT~s(b=x^F)AZ8y5^ZWR3F8ffValJxE$Q$d)BPfI-QqJlO`O`hj#8 zgJ7auY&L#WZoLSF zk9jG3^==69Fqp=Z?kbLw=_UAM`K?wS`FYQsXth6*L~^@!Cbp5oev}me^ z`W6N{hGat0v<2_c9o05Pj(*>4yvS1PDGMcoC5mc1duF2Q@u`M^2`452CIbM%zkr-|TDym>y*1}rEn8e+S^5~f zG1Po~_!L$9%Ann-JmDn4km3QmI=*z^qoVq+{=-8VKZ_`SF$U}%d4CMrh!&I%g$q*^ z$sP*bO#VEDi-IoF67`S2s`oVeG#ioMMfrkkZCfOzW$sjPq0RH6vpat*E!8)c@klbN zp31g-Y3!<^9?zwNoQ1r}RT7=bPk;At19M(QNharzgkTLf5_p&V_;nC`kKk9!80knk z4VvHW`G@wIvEQq|)@0tr=8CN=p7ZfXr+{WDUZ;w*Smf$DCybqs(aCVf8R+5TM;`nU zeHgx>!%tl2R0u^W{xgonX0cLU{`mwN)3;b1kvEa+nPL@#s6HU)22NauN5vnZ?kfZr zpt>#n`sh+9YdP<;`26gSvih4b_3b70x1~PiMf-h&!LNtMDBCC~4CAQ77`vUG*scmp ztsA!vQT0mm{}*##;TP4n#fvDSprVvRW6(J;bcw(q-5_0#3@r!}LmK1|l1j_ajg-<# zH$yi{4=D|U1Ml#=_q}^B`UgDyz|3dO+H0@9_NsHv<~!e>A-K7-%L=WOzzQv7*Q{AS zaa+__>7LCH7k?~jpf$Jjp+B-{cDu=zP-_qSUOxNGES$MED1igmiU8i!Q`qWDeQIcO znYib`FUr|TUZtsTnKlDtr#`NF_f1+V`e>@|hHkZ9_niRyl>NM0>JT~i)_=KoPk18+ zF9=Mz3t(Y?E1-#V6C26(^N-|t1`Rrls4<# zbK|%8wqP+jeaZIgKPK+g@tC%f?=R;33S?uSuc~_}|CPeDBFW7$w80_SQ)XN~epQ~Q zkI7Cz2~hg`G$g@&-CJ$f_#~j$5z!5G{QcZSSCO;zns59w!v^!b)FAVIt0%ouLj2xR zxHpQ&KNdZbR&r;xHnY%G!F?GX!-Z^E_o2?ivnavYVUU9To$s?0Pt6Ld~c}Ckx8KQ|Ab3@8|>Bkniv(A>7%0tk_pK-^F(=(l;t~ zh56s~WihJ5eKfUiPfX7cHK9{DJ3{~%9IG#tb)_pOtj9F{_JkrjEmLd+8P{*Qa9v86 z%PHin#S+!}eow+my!wemmnZVFDU$f-ue(tAW^a&uDX3(qJXNlnPp)CM@toQ!`ni4C zb-A*~TM)^-+cq2TBYzBLR*-CJA{;;U7&K`ve6ISY=HqiC5g)yi1ba#Lvj^F;YcM`v z&ooauYq9Ey><>T}v3Y$zI8f|ZegU~-Z=8dmmM`idtEr}^byLwkD;QlL5h11_6`e{> zD-`KxJ>~x*+1^w)gyUwNV5N(QsZ~optD*H!D@fu6=Z#BGP%Z3BnKrNZxGy2m%}Qa| z(0iAOldO!+^H5gr)m%UlqQmgypWF={f?v$Qs`s-Ft?>J;R7qiOIz*O$D3x*w> zCb$!S&ttE9nO#58V|sH#e_d-N#pa_|6(Es&NVVRX z=`0c|P`6jFb5GDWh5|uSa1T1!_VwAzIStf|d~xR?6^@B_daf&Eahfjm``OGicGWe2 zHkOdvon+z2b|0ft<{_Z8$iZh`S2Ob&oGiYstt!wWq;XtqRw!#{lokQ9XRtEgjP?Oz@a^H=t6oTQA5v9uUNC^acPJZHn2N zF6Ifx4$5!yu@Q2#-uO~`c5UA`c6A+#$mJL-a!_jA>>l8$iJU$4jKyl|ir7rI0#fd| z$$JlcJDgEXNjp{&tsjtqi6s?Be?53{dW5b;B!Q7X82lCEXI6CAS`@d?dM#s zNJ7=@_NJ-E;q#*ziN$5T^6=G!Ym0MmofDzcmy@Bo^@!XW9{-zec&)jpnd3yno96b)bK=Nb`Pc5W3PS!cDPyr*3Qikmu?hWp2X2Fx`Exq=dY}<=yhXaEnA4)6ayJhugJw!hcX=8vWU?WG>$1d+mc&Ahijg|tPGo=iy^>Ca@0^G$e9l8sv)7=me{ku``+DYdFS3MDptGJg!AfGR zy&ll-7H&Ko#mIDtro5Q(j$1q{-uk!=(O6O1-(ks7x!mUl!d^ve!YGa?toM4d_lGe2 zwWWSDkjrde7$g2X#1WpHQ90shL89#q-^oJ`xhys zG}(MnGaNqiUzX<91TjGkmth~bRqOBk8kbBpx=!pE6&tD!z!5Rm+&Gq`W|j>xRd|BD z+R&IX2mj?p?ke<#xjBvvNs21ef{*27Tgf&EeTki7r9t21JmX!DQEWc5x2B47%=px# z%z620GrZZ2b+#+%+Yd=wZ=@&t<#&8YkUfnRvR>Jf`WVyo)G3$eJIX20nZ9yAvYKRuof9*41KsgV>N5Dn4z+SWQV`qfy z$~zB^can9zP_IlLZ;NtfQ&3K#Vs7Nw2-8joJ}r#UI0j|j z|Ci>>#>{>**3Cl@>-Lv3FF*asdK9r*Yq?%h2 z(3Y~L-|CZ&=AYGg>N<2jBP_$vW7qD;Yadybn-UzhC#s4Hp2<-H>d?MKLi-+7)M|A| zdW%^?L?2u1<; z;40Z#0jc{{GWA%lG}2^t!=T9J9^q|NCI9Gcn$k?EB-nKg3-YV)EXs2`(&h@HS4d-E z4{P^KgV8_q3F=~!cY*u=)B+J*Z$b7SAA?s76CRF=hFP$E36T&5PjuILMdh=4(vYGS z@d0=E7}pFhQdPBp{1O|F0PQ;p(S?KF8RUZYW2l|ERvCMTi!35kk%BLI81@!%=J)TQ z)Hx_n^~?oEh_Mf*CLQrL>tBHVQ8bC-KD!squ-+Id=4favi1dC&6AfZA{VRyA`j8% zJ6Ih*C?`{+_OKhg0>zxsKSH*iJ`4^U6CflhM3d;f7P@mMqK7kUUWXOxJ-@|eXkcLU zTs-^eBei})2A^@man##02C?j!YDlm(T@?C-W{+5Mb@8N&rkWbt%hiZ@erPI-L|E;~ zcl)9GsHnQ9tXqH%mCtxabh9eY>dl|JW|%7dBqK#Q(q@o|o1&3G z&*$Lwd(H3#wWzV9&sth{WDdW<)5mA~aEE~7M;u!1pP&|z=(B;8wo#o_>jyMWT^(Nx zo>p^h+`0#yw;^**>^o>_ck_X1V|?PK<*$v&HorP~03_bsR1Rn#Flw?bCL5+8X+M&@fpc1?EN-BSjlH8CTDE5g!%b-#%kdy8N>ATFR zhxZ_a#2M$1os_u#rb+-Rf9Ft>d%-@!EvgL}&Z+-AT;nTmq-|ZBeKUMACP914_^XK4 zT8@!RaL#boajjNi%;~OlGrS)0)!Xc=s$=!X5q;&9BJ79MDA{-wq~KgEKXmq?WVCOA zX?U;aS3M!GTX|EDs!l>9*Lxi8kpX8!p44JhK2Zy^mXln!cWNY|5;uuHGQYPwZ;_}_ z!Gz)LS5Ly6IpbNjYH5D=ckIGn*}8Rbr&oZfeRvk_s0~l6z+LY|J%LitYCw_DE{m)T ze@U%1cSr5?o*KAL+1Sceu_%DRjjsb!J1H5@={rFK9DYdg;;HhDYKFctbW*35oI8T- zR8SIufvl1=zeN_|z9Z>>KL-RK`qhlht_iL)JA~kXIr==r7DzqEOoi(bzg#PQ%9B1W!mh z8iTR-k8{bBl@DmAS+zK^DQD_UQ~Yx@)jC2B9OGTHtDp)YffFJ(;z6Acn;F0JH-A5@ zpW;z3jE7Fb48Gi@?=8YgE$6B&5|<+$XcgH@Xu47$)I8-EUxiJ~*?*32mOfj2-@%48S20nRkk4XLmpq zi#b&R;+4kl7O%QjS57V3Y`KU}s&zD*cv3KYqkZ}G$9~qBIJ3Re-ob&U{-gDg;gVOW zO0#c#t=*&QPEeEaS4DF`6>o!li;p{BQN+C@^bS)x^+&|54mI!Xu@%+zzty($1nR}m z?gv$JM6h_hRDxC4_M@%0B|>#=tDd;-lcmwXbQ}-EWP7HonmZrT zv{fetsj<{9C5t%?eL9l0F2bIi*ouBGYzaBW^MYC7a5O)0I>}+fzH9c%AU0IDY0|5= zL3=EOl&GRhwFd}&1b>UEg{wyQ34+@tj&12ZtFI#VFy35M65O6UEcAW4EvDW{yMku? z&GsWtx@8L|i+l^I-PA%&BQH~@{qVr2*l3wh^knHAne`@Yqlf%Vby8(h>g#P(LYM-N zfcXgSaMyv(h2`|xSPK5bITO4y7&e;iu(eJoL3jz|nq9xMIAU1>#)!z8KcIxu4vo0r5Z%6Nf z{ZqFyPz&LFht~sH(Kqj~YSU6j1F3X2$eFK6y(2O`FLCo81$h&3Z+&X*BK4Z_avM=ib#@O4Hj<4O4W~T|Wo%nQi=quu7 zCYs8N)y9@x4oAm!1tP5T1B8_h5?*aq*LTc#S$x6oTgB_N!Of#LnI@lV8&c1}MR_IY zN=T{YXJI0r%!jmM9LCEO>u3gM={%8F?GwDSCHK}Eu(I5jI{mu5xyVYq z@^WZQ1JSqV(%3yMoZwYjOCb}>+FHA=-xB`_D`Qa}p#E8CnB%d>XkO=benB5P^48VC z!Atyy?dU3fn|+wG%ksxPd%ZjIK3f?L520?siS|}FdIsQ@UKf7>fQNirNP>>{rCNU9 zfS#}BLwVbjC$2(TcztFP7`0$DopaI^9qX{)9NEv0bw-xpPo%h-cKBa%PbXy;pJqG? zAL-TVjYrkr>l>J*e~iqkE3L1wUOOgEf9faNHJH(Z_A_i0RQXuPGo|MnB_U78V^KuA zuS%Jt-OeTY^0! zFil|?TKj=tjG-md5v4qC1=RBkl6#jDJnLuy5js@8|1ltZMrk7bpuMa;!olNSo#L)S zw8YFt7#b3sJ}t$3YS&d89*$mcWykF(FTKaNCp4Hul8_s4r|>Rqpz73)L z!bFV2WHiyLB3pI1wNLKaUpoc=uA;C@0L517K}xh&WE}heC4&ybDCGCMJ|R}R0ywI5 z`rqlHNqqKC7fZ`bUQ1gImln-plCR%N?ko*2)9&su z%58mpK80RVupF3tdMWmnt2&r^!o(uyNwcJ|t?TkLu&47tsOMV=8vn0U3CH8#>ivk_ zkSP?x-gd!|6@eR)m@({-5}{ZD=%x)1ICfpgwZMLDttO5ciOyk6e#=9Xes=Hr(UTI- z8MxD4b9qzKrezMT{TIXldm5>yHG@A3-Dbaq3u08M*l8*Vsuk&QRd)bXM?7HN-sk*6 zVdw~W285sS3B?qGQZnumBvErTr>M9xHZ{^Gn(#IzDdPdX2{Pfe3rLEy7KJ0KQt}SA zbg6Jn!dNTrY^%AjFhP~vXI-3JU|AFIPCMYFLy(3~LhA8v?(UXEHL07iE2zD@JZ|!2 zmr3CeCgdhoUAOXP?M@;#r1*vS*zx94ub3Ilx39gwAk!wdNV)J=dHgu9u9p~nU!x?K z*Q;K|JqXlEh=1{Q_TyR7yn10qw6U`1{t0G zh~(USWl?I~?rfeV<#zOxrF@m^vX%!+$IoYxSoDgdovb0E8uNSQE=g7EO7dyP5620h%232oyq>J?g{^?qG+rFo@ux<&$5rMK_Ayo#B=`(gj_ zNpf$2!AG|Q+@@#p@oHHF14Fi)jw?}Is74C!65H%f&)D5hU?N&F>u#+qiN~cVk5%Bha-v$${1hr`7?YAL81TG76ti#v`P$a(MfOgi7QcRjekjA66 z#|0w|HrSnhatrc)PZmmfQwT7S2uzfd94yi~g|(DJ;@cFrfM7#|rT3wi>f$Ys1LB?; zz(_Uvy}e}ulzap!mD#=c&_>Q_WWLT)y*nk`}qtwL)oWsvx~^)sFU z5%j>}3SPuvAZ-g7FchH{K6f1$m)yW8za0c3z^^?Jh=@YTzXcIIuH9}~Nj*d&L@AqD zZ#PehcOIcp(nx&s+8)%x3h=<$pLcm?2`zQEK+BmtNP{ub(HWOq)t6BJg!~Z_|I_bj zW#7|UYwZ2^3}B)wVmb?~ms4*6Z{O~^jh-F9gvt+odI0smcDm_F*(~J+%f~`Ey$ld zPDT0lnh1E8J_!L447g_|+{O0GDn`yR@hr7v7tgY+6nz!i@E4jlpJTvo?pi+5IO7nfO3-?ST2vk zZ;W1E0(4(Sc1qxhe-@0Ce8>jV#B+b)Lm7nv%;XLLm*N$?)n4Di9VW37`1#SFgkf<8@af~OAlka!gY~_ei<)@y8{=|Dku!l9-|W6r2^0-OqeciWVn9@Ane2pz@kS|VDRkXP>Sow}Q=wp0kOrTjkfkc+DvtqILS;oX^e&Jp2rgm!T@ zn>k_hwd@3}l?vvJy!NNX+8L zr%r89uW_V#?IHy{97*UUgyW7BBec3|v&yVVU?4GUA?AhOH}lR7-D7 zgpzOYetI$?xZKB+&`a>)HtOi)nE3Lp;S!bWiTC47dZpAA2OFx848+5>%l$478Jlm0 zW!Uc~=nmi-{aD2BKbc&fMnsk4W)_ZCeFc&UugOMM_SYQVRcX5-t9-R3i$*ljU}fLB zTQa>JnW<2+K_)!`U3O%RsU#;}G#_2&XnkKf2%M5)i>2_<$18Vw}%wAzfHp_bMWSy8e)FD43=J0 zDBB4#{!Ow<1UXpOcCuX{W`o8SB^+#0)}GFv3R zu;;9#NlSs#ujJTNydMh-hohEtB=DwAD0NwJ%4%eIzjuG? zJ&DpRCG=yNt)C!_Kh3Zp!L#-8{e0_fdK*2Spw|(MGiu-wa1yB~tT3Y?q23G+p6FOP zY$++=U5D2%TFHo$l_vuZC=p=XoXPQCJ$=0=mOpp7vB(kIq9n=x>pGr|QU3B|Gzw3B zE&tUcNf9Ev3Q2h4lMW5M9Jj)xG1+i@za|QRs^PO%ozSj<_u}|*IQ}DYV`4Ud$jzY5 zA%!zD0Zt%!4KJ|nt^wzas-AP69%dDj;!Ui*c^)UR7J=`V_*`{RXYlTIJgx(!_o#Ia zd_0HIkgJ*j5|{iY9nY=}^tp^)_M0R;8&a=cyYwin9Y~tM%hR_p4$OFw?k{7;_wg8a zmRQRkUwX9Aqmdwg{(_EBF};UZp2#CFoBgcDIPq3|z0A3j|g@k^rDPfx{g@kw{eZMqDSTEsy%&tjcO1jk8 zG-!O5iCMescXPN9n!Ui}QT-7lPL!r&079{tuE6k1q$YrO&bupZSzAURLQ zMRpE*hM@2aP{4gQ-8Hswe0bPMYfkM5SqoYh^C|`6&GpC$=uV?)LQq_zTQ~ArUX07$A?Q&@$yT{VIM=|P4YPjdFb>|u17(QIY3 za%cpx!4oIp2?o^nF?65GY~AJfY8{tp;l&$hZjQZ;;F+C7n>ICgXJ8f%%uXRMCFo%s z$gTD6G!@NxfEC@f0Z$n{go)GY>SW8Q6xa2p1@GkSGpzN6sVgr)q`YxzeBdUgIMLQduFLU{IB1{&L&p`AX2F}7mi+2;QY?O<_3 zewI{V-o!BQvO0m5!J^xz)_b|y}bpHNYQi2KnAZ*;y<@s0Z4rUCL^I0@sSN!e=%Y>|eUK{0R zX&qBTW?=3IzXsQyxhuayDqrUda>z@*T9=k5qo;N@8)K*xhcOxf2}C@l$+}khWw;Tl zvqvEA^E>CZ@id8iB$R$+w_D(`h8tUhMHATEKxP2ix~^5+)#7lPY)V*o&+TN@$T!;@+{>1n4=o1hero`u z1HyfHoKNT9|GMNd*pMH*tf_C6mUqiw+SK}mv~nUU4s`R3y6Gk4Cy9xXqu!&J0ek!F zM#`NiDONHR1J#Z;^t~$f9oG43N1y!~u`b*WHbRd3$CEup2r@5F6sAwRIG|~%O;>7QVxw)C#4W!Mrq;{+ z5Fjp+E5`gr{^lNExuU&9DMn45^?WQ3jQfZ1LoN+rWI4~1tQHMU3dHcG{Qj9t7Rw$9 zjW)Pl?EN&wWS^Z9d!y|^ndnT$lHl3{C5)&BH}8YyCAt670uUO)!>^UB9qP zhkjlX7-)s>eK8$`m&elDOG}F+kVmB3r$}IZPNB|v$ zTfINtY}6?ps{9)JVWgP-?s5Z?Wo8DOBz9v{oy4mBB-9F|^M2#;s|iZJB-B#?-_A0| zdY|oWm>2Cx{3qZh-NWHZR|?FkX2n3VN`ia?JuG% zz^gOk;mNX^*E|#gA6Fq9tLXwJoFJudcq_;)uXY->H=IQOq#?>)Bkrz`3==f{tYZcx zgk+z}+R(!Ae25Od=Qt+Mo{k`jH!4~A3eR+P2H0GfNW>4z?2xO}W;=bOIohM|x3a6U z^zSw4hqUA%Bw4+cua6nVk#{lekIeFIsArSD#M?ZMP9MuGaAzNYS!xS3Ry8x_=IxLB zc&?>P+{k95L~XuDaZV@sRq*CbInS|?RdDE2>I-0{7i&H^$A7*bKS~VTkKvSA3P2+g z-gb=CX(Py}v60qa?slE%u2bKNbM4OYP|WtcX7&l-s8`e2F{l;FeSu^jkt{hqxD|Lf zd0j0au3_)Y%xW#Vz%-PzDYz5R)UsyHV)R7`Qp=QkUyN;fIR^{2*1K$m4&8R;qX&unZN3k9E=;qw(7AOZoBb|o^U{&aaNB3sHIN?OYWN=N zOi_7lmeEunlYeLVN=&%MEFPyTRlfvx5z6NKHiW0SaGyPCWdDPeu--QiPFl~KyYi~S znyAgoeFpOJNsl+Rq`tsN8Y;!MoMps_kKPWpC@*uBWBal5biB-D_No6%`oRe2(H9@= zT(Rxb519OMN~oVNKoiuABPjk_UcTHZ9dRhfc8w+_@F;W+lDux=1kn?Vtg#ku>L6;( zLC|Y`wG~#*60P%LVd6je5{tIz_wZ^H-3qWQ4t>>e(Pec{_BhlsphCcFLlYl2}P)_LO(N)`i&{lOq9DV$v z*J#zOMcXx5G;{f(pTHHCn%u!;gQmyq;S;lc8Wre?38Bp?lk0cMZlj`mNU}Euj}P@T zwiHM6a`g>l-=n@aU$NX;SuL_N)|n={T`7Qdxk)=g(Y5RBHq1ZA7QRT;eNX4SI>Z<#y6%hakF!bIkjpejvxbsm`(#QB|K_tu{RA2}iD- zZD}E^g-DOb_A)nwl-j~weXkR1>0AHwX*;RpdiBn2`Y2?CB)j6b=o5lRmdJ3M$r2SB zDYG2+0H3XPtXzJf!#~iQ*uKLzFdfOP4;K zK1ay0gva+qp&noA2Ogb7rNQn*bb4^*<&~fGRk2?6sdnyh*UD&HvL?8;o8p6@KuHCm zId5b(CP{<@tbTN(jLl`=;P(75qv1wNQ~m3$UQY~Y^C9n^UCqe{$f9>s*_H3`OO4o-s76EO)Kt6hMl?)bSko;?^}s##4DA(+t|F~qPsf2 z;%0M;;JF9wWZi(PzVpph*kzC597z08qS&p*?mm(J5d ziqgqLY`AX=U|?`6Q>s=VOj&AXqx z9alG0fjuHY9c7|*AJG=lg=j!eZA*Nzkc&pI!ng2Pp#&Nr6=V~oZrt2wm?`xEgqsF@8H_VaN+Oev zb$si3)bN?2f4`&L4GJDmpM?LEd@Ts2FpaJ(t|zL`9hurO)#z!{tXkD;1)$dRL9TsQ zVfIlkus?Z3Y@wYoU?C1xV*l`lkr^sEI9y3}^8wtlTyrWeN5(pfSHFR!MGN$M@i`B0 zYO22t9=px%A~AmO3BPkLXa2h03`hlRj7Xf8?{Cd{&iaPWZ?(ZmC3T_y$u@4wi`!LA z-!+N~?76oGPACgx8$C&*P1{&e(wG7DT)S#N8GLeW0yGRv)U#0Dr(_`(k&5l;F$NBG zePRgzunkr};H4sbz>P5=&cjOS&VYs=+AWr*H?ZpLmO8V2o0udmQFkU4yNmL6=HDFs zGyzb1Z*-)EIBz8cs)RpUlbc^})a{k$5<6(_% zE=Z@Ypfkh+Pyu%PU}IqmIG1k@oHdSppwo(ceXEeeeK>-2VEV4H)3TYxP_3;CyZNl@ z$R*~iNd+W*iLMJc*QBRp;$xBJ*V8i0CNbEcRTm~JW(Tp1uHSf7`4HN02Sv56hP>LF z9&%(?ed7EKY%_7nnyK1v)jm!MtBHwg&OxZ+0<(DRBQWvxJx|3n!>;D>(I{@16r{4s zYY}rZ&hm;>Tjqa=HpLcu#NdQCK6pyQ>jGC&B!NQ}$LXY~(jfFhhafcVUQ_Pw#{DR# zBMGpf=ryBr#7`6bZsj?8R;f zH1XArKYzQT*euN>drFZKgyzGyjLB3G{31WgpD7vOX;p$$d(D0SHBr;?C=C;|YwYw{ zC+^B?%9F}_&`;j9@y@P@Rl#33CbU_WUS`&^0p}tDD)-7OfqBB0)<6O?y|0>CQCs}_ zvf`b=D=AHps(QwQIS!%FA^C&aCkYFuy7^fZ?bTV`2FkCxmn5f0gG4EGwDj(QNPF+7{)ExKy$dpd6B zkzB@$J+BmQ0j=msrxf-m?q|@Y=rw9%V@o_2U$1B2ygZSZpd0&rz6bkThpfUY07mztmiLos3A|YP?8LWg)?!eE^-~n zl^h{f0qwBzcXn2gY%h5d7O1NVQ|yxHeN233&*Wl-6c`&Cjzta?DwFJOpl~N{y{5wG zINJSeU9~DF=CaMc2yUp3UR2Hfx{TC#8^`{t{^?#);wL9xmU+=lWt{StNKgHAj8R*L4ouFkYS$P1<&*u%8!A8~&Pxsobp{RbpXk7uqdgC^{ zEnHlmHKprxB7ZBcI|4>N0I^-wt5)-)6$vJnn~Ktm24QOw zkV;@a%{qN4oCjKQ;s$b_FWuoSFVD5%SvUG-#69N|UCfI*uXeQIqo zHtvPCc44D5XNtG>FF$A4YSt&_ogmb~c`0LQd8k9TiI-UuJ87qnPrawZ*D#Ld()y#b zXcM`n!PN%dBS|rYRb8@CKHugJf5K1<*V0Jg>x@w&x3_>UT|p|2YR`-p%`-NZON(2i zaz%jl@ATRF%>hXlkHBKjuYqD{NJ5CmIy^{qf&-+sY)kUos-%QcbpikL=Iu5RDlWjz zLm1wNjw|aIc$g51s_L3GJ=3!665Dq?%<0(4K`^tbpfDdMm@jv{H2f}@OiQANKa*{! zmf!Ht@JbKI&3kmlhfXnC2($lnryLVV%sH%U|zdG_@Xf?++zZuc^=x&v&Gi{eFg1wKCy~ zIY%w8JO7^RcS^R~Qac1@38*Qta!?r6J*%Io&r=H>hj=+4+@1T&!9O>%tvZ&< zZzwIjC6o6CcIFZ2e38ANi1Z|Vll{bh3_)?I zNoR%mWl?TK+?}JiwEYS8utLgAJC1 z+&Y9r_zjs9r;Qo*Ct`v$4%5&LrbJ1nV==jT3LD09uZ;xM^|uRJFhFb`}N)KmwHYO zH^HLblrZP=*>wW#1hG$ILwDNXA2rkx{N=|}s5VBtz>7cefiAn`ql7d+TNG+QcJ6n6 z#>zykOPeTgsu^4^<3u5nN(FQCRvv|)f?OTH?;UPcty@WH$~kK*ex7!dQZG1%c5Z!d7>!C;KJx21SNZMe26x2xQTbkKz#mAsn*W53nK(%CM+;yl7b+!{5q z0tmWPSE?V-dm{yBIFI+bpLL>3SJ%Huv3#}EW=EP>NgkrR#vkP!dv7g+v^Hkve8M&( zn^zQA)Vy;0&T`_j!HUO&G@d)%z%6ko6(qs9J_M&O6!Z`Wr#|~^yB5O$buD-kj^^Bk z%D^@F#CAAE=n0(U@rpnXi!RQmqEc?rvjyeqwoI1u_?wJ=h zXPwwFBdK6IQJU4khOFM9DfWq(o??Y8lJ}tY04S|m(evHgRnpb?nD&w@r<&^F6Pso4 zA^yQs%8L()KW7GeWN!H+P--sLFVnclGea3j5?5LD%3_l<;`2x?dB_59P?pSCMbq4q zquS6O5Z8&k^@fP5?#SUy2^mL3q38(kr`zc)EmaeBt7KVE_R_Mmtfb0tl556F?=+xd zu>xx7neuBzxpG&np9TR}XzkIYG#Q^vM|!Q0HwS(Y=N1qQM4l8d74TyZnddz;^2zic z(vIivuHAnP`CX??Z|YiL8H{dcRG~=ESEC}!0N=Ine%iEI)=BXW9f!I3eKw-Yu72{E3*qCfvQ)@eFz|Qo)q|0& zG-_mZbY;QYg+a=W(+WXoW1N2-1srsyWoY?)%ibX4OJoXGC%D({+s$|ASF5+|bTWT3 z!mgvFMp>*U_?vR9Jk=?W>p8plqgiFPddL+-3!GKWu}AUW;kNr$v70wLc9AaRRt-tr z&gAxH1F0$k?*@KCPU45@ILS3cld^nF5WARu?@H-yl0dNHRftxS;(3Hv0RD z&x)IyblznX;6JMAm{#gG#m5_G z5s^DPb-?0ZN*fkW-n&6XfQRN$3-NLMhvLPi$RmOG#K-rbnY9j>_*WL|<*iGPeyC_r zzQV`z7n1v^M1qI!8^Sko9dGJ4Tjw4V-iwn(6%A@Y=^i1IJ)Y=C0C(ag&t<*A(Q9}c zi-utfJcj=Gev^STz^GVyJR=`py$sfACA=RmHa61F07d_Kzeu7Xin-Ld*rcWsGN3?$ zx9@YtD0hi5Dev*wq0(Udb-ax|6>u+x=Q1$SPgHtf3IG8?Xgo^_AV2K?UK4qrj`t(O zj)F<tdJbdYD|qzYHJTA*-i?n~@Z9&5zT$UKi^k*q z__oioi2u;mr=ODukE5aCDaS4(6uiPfFcJ_fIH*4yv zuX6-&<1!wT{Pa`i%o$?VO+8gvQiOgrqf|`Zb=6pBGL67l z)({_iEap=9bQO=rz5)c9&f3qrf7Ke!WFY6CWqr6RC+400h93UGGf|-&?baHCuW~Z% za*Gk)Sn1KOj}YCJpkME>;{D>D@du+U*kympG~Vhe2RJ!_!MDm55`#Cl2`27)-nyUg zCo~9{jhD0Ax zJb1|roCbF!3nm|9DDau(6Ec+~{nY4V&XZMn@A{Y} z@TaJ6w}ga*^xU>PUaS0ni+lqp)@Xjh;HI-Q@He0+Aprf!(ZYsET%1P+YzXG&W#s1P z6%l6S7Jvxw8uO?!f_b2>4wf#~jt<%$FJO#(JTkn7+=3#!U`B3UelWk#IX_s$m`B#p z-rU*&fMEikLkPhfo}Eht3kd(0d`CM+XRR0JmazXoJN-Z50Kn6^Dw;f6aPw!@4p#r3 zOwQc}5M}7%>Z_q0Ru!V>qKkqXLOb{k0ASeuUSbuPyBWY<$O7f^XJ6dYNTnu^C zWuZLUFn1T8f55N+t8efHyOa%0syowG6P6aeBeK*0O}MN(BvXBm`{-LU(5ZU z3Bmk~0{^V*f2jX}d;O`M0GJWX|1WH^|JDx7FL;j1b#X=kU{irlE|U2HOs_x9AuPlw z^rzAQkQe`-D+E*!{*zZ2*s?#B5CT;G*9QK##sFRyzSlVAK zDlEtdIMc=H1O*uR|5O4nGWbubh!7*-9Tz2lc>#;^{>5X!z_tL;Zx{ImMb6>hE>gjK zB8&omAqxRA{$e|R0Y;&}*dN&Qzl4RLFeCq;GJs_Z|EV<)#b5xC?V@@>44!+=MKZt) zto|bPAMiIom5X#B`+$_0RdhKpOl@3<2{2 z>E@zjAcp}}?jjYaS3oAaNCoO981knGAVmP_<07-55aVCs8;BktBVXhNHtBC60j&AY zs5+0~^MJc31IUcvzof8ph5yns0QCyY`=^lefB>q?McIM?Lg$}$29yA{_9E~3KK&&- zo_q739(t}E(Chpm<6OPJ6f+2r!v0bLfK2e0Z8&_1Q8KNK*Ng?0PFu{i9mx07WmU3=cy40=8H0bB*^=xZa^OaM8HK}!2Vz$ zEnOr73GlO{v{9r^Mn5u z%s}D>|Hb*v?E$na7c~J)3jRy!0}|U`TF~W=9RaTUbmrnqwMuK1w;EezqkG8do9gJ1R(H^*DbYYbF zKb>85TvS`TJ_raB(jg!WB@74yLk=-YN_PvwFf(*BbUKK1qog2$fPyqgBO%?PAWBGg zh?2q`z;ljY{J!tr%RjUB-fyk7*M4T!^S-R@=-_}^&jEMokNL9O=TT;s&=y0JI$~P#!QBBtX#t&I<%#HZne66<%J1TftmV zK#*bP7-rGxpl#)5?*a1yf|QY79_LFJL z=}?T~6A<;aSe$~Ys_xQqpPIf~Q!4R323y!X&Jy)w3Swbp0{Z(M)BACM$T|zs{aBYA zZ;7=-We+t4IskaUh6+ZsK!Otk#S!wRnjGE7)6(Gkvu`QO6?WFzcn`eTg*!YJq<3+r zl3vaC&xj}8=sjl7o8O)zCP<$9195XAS9pJ=Ih%IR#v>0Ipfw2 z#kbTVUA&tEJirMsAOtfs_Rx5)Z{JgENuNs|2YkRi$`9pTs@8hp-1^5&P8IIf08ir` z60}|oA|gHDA=Y!EDDtTIpQ2r5k;PnHMBe@HVteNg2pPKVx;bu4SPb#SFF+WBstacV z2=|Q5jBkkbDFy9gnqp9?fufm?AuEO7k+J2DH#E zZbb{`b^{sjp&5SeM=H)LU`YghJ+|1N|BQ|g+Au)H!txW`GIIb8MDHmw!jW^h-r%40F|a$#UYxr@QK96L}s?;PMY*?q@aw3%ZPeh z!slFT&;%qF77l2Kh{nNQ-@Q(MgKiyiZ-70t8$j%Jjk(2F`a{ zvrGXts$LgTpdi?pdL;Sby?!mdEa4(1@lz+d14Im0PF)%N-54Y}lL$(Snr#ZnN zzIcbqH0~OfReTGNpE|{93t2n!5$vUt-GkvvF(hg&OBx~jE!$@KEE32u8QIQQ(o+b-3Y<#D=F!sRt7|0WaOU+?{SeHC+3p!=nqqq-^1liMT>HU zHa;}VA*I{X--Cu$C1nin`NuU55313M@wFO0mO~b>-F6Bjd-P`fwUxy~Y+1Co8O{>( z#}i>8szYP)d%>r~eiXg8kGNM>sMlk7$)LHqS|HmJtkj0+oVZx%;`e5H$}Q7-*0sIs@u0x0Y|Pmw$-%58@S4S7h?DQQNm# z*7??~Nn54B^u;r@Tcn+4J_DX3pO2y-8fwKT6a%bGPrHPH=gyA+IuDkw<9D3qH!$5A2n~ zTWKF^IpDFNcP8~yQ(`D7v?9A_z49$I=vLIp6~Vf%k#s`M|VM^-(0XZ4OYEU z;T&qRNsW2zMeQ7bOGB_xhZr3PUE{h4mCsB`vd~P_CDC?AJR^ zRhUe$>Rx8Rxv-QaJqT+ZRw3rh(tM*Y8YiGO=AyA$Gz=#4N+wQ;HdoR}9X?nJ>70EMn)Umdond8{d==C%2$B6TE{VC?(=jt0* zesm!Bztq;8%`s8z-5u|3C1sk#o1s3yETX+SCf_HQz36SYOmpyBQ+9&e zGWsh~XT#?z9UDl2T8S8>0G*Cp%Gm2`v~^rViey?;dg@Bmm!TS^(#`$sdNW$-)(xNb zbtnxSf3c{XtV(lGdiW_I;H)~)!K#WwLY6K>jNuGYKhVQ4(7N9PYS(s za7GVIs^7ic@=w=#^F(;F7;~2YTRwjNkd?;I4d=sw+%)o{O8DoWge9+Mdb6)QH{`?* z@cH)O6!rOboqJu%V1^B=po98V1Kmf9nsM%`ITA1R>0dYxjAjdggcLs;kC_vbTden# zzFy!Wc0ukQ8caKEd2ADOO>S6>%k!_6NWU=bIeKvPY;jiK(zFa+&>kyuFl^cdOv*zkRsN4V}9e~s>3Dc4B_4do7+elPlG0yN_o|TB<;xmcykg5g!xo@WOSlJ zd8dxH_o?4HM==-_K5GmIU+os^_X8UgeDQ!o`?7V%(NY5oM_*G9MS1x7bINOD`>%$i z*;1apb)UYuR=NITW2x2_Q8UG;zdKtnJ=OGcyYlnq+kN!nLg}l9@+i&BF*!fsPWS_hY`m^J42@`sK#?Vfx{+~EGPMaKn zDA#@I=qH$0qWhgW>#7^(T_^ljz1^uiP}X6NdV*}0pmyDV*B>Yt=driaw5c~3HGkZf zx41nTU#<}n>P19G$izZj`(2i-tt7b)1;rlDA4ZkffQ)fK#+2Xe*C`8(KAGFR&^QQ$yhq=d9I(AvJD&SMGH)@)!X5ZjC}4ACNM;BZ_s33)(LR$uT+ry*NWnv_+BG@;(M z)*@iqVH^6|ye@vjzKk%y-NTK;bTx>qvOoH?R2)F zPxwge?kH2ETI1)+66c$kP@=c#fk)93z;OlQ0b=UV=_3aZ1Cz0h4!4Q4F zq>b8UjR)-e8?_s&KZ6)*YsJ4J8;&A5vlLhT&#*{R9S(^}&uYa5k9+2es(R)mgTmcM z?B|vf+{ir(1-_8FwVUn0(%4 z7zX;W53p`;Wj-cRGRg>1zT<0rha>^QL{-;O+DbC*piiRiK~_KH|_wH{G&0D@IA11Z6NSG3xCH~@0>zE`wD zXBA&?fi!8zf%+$}pQ>e46_= z>o1-Ad+4&DV&kJ{_m7rnS#YMk6i$t{t_CnZTeF`m#0$vyT$DCF8MBOerK^j(TSt7N z%sZFrf_?6T=_=cy_pRKA227;nb|ULh&9dd_km=1eb=5NK)UjFm!vX5*>pgUX9nVHX zPQ6+e1Q|EP_aaUaZ;O(mP?L!AjZc$!5XSi)lk%`-q@A&BU{-naQdDx+DN|?_A&oSSXZu16^D3p@@Tm@Jt9xT4up$=Yr@CV#+%S!sd0v3TWM$`_jI)c;{B}u*$6TJq-dH zF?4FvFue@Wyn`&U;cnBCD=1lRm+ziM2%*yx%Z)cR;sX_hOyOYbD_`R-P%GKrcF<#N|UBaj}yaz4JQYP zkR3w;S7v2r)eLigo=Spyl4vc_Rm=ct#yqEt_QrW>5D@*Ga&3TObiMY4D`%A2b@p4m zP9Ky#gXjBdr>50$x1>KU-)MHt3s(bqLJQ!$2)CkZqbu9}>tb21oWZow&jlm8IS8VA zgN=t>jU!3}M^c<=;14IBKpK$st#q4rhmkdoC4j|nhnj|_X@?^)SE}@>e9sES_=fj0 zU%L{)I{P20rs@RQA#%pM`Q5i`I%z9sV>rAc2V`z!hJc|YNMFY81Bt{MC(1=GY`7q} zZf2S2wT9IyzEY{SAUZX3lUBxhbP4C>H6D(ULMRR5Yt8ctjnP&TH&>UDsjUyHyKmvH zcJo(Ht4xHuLJod7ip1=aufk<+G18YoeUN@Xx*v2tFiCj3+6D`zp_2mE58^9ntBCNC zJ9mb6W$KoTZO)%oZyLFH<(oqI9%M~1tsXBXR-*4fJ~PY57rmduE=d=ddQBlMeCtu6 zI{pMKpuS1a?7;0*pQ>a$`I}N#rL!{r1F~NXRs$UE}NsW>)~sL(D?P*ABaN=D}nYD19dk3ta zqyW$fY zX)?!QUBPqys-~2?1g*t|Ri?sg_I(5dsMx&@Y=>Tyl}$~QA-8S)XF4WaTF1^v5d=;2 znQ_9Bww|(Py8!z2;=L?5XYd@o6p&#WU&(cRu5KW`&2fK&U~$Q>F75L@v#3wrsaBGp z`Iw?U6&1#pW9v7=>1?w%ll(?b#!sexzVwQ<+be8J;o!x-YUDpvl-Bg2?(~SUk>!|#4TE0^2`b+I`J!3udsejCgaV-|0w#1{(8$B ztXYNtQB@|jmm#iGv*RdHG(+@{$E{Vf2ezVqbp3jdN1|tSV4`nm`rCsAEN92{MaihH zvjzu=&(0`|z9)<3d$fA0;mE`_Bctk_PfqfrBZ~ou=#%**Ht#Q^B$PudBhQ|5tA#OO z{5FZt0PI+J-S+WDZHW*05A2p*k;0f3Fq`aq=6Hf;DW}5bIUE{NI~P7Pshg+#_+xcd zrxnFp4Ftpk{t{D6r;W>7(W@Lr!+((SMv@2oj6blAXq(1nJ9S~)@%R$jBq4voe*5c| zKwMWJ;45f=G$1ATV2!=abQEyfg1w`UA9^Pabc(7wpji4k5Eyai36DC19d!VX!AX|! z4xYh#CN8P~BJGnQg_LwY%78%ilS7dm%lGqP<$7R<0$aj};8&7@&^u6;)4L%Dtkhpg zBs*WxLUph+$2@ypMIIGa7@B|I?&0!g9}S0UHUaa(8#Jm=da<|i;)~0m|6%U*4erL z=*4F;iY6Xb^>wq(Ufe2)QY(rk-Z(`S5Y&vCsX3ekh_oPLZ-#T6;iazn2K{Sgjo<~b z{fh>?Wcb`rcGeC^*Iyi;;4ju%1%@H~{ydOzbaTMaf4{Sd=gjJ#X8w$nvArJ_N4-5i>c`y<@u3io(m?w{eE9-@gUsf=K1$VJ_I;Xk; z=T>3nj&7*)_%DM(!2h!B|rP<%*n~l8h$~{gD{M`1nBQem|;2p zI~PWs&qe6PAW7MwE*o~%=TVB;!5l0lSv+|VNDG*o9f}3|zo+meK?3xPb(ERr@!wf6)H* z5dL@l-vxiOpsFe+>j-x{pRbCtP&YSggcuS61H%LnB0T&sI3JIoh@c3Ms35-(51fx5 zX~74x5HN>h<`$+)|ET+qMim`gQRlgy|Bun%bT5ov-jLs(#fzu>V!~k_a|~F3{z41P z!Czp5DL<5C@i<2lOzT%h=2vy2;6>u>7n=PEWEdp9%w3*?pDW-p53dSyu~l@iaKsex zoi{PLtMg(;Of8`LuP=qa@E0>xEjf`6m`oK$0*=ygK_ayr9Z?srpgd2<=~*K^kS-c7 zNDCwex8X=vj7eE*S0_7|7v>IJ!0w;6FTQnTFxYD6X!$z?pXc>{1+6H1UqVna=aUEAa`O0dIT%m9;65=o8zcScEle&WjucIcAm6SOPgp={N!k zIfW{#a<&)bpE#{)UJROBXI5HU6s;Vh&Z;wAr+hXgr>D0X1`Zn)wRV+WN_Y80v0YP4 zu3RSV>eq|Qu}3Ue&?L#bkGV{t3{aVhJ%+`;HRNC7W^24HO!-7M*q+jJO$GjS+ z2AVKQ@?)*BTI&vB`HR^*=coWH-T`{w{ZN>wT@DT#Ck5&gYbDVKvn$ATzilJ!mD0XT*$SGM0nsomd^D9Y06jcj5*Q;=P@9|NOAAn!+QCX|r9jn&zi7 z(^WIT)meXuKs Date: Tue, 16 Dec 2025 14:08:18 -0600 Subject: [PATCH 120/137] Reduce complexity of migrations Given the measure we take to prevent cross-index queries in search we are removing the 'rest.action.multi.allow_explicit_index' setting as this significantly complicates future migration effort and introduces risk of outages to the search system for any updates that require blue/green deployments. --- .../python/search/opensearch_client.py | 19 ----------- .../tests/unit/test_opensearch_client.py | 19 ----------- .../search_persistent_stack/__init__.py | 6 ---- .../provider_search_domain.py | 33 ++++--------------- 4 files changed, 6 insertions(+), 71 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index b4e3236c6..38adaa81c 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -211,17 +211,6 @@ def _extract_opensearch_error_reason(e: RequestError) -> str: ) return str(e.error) - def index_document(self, index_name: str, document_id: str, document: dict) -> dict: - """ - Index a single document into the specified index. - - :param index_name: The name of the index to write to. - :param document_id: The unique identifier for the document. - :param document: The document to index - :return: The response from OpenSearch - """ - return self._client.index(index=index_name, id=document_id, body=document) - def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'providerId') -> dict: """ Bulk index multiple documents into the specified index. @@ -241,10 +230,6 @@ def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'pr actions = [] for doc in documents: - # Note: We specify the index via the `index` parameter in the bulk() call below, - # not in the action metadata. This is required because the OpenSearch domain has - # `rest.action.multi.allow_explicit_index: false` which prevents specifying - # indices in the request body for security purposes. actions.append({'index': {'_id': doc[id_field]}}) actions.append(doc) @@ -269,10 +254,6 @@ def bulk_delete(self, index_name: str, document_ids: list[str]) -> set[str]: actions = [] for doc_id in document_ids: - # Note: We specify the index via the `index` parameter in the bulk() call below, - # not in the action metadata. This is required because the OpenSearch domain has - # `rest.action.multi.allow_explicit_index: false` which prevents specifying - # indices in the request body for security purposes. actions.append({'delete': {'_id': doc_id}}) response = self._bulk_operation_with_retry( diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index f71a9847a..b20c7a1f2 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -200,25 +200,6 @@ def test_search_raises_cc_invalid_request_exception_on_timeout(self): str(context.exception), ) - def test_index_document_calls_internal_client_with_expected_arguments(self): - """Test that index_document calls the internal client's index method correctly.""" - client, mock_internal_client = self._create_client_with_mock() - - index_name = 'test_index' - document_id = 'doc-123' - document = {'providerId': 'doc-123', 'givenName': 'John', 'familyName': 'Doe'} - expected_response = {'_index': index_name, '_id': document_id, 'result': 'created'} - mock_internal_client.index.return_value = expected_response - - result = client.index_document(index_name=index_name, document_id=document_id, document=document) - - mock_internal_client.index.assert_called_once_with( - index=index_name, - id=document_id, - body=document, - ) - self.assertEqual(expected_response, result) - def test_bulk_index_calls_internal_client_with_expected_arguments(self): """Test that bulk_index calls the internal client's bulk method correctly.""" client, mock_internal_client = self._create_client_with_mock() diff --git a/backend/compact-connect/stacks/search_persistent_stack/__init__.py b/backend/compact-connect/stacks/search_persistent_stack/__init__.py index 7cc9d4b44..5c5f9a8a3 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/__init__.py +++ b/backend/compact-connect/stacks/search_persistent_stack/__init__.py @@ -23,12 +23,6 @@ class SearchPersistentStack(AppStack): - KMS encryption for data at rest - Node-to-node encryption and HTTPS enforcement - Environment-specific instance sizing and cluster configuration - - IMPORTANT NOTE: Avoid updating the OpenSearch domain in a way that requires a blue/green deployment, - which is known to get stuck. See provider_search_domain.py for detailed upgrade notes, root causes, - and recovery steps. Note that worst case scenario, you may have to delete the entire stack, re-deploy it, and - re-index all the data from the provider table. In light of this, DO NOT place any resources in this stack that - should never be deleted. """ def __init__( diff --git a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py index f42938370..3a6ee2aaa 100644 --- a/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py +++ b/backend/compact-connect/stacks/search_persistent_stack/provider_search_domain.py @@ -159,30 +159,15 @@ def __init__( self.domain = Domain( self, 'Domain', - # IMPORTANT NOTE: updating the engine version requires a blue/green deployment, which has consistently - # failed to complete in both production and non-production environments due to failed dashboard health - # checks. We suspect this is because of the 'rest.action.multi.allow_explicit_index: false' setting - # interfering with the OpenSearch dashboard health checks during upgrades. - # See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html#ac-advanced + # IMPORTANT NOTE: updating the engine version requires a blue/green deployment. + # During development, we found that if a blue/green deployment became stuck, the search endpoints were still + # able to serve data, but the CloudFormation deployment would fail waiting for the domain to become active. + # In such cases you may have to work with AWS support to get it out of that state. # If you intend to update this field, or any other field that will require a blue/green deployment as # described here: # https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-configuration-changes.html - # You should consider the following migration process instead: - # 1. Deploy a NEW domain with the target version (use different construct ID) - # 2. Reindex data from provider table using PopulateProviderDocumentsHandler - # 3. Update search API to point to new domain - # 4. Decommission old domain - # This approach provides full rollback capability and avoids blue/green issues entirely. - # - # During these upgrades, consider working with stakeholders to schedule a maintenance window during - # low-traffic periods where advanced search may become inaccessible during the update. This will allow you - # to perform the search api cut-over to the new domain within one deployment. - # During development, we found that if a blue/green deployment became stuck, the search endpoints were still - # able to serve data, but the CloudFormation deployment would fail waiting for the domain to become active. - # In such cases you may have to work with AWS support to get it out of that state. Worst case scenario, - # both the search API and search persistent stacks will need to be destroyed, redeployed, and re-indexed, - # hence why we recommend you create an entirely different domain and avoid the blue/green deployment - # altogether. + # consider working with stakeholders to schedule a maintenance window during low-traffic periods where + # advanced search may become inaccessible during the update, to give you time to verify changes. version=EngineVersion.OPENSEARCH_3_3, capacity=capacity_config, enable_auto_software_update=True, @@ -206,12 +191,6 @@ def __init__( node_to_node_encryption=True, enforce_https=True, tls_security_policy=TLSSecurityPolicy.TLS_1_2, - # Advanced security options - advanced_options={ - # Prevent queries from accessing multiple indices in a single request - # This is a security control to ensure queries are scoped to a single index, and thus a single compact - 'rest.action.multi.allow_explicit_index': 'false', - }, logging=LoggingOptions( app_log_enabled=True, app_log_group=app_log_group, From 10de04e043b60c6f8369b1c0c5ca12a9a4a06e87 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 14:11:03 -0600 Subject: [PATCH 121/137] PR feedback --- backend/compact-connect/docs/design/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/docs/design/README.md b/backend/compact-connect/docs/design/README.md index 2513a6c72..1bb5e620d 100644 --- a/backend/compact-connect/docs/design/README.md +++ b/backend/compact-connect/docs/design/README.md @@ -700,7 +700,7 @@ The search infrastructure consists of several key components: ### Index Structure Provider documents are stored in compact-specific indices with the naming convention: `compact_{compact}_providers_{version}` -(e.g., `compact_aslp_providers_v1`). We use index aliases to provide a stable reference to the current version of each index (e.g., `compact_aslp_providers`), allowing read and write operations to be transparently redirected during planned index migrations or upgrades. This enables seamless index schema changes without requiring app code changes, as applications and APIs can continue to reference the alias rather than a specific index name. See https://docs.opensearch.org/latest/im-plugin/index-alias/ for more information. +(e.g., `compact_aslp_providers_v1`). We use index aliases to provide a stable reference to the current version of each index (e.g., `compact_aslp_providers`), allowing read and write operations to be transparently redirected during planned index migrations or upgrades. This enables seamless index schema changes without requiring app code changes, as applications and APIs can continue to reference the alias rather than a specific index name. See [OpenSearch index alias documentation](https://docs.opensearch.org/latest/im-plugin/index-alias/) for more information. #### Index Management From 84f113e9a795d43d29e58c9e51be58b4314451ab Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 14:39:17 -0600 Subject: [PATCH 122/137] enable search api deployment for prod --- .../compact-connect/pipeline/backend_stage.py | 43 +++-- .../tests/app/test_search_persistent_stack.py | 165 +++++++++++++++--- 2 files changed, 164 insertions(+), 44 deletions(-) diff --git a/backend/compact-connect/pipeline/backend_stage.py b/backend/compact-connect/pipeline/backend_stage.py index a6aff227e..cf897ea11 100644 --- a/backend/compact-connect/pipeline/backend_stage.py +++ b/backend/compact-connect/pipeline/backend_stage.py @@ -235,27 +235,24 @@ def __init__( self.data_migration_stack.add_dependency(self.event_listener_stack) # Search Persistent Stack - OpenSearch Domain for advanced provider search - # currently not deploying to prod or beta to reduce costs until search api functionality is completed - # to reduce costs - if environment_name != 'prod' and environment_name != 'beta': - self.search_persistent_stack = SearchPersistentStack( - self, - 'SearchPersistentStack', - env=environment, - environment_context=environment_context, - standard_tags=standard_tags, - environment_name=environment_name, - vpc_stack=self.vpc_stack, - persistent_stack=self.persistent_stack, - ) + self.search_persistent_stack = SearchPersistentStack( + self, + 'SearchPersistentStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + vpc_stack=self.vpc_stack, + persistent_stack=self.persistent_stack, + ) - self.search_api_stack = SearchApiStack( - self, - 'SearchAPIStack', - env=environment, - environment_context=environment_context, - standard_tags=standard_tags, - environment_name=environment_name, - persistent_stack=self.persistent_stack, - search_persistent_stack=self.search_persistent_stack, - ) + self.search_api_stack = SearchApiStack( + self, + 'SearchAPIStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + persistent_stack=self.persistent_stack, + search_persistent_stack=self.search_persistent_stack, + ) diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 0639e3f65..417791bda 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -256,27 +256,6 @@ def test_capacity_alarms_configured(self): }, ) - def test_multi_index_queries_disabled(self): - """ - Test that multi-index queries are disabled for security. - - This verifies that the advanced option 'rest.action.multi.allow_explicit_index' is set to 'false', - which prevents queries from targeting multiple indices in a single request. - This is a security control to ensure queries remain scoped to a single index. - """ - search_stack = self.app.sandbox_backend_stage.search_persistent_stack - search_template = Template.from_stack(search_stack) - - # Verify the advanced option is set to prevent multi-index queries - search_template.has_resource_properties( - 'AWS::OpenSearchService::Domain', - { - 'AdvancedOptions': { - 'rest.action.multi.allow_explicit_index': 'false', - }, - }, - ) - def test_sandbox_uses_expected_private_subnet(self): """ Test that the OpenSearch Domain in sandbox uses expected private Subnet. @@ -314,3 +293,147 @@ def test_sandbox_uses_expected_private_subnet(self): f'OpenSearch should import privateSubnet1, but is importing: {import_value}. ' 'This is critical for deterministic subnet placement in non-prod environments.', ) + + +class TestProdSearchPersistentStack(TstAppABC, TestCase): + """ + Test cases for the prod SearchPersistentStack to ensure proper production OpenSearch Domain configuration + for advanced provider search functionality. + """ + + @classmethod + def get_context(cls): + with open('cdk.json') as f: + context = json.load(f)['context'] + with open('cdk.context.prod-example.json') as f: + context.update(json.load(f)) + + # Suppresses lambda bundling for tests + context['aws:cdk:bundling-stacks'] = [] + return context + + def test_prod_instance_type(self): + """ + Test that production environment uses m7g.medium.search instance type for data nodes + and r8g.medium.search for master nodes with high availability configuration. + """ + search_stack = self.app.prod_backend_pipeline_stack.prod_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify production uses m7g.medium.search with 3 data nodes + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'ClusterConfig': { + 'InstanceType': 'm7g.medium.search', + 'InstanceCount': 3, + 'DedicatedMasterEnabled': True, + 'DedicatedMasterType': 'r8g.medium.search', + 'DedicatedMasterCount': 3, + 'MultiAZWithStandbyEnabled': True, + }, + }, + ) + + def test_prod_ebs_volume_size(self): + """ + Test that production environment uses 25GB EBS volume size. + """ + search_stack = self.app.prod_backend_pipeline_stack.prod_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify production uses 25GB EBS volume + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'EBSOptions': { + 'EBSEnabled': True, + 'VolumeSize': 25, + }, + }, + ) + + def test_prod_zone_awareness(self): + """ + Test that production environment has zone awareness enabled with 3 availability zones. + """ + search_stack = self.app.prod_backend_pipeline_stack.prod_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify zone awareness is enabled with 3 AZs + search_template.has_resource_properties( + 'AWS::OpenSearchService::Domain', + { + 'ClusterConfig': { + 'ZoneAwarenessEnabled': True, + }, + }, + ) + + def test_prod_uses_all_private_subnets(self): + """ + Test that production OpenSearch Domain uses all private isolated subnets (3 AZs) + for high availability and zone awareness. + + Production requires 3 subnets across 3 availability zones to support + multi-AZ with standby configuration. + """ + search_stack = self.app.prod_backend_pipeline_stack.prod_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Get the OpenSearch Domain's subnet configuration + opensearch_resources = search_template.find_resources('AWS::OpenSearchService::Domain') + opensearch_properties = list(opensearch_resources.values())[0]['Properties'] + vpc_options = opensearch_properties['VPCOptions'] + subnet_ids = vpc_options['SubnetIds'] + + # For production, should use 3 subnets (one per AZ) + self.assertEqual( + len(subnet_ids), + 3, + 'Production OpenSearch should use exactly 3 subnets (one per availability zone)', + ) + + def test_prod_index_shard_configuration(self): + """ + Test that production index manager custom resource uses production shard configuration: + - 1 primary shard + - 2 replica shards (for 3 data nodes across 3 AZs) + + This ensures data availability if one node fails, with total shards (1 + 2 = 3) + being a multiple of 3 to distribute evenly across the 3 data nodes. + """ + search_stack = self.app.prod_backend_pipeline_stack.prod_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify index manager custom resource has production shard/replica configuration + search_template.has_resource_properties( + 'Custom::IndexManager', + { + 'numberOfShards': 1, + 'numberOfReplicas': 2, + }, + ) + + def test_prod_storage_threshold_alarm(self): + """ + Test that production storage alarm threshold is set to 50% of 25GB volume (12800 MB). + + Production uses 25GB EBS volumes, so 50% threshold = 12.5GB = 12800 MB. + This gives ample time to plan capacity increases before hitting critical levels. + """ + search_stack = self.app.prod_backend_pipeline_stack.prod_stage.search_persistent_stack + search_template = Template.from_stack(search_stack) + + # Verify Free Storage Space Alarm threshold for production (50% of 25GB = 12800 MB) + # Note: FreeStorageSpace metric is reported in megabytes (MB) + search_template.has_resource_properties( + 'AWS::CloudWatch::Alarm', + { + 'MetricName': 'FreeStorageSpace', + 'Namespace': 'AWS/ES', + 'Threshold': 12800, # 50% of 25GB = 12.5GB = 12800 MB + 'ComparisonOperator': 'LessThanThreshold', + 'EvaluationPeriods': 1, + }, + ) From da2a72e5814e7b1280582ade6be083ff9f5e1e25 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 15:27:49 -0600 Subject: [PATCH 123/137] Instantiating opensearch client outside of lambda handler --- .../search/handlers/provider_update_ingest.py | 4 +- .../lambdas/python/search/handlers/search.py | 11 +- .../python/search/opensearch_client.py | 9 +- .../function/test_provider_update_ingest.py | 126 ++++++------------ .../tests/function/test_search_privileges.py | 23 ++-- .../tests/function/test_search_providers.py | 84 +++++------- .../tests/unit/test_opensearch_client.py | 6 +- 7 files changed, 100 insertions(+), 163 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py index de6b8b2d3..442dfe4d9 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/handlers/provider_update_ingest.py @@ -20,6 +20,9 @@ from opensearch_client import OpenSearchClient from utils import generate_provider_opensearch_document +# Instantiate the OpenSearch client outside of the handler to cache connection between invocations +opensearch_client = OpenSearchClient(timeout=30) + @sqs_batch_handler def provider_update_ingest_handler(records: list[dict]) -> dict: @@ -83,7 +86,6 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: logger.warning('Unknown compact in record', compact=compact, provider_id=provider_id) # Process providers and bulk index by compact - opensearch_client = OpenSearchClient() batch_item_failures = [] failed_providers: dict[str, set] = {compact: set() for compact in config.compacts} diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index b28ba6f43..bc208db7e 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -52,6 +52,11 @@ ] +# Instantiate the OpenSearch client outside of the handler to cache connection between invocations +# Set timeout to 20 seconds to give API gateway time to respond with response +opensearch_client = OpenSearchClient(timeout=20) + + @api_handler @authorize_compact_level_only_action(action=CCPermissionsAction.READ_GENERAL) def search_api_handler(event: dict, context: LambdaContext): @@ -105,8 +110,7 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus logger.info('Executing OpenSearch provider search', compact=compact, index_name=index_name) # Execute the search - client = OpenSearchClient() - response = client.search(index_name=index_name, body=search_body) + response = opensearch_client.search(index_name=index_name, body=search_body) # Extract hits from the response hits_data = response.get('hits', {}) @@ -200,8 +204,7 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu logger.info('Executing OpenSearch privilege export', compact=compact, index_name=index_name) # Execute the search - client = OpenSearchClient() - response = client.search(index_name=index_name, body=search_body) + response = opensearch_client.search(index_name=index_name, body=search_body) # Extract hits from the response hits_data = response.get('hits', {}) diff --git a/backend/compact-connect/lambdas/python/search/opensearch_client.py b/backend/compact-connect/lambdas/python/search/opensearch_client.py index 38adaa81c..0fc2214b0 100644 --- a/backend/compact-connect/lambdas/python/search/opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/opensearch_client.py @@ -12,7 +12,6 @@ MAX_BACKOFF_SECONDS = 32 DEFAULT_TIMEOUT = 30 -SEARCH_TIMEOUT = 20 class OpenSearchClient: @@ -144,23 +143,21 @@ def _execute_with_retry(self, operation: callable, operation_name: str): f'{operation_name} failed after {MAX_RETRY_ATTEMPTS} attempts. Last error: {last_exception}' ) - def search(self, index_name: str, body: dict, timeout: int = SEARCH_TIMEOUT) -> dict: + def search(self, index_name: str, body: dict) -> dict: """ Execute a search query against the specified index. :param index_name: The name of the index to search :param body: The OpenSearch query body - :param timeout: How long to wait before raising a connection timeout exception :return: The search response from OpenSearch :raises CCInvalidRequestException: If the query is invalid (400 error) or times out """ try: - return self._client.search(index=index_name, body=body, timeout=timeout) + return self._client.search(index=index_name, body=body) except ConnectionTimeout as e: logger.warning( 'OpenSearch search request timed out', index_name=index_name, - timeout=timeout, error=str(e), ) # We are returning this as an invalid request exception so the UI client picks it up as @@ -309,7 +306,7 @@ def _bulk_operation_with_retry( for attempt in range(1, MAX_RETRY_ATTEMPTS + 1): try: - return self._client.bulk(body=actions, index=index_name, timeout=DEFAULT_TIMEOUT) + return self._client.bulk(body=actions, index=index_name) except (ConnectionTimeout, TransportError) as e: last_exception = e if attempt < MAX_RETRY_ATTEMPTS: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py index 676c79761..f61add67b 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, patch from common_test.test_constants import ( DEFAULT_LICENSE_EXPIRATION_DATE, @@ -119,10 +119,9 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_inde if not bulk_index_response: bulk_index_response = {'items': [], 'errors': False} - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.bulk_index.return_value = bulk_index_response - return mock_client_instance + # mock_opensearch_client is the patched instance, not the class + mock_opensearch_client.bulk_index.return_value = bulk_index_response + return mock_opensearch_client def _generate_expected_document(self, compact: str, provider_id: str = None) -> dict: """Generate the expected document that should be indexed into OpenSearch.""" @@ -184,13 +183,13 @@ def _generate_expected_document(self, compact: str, provider_id: str = None) -> 'militaryAffiliations': [], } - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_opensearch_client_called_with_expected_parameters(self, mock_opensearch_client): """Test that OpenSearch client is called with expected parameters when indexing a record.""" from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') @@ -215,27 +214,24 @@ def test_opensearch_client_called_with_expected_parameters(self, mock_opensearch mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that the OpenSearchClient was instantiated - mock_opensearch_client.assert_called_once() - # Assert that bulk_index was called once with expected parameters - self.assertEqual(1, mock_client_instance.bulk_index.call_count) + self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) # Verify the call arguments - call_args = mock_client_instance.bulk_index.call_args + call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) self.assertEqual([self._generate_expected_document('aslp')], call_args.kwargs['documents']) # Verify no batch item failures self.assertEqual({'batchItemFailures': []}, result) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_provider_ids_are_deduped_only_one_document_indexed(self, mock_opensearch_client): """Test that duplicate provider IDs in the batch are deduplicated.""" from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') @@ -284,17 +280,17 @@ def test_provider_ids_are_deduped_only_one_document_indexed(self, mock_opensearc result = provider_update_ingest_handler(event, mock_context) # Assert that bulk_index was called only once despite 3 records - self.assertEqual(1, mock_client_instance.bulk_index.call_count) + self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) # Verify only ONE document was indexed (deduplication worked) - call_args = mock_client_instance.bulk_index.call_args + call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual(1, len(call_args.kwargs['documents'])) self.assertEqual(MOCK_ASLP_PROVIDER_ID, call_args.kwargs['documents'][0]['providerId']) # Verify no batch item failures self.assertEqual({'batchItemFailures': []}, result) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_client): """Test that a record that fails validation is returned in batchItemFailures.""" from handlers.provider_update_ingest import provider_update_ingest_handler @@ -339,17 +335,13 @@ def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_cli self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opensearch_client): """Test that a record which fails to be indexed by OpenSearch is in batchItemFailures.""" from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client to return errors for specific documents - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - # Simulate OpenSearch returning an error for one document - mock_client_instance.bulk_index.return_value = { + mock_opensearch_client.bulk_index.return_value = { 'errors': True, 'items': [ { @@ -412,16 +404,14 @@ def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opens self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12346', result['batchItemFailures'][0]['itemIdentifier']) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_bulk_index_exception_returns_all_batch_item_failures(self, mock_opensearch_client): """Test that when bulk_index raises an exception, all providers are marked as failed.""" from cc_common.exceptions import CCInternalException from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client to raise an exception - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.bulk_index.side_effect = CCInternalException('Connection timeout after 5 retries') + mock_opensearch_client.bulk_index.side_effect = CCInternalException('Connection timeout after 5 retries') # Create provider and license records in DynamoDB for both compacts self._put_test_provider_and_license_record_in_dynamodb_table('aslp') @@ -462,13 +452,13 @@ def test_bulk_index_exception_returns_all_batch_item_failures(self, mock_opensea self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) self.assertEqual('12346', result['batchItemFailures'][1]['itemIdentifier']) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_multiple_compacts_indexed_separately(self, mock_opensearch_client): """Test that providers from different compacts are indexed in their respective indices.""" from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create provider and license records for two different compacts self._put_test_provider_and_license_record_in_dynamodb_table('aslp') @@ -506,7 +496,7 @@ def test_multiple_compacts_indexed_separately(self, mock_opensearch_client): # Assert that bulk_index was called for each compact that had providers # Note: The handler iterates over all compacts, but only calls bulk_index if there are documents - call_args_list = mock_client_instance.bulk_index.call_args_list + call_args_list = mock_opensearch_client.bulk_index.call_args_list # Find the calls for aslp and octp aslp_calls = [c for c in call_args_list if c.kwargs['index_name'] == 'compact_aslp_providers'] @@ -522,7 +512,7 @@ def test_multiple_compacts_indexed_separately(self, mock_opensearch_client): # Verify no batch item failures self.assertEqual({'batchItemFailures': []}, result) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_empty_records_returns_empty_batch_failures(self, mock_opensearch_client): """Test that an empty Records list returns empty batchItemFailures.""" from handlers.provider_update_ingest import provider_update_ingest_handler @@ -536,9 +526,9 @@ def test_empty_records_returns_empty_batch_failures(self, mock_opensearch_client self.assertEqual({'batchItemFailures': []}, result) # Verify OpenSearch client was never called - mock_opensearch_client.assert_not_called() + mock_opensearch_client.bulk_index.assert_not_called() - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_insert_event_without_old_image_indexes_successfully(self, mock_opensearch_client): """Test that INSERT events (newly created records) without OldImage are processed correctly. @@ -549,7 +539,7 @@ def test_insert_event_without_old_image_indexes_successfully(self, mock_opensear from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') @@ -576,14 +566,11 @@ def test_insert_event_without_old_image_indexes_successfully(self, mock_opensear mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that the OpenSearchClient was instantiated - mock_opensearch_client.assert_called_once() - # Assert that bulk_index was called with the correct parameters - self.assertEqual(1, mock_client_instance.bulk_index.call_count) + self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) # Verify the call arguments - call_args = mock_client_instance.bulk_index.call_args + call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) self.assertEqual([self._generate_expected_document('aslp')], call_args.kwargs['documents']) @@ -624,7 +611,7 @@ def _create_dynamodb_stream_record_with_old_image_only( 'eventSourceARN': 'arn:aws:dynamodb:us-east-1:123456789012:table/provider-table/stream/1234', } - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opensearch_client): """Test that REMOVE events (deleted records) with only OldImage are processed correctly. @@ -635,7 +622,7 @@ def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opense from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('aslp') @@ -660,21 +647,18 @@ def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opense mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that the OpenSearchClient was instantiated - mock_opensearch_client.assert_called_once() - # Assert that bulk_index was called with the correct parameters - self.assertEqual(1, mock_client_instance.bulk_index.call_count) + self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) # Verify the call arguments - call_args = mock_client_instance.bulk_index.call_args + call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) self.assertEqual([self._generate_expected_document('aslp')], call_args.kwargs['documents']) # Verify no batch item failures for REMOVE event self.assertEqual({'batchItemFailures': []}, result) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_provider_deleted_from_index_when_no_records_found(self, mock_opensearch_client): """Test that when no provider records are found (CCNotFoundException), bulk_delete is called. @@ -685,10 +669,8 @@ def test_provider_deleted_from_index_when_no_records_found(self, mock_opensearch from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.bulk_index.return_value = {'items': [], 'errors': False} - mock_client_instance.bulk_delete.return_value = {'items': [], 'errors': False} + mock_opensearch_client.bulk_index.return_value = {'items': [], 'errors': False} + mock_opensearch_client.bulk_delete.return_value = set() # bulk_delete returns a set of failed IDs # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted @@ -714,31 +696,26 @@ def test_provider_deleted_from_index_when_no_records_found(self, mock_opensearch mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that the OpenSearchClient was instantiated - mock_opensearch_client.assert_called_once() - # Assert that bulk_index was NOT called (no documents to index) - mock_client_instance.bulk_index.assert_not_called() + mock_opensearch_client.bulk_index.assert_not_called() # Assert that bulk_delete WAS called with the correct parameters - self.assertEqual(1, mock_client_instance.bulk_delete.call_count) - call_args = mock_client_instance.bulk_delete.call_args + self.assertEqual(1, mock_opensearch_client.bulk_delete.call_count) + call_args = mock_opensearch_client.bulk_delete.call_args self.assertEqual('compact_aslp_providers', call_args.kwargs['index_name']) self.assertEqual([MOCK_ASLP_PROVIDER_ID], call_args.kwargs['document_ids']) # Verify no batch item failures (deletion is expected behavior, not a failure) self.assertEqual({'batchItemFailures': []}, result) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_client): """Test that when bulk_delete fails, the provider is returned in batchItemFailures.""" from cc_common.exceptions import CCInternalException from handlers.provider_update_ingest import provider_update_ingest_handler # Set up mock OpenSearch client - bulk_delete raises exception - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.bulk_delete.side_effect = CCInternalException('Connection timeout after 5 retries') + mock_opensearch_client.bulk_delete.side_effect = CCInternalException('Connection timeout after 5 retries') # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted @@ -768,7 +745,7 @@ def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_cl self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) - @patch('handlers.provider_update_ingest.OpenSearchClient') + @patch('handlers.provider_update_ingest.opensearch_client') def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock_opensearch_client): """Test that when bulk_delete returns 404 (document not found), it is NOT treated as a failure. @@ -779,28 +756,9 @@ def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock """ from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client - bulk_delete returns 404 not_found response - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - # Simulate OpenSearch bulk delete response when document doesn't exist - mock_client_instance.bulk_delete.return_value = { - 'errors': True, # OpenSearch reports this as an "error" even though it's just not found - 'items': [ - { - 'delete': { - '_index': 'compact_aslp_providers', - '_id': MOCK_ASLP_PROVIDER_ID, - 'status': 404, - 'result': 'not_found', - 'error': { - 'type': 'document_missing_exception', - 'reason': f'[_doc][{MOCK_ASLP_PROVIDER_ID}]: document missing', - }, - } - } - ], - } + # bulk_delete returns a set of failed document IDs, empty set means no failures (404 is ignored) + mock_opensearch_client.bulk_delete.return_value = set() # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted @@ -827,7 +785,7 @@ def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock result = provider_update_ingest_handler(event, mock_context) # Assert that bulk_delete was called - self.assertEqual(1, mock_client_instance.bulk_delete.call_count) + self.assertEqual(1, mock_opensearch_client.bulk_delete.call_count) # Verify NO batch item failures - 404 is not treated as an error self.assertEqual({'batchItemFailures': []}, result) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 5eaa34328..fc807c41b 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -1,5 +1,5 @@ import json -from unittest.mock import Mock, patch +from unittest.mock import patch from moto import mock_aws @@ -54,7 +54,7 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, search_re """ Configure the mock OpenSearchClient for testing. - :param mock_opensearch_client: The patched OpenSearchClient class + :param mock_opensearch_client: The patched opensearch_client instance :param search_response: The response to return from the search method :return: The mock client instance """ @@ -66,10 +66,9 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, search_re } } - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.search.return_value = search_response - return mock_client_instance + # mock_opensearch_client is the patched instance, not the class + mock_opensearch_client.search.return_value = search_response + return mock_opensearch_client def _create_mock_provider_hit_with_privileges( self, @@ -140,7 +139,7 @@ def _create_mock_provider_hit_with_privileges( hit['sort'] = sort_values return hit - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_privilege_export_returns_presigned_url(self, mock_opensearch_client): """Test that privilege export returns a presigned URL to a CSV file.""" from handlers.search import search_api_handler @@ -191,7 +190,7 @@ def test_privilege_export_returns_presigned_url(self, mock_opensearch_client): self.assertIn('00000000-0000-0000-0000-000000000001', csv_content) self.assertIn('PRIV-001', csv_content) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_privilege_export_with_empty_results_returns_404(self, mock_opensearch_client): """Test that privilege export with no results returns a 404 error.""" from handlers.search import search_api_handler @@ -225,7 +224,7 @@ def test_privilege_export_with_empty_results_returns_404(self, mock_opensearch_c # Should have no objects self.assertEqual(0, response.get('KeyCount', 0)) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_privilege_export_skips_provider_without_privileges_returns_404(self, mock_opensearch_client): """Test that providers without privileges result in a 404 error.""" from handlers.search import search_api_handler @@ -282,7 +281,7 @@ def test_privilege_export_skips_provider_without_privileges_returns_404(self, mo # Should have no objects self.assertEqual(0, response.get('KeyCount', 0)) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, mock_opensearch_client): """Test that when inner_hits contains multiple matches, all are exported to CSV.""" from handlers.search import search_api_handler @@ -477,7 +476,7 @@ def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, moc lines[2], ) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_privilege_export_without_inner_hits_exports_all_privileges(self, mock_opensearch_client): """Test that without inner_hits, all privileges for matching providers are exported.""" from handlers.search import search_api_handler @@ -740,7 +739,7 @@ def test_export_query_with_nested_index_key_returns_400(self): self.assertIn('Cross-index queries are not allowed', body['message']) self.assertIn("'index'", body['message']) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_client): """Test that a privilege with a compact field that doesn't match the path parameter returns 400.""" from handlers.search import search_api_handler diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index ae5e42d63..08d3622a3 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -1,8 +1,8 @@ import json -from unittest.mock import Mock, patch +from unittest.mock import patch +from cc_common.exceptions import CCInvalidRequestException from moto import mock_aws -from opensearchpy.exceptions import RequestError from . import TstFunction @@ -54,7 +54,7 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, search_re """ Configure the mock OpenSearchClient for testing. - :param mock_opensearch_client: The patched OpenSearchClient class + :param mock_opensearch_client: The patched opensearch_client instance :param search_response: The response to return from the search method :return: The mock client instance """ @@ -66,10 +66,9 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, search_re } } - mock_client_instance = Mock() - mock_opensearch_client.return_value = mock_client_instance - mock_client_instance.search.return_value = search_response - return mock_client_instance + # mock_opensearch_client is the patched instance, not the class + mock_opensearch_client.search.return_value = search_response + return mock_opensearch_client def _create_mock_provider_hit( self, @@ -102,24 +101,23 @@ def _create_mock_provider_hit( hit['sort'] = sort_values return hit - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_basic_search_with_match_all_query(self, mock_opensearch_client): """Test that a basic search with no query uses match_all.""" from handlers.search import search_api_handler - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create event with minimal body - just the required query field event = self._create_api_event(compact='aslp', body={'query': {'match_all': {}}}) response = search_api_handler(event, self.mock_context) - # Verify OpenSearchClient was instantiated and search was called - mock_opensearch_client.assert_called_once() - mock_client_instance.search.assert_called_once() + # Verify search was called + mock_opensearch_client.search.assert_called_once() # Verify the search was called with correct parameters - mock_client_instance.search.assert_called_once_with( + mock_opensearch_client.search.assert_called_once_with( index_name='compact_aslp_providers', body={'query': {'match_all': {}}, 'size': 100} ) @@ -128,12 +126,12 @@ def test_basic_search_with_match_all_query(self, mock_opensearch_client): body = json.loads(response['body']) self.assertEqual({'providers': [], 'total': {'relation': 'eq', 'value': 0}}, body) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_with_custom_query(self, mock_opensearch_client): """Test that a custom OpenSearch query is passed through correctly.""" from handlers.search import search_api_handler - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Create a custom bool query custom_query = { @@ -149,7 +147,7 @@ def test_search_with_custom_query(self, mock_opensearch_client): search_api_handler(event, self.mock_context) # Verify the custom query was passed through - mock_client_instance.search.assert_called_once_with( + mock_opensearch_client.search.assert_called_once_with( index_name='compact_aslp_providers', body={ 'query': {'bool': {'must': [{'match': {'givenName': 'John'}}, {'term': {'licenseStatus': 'active'}}]}}, @@ -158,7 +156,7 @@ def test_search_with_custom_query(self, mock_opensearch_client): }, ) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_size_capped_at_max(self, mock_opensearch_client): """Test that size parameter is capped at MAX_SIZE (100).""" from handlers.search import search_api_handler @@ -175,14 +173,14 @@ def test_search_size_capped_at_max(self, mock_opensearch_client): }, json.loads(result['body']), ) - mock_opensearch_client.assert_not_called() + mock_opensearch_client.search.assert_not_called() - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_with_sort_parameter(self, mock_opensearch_client): """Test that sort parameter is included in the search body.""" from handlers.search import search_api_handler - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) sort_config = [{'providerId': 'asc'}, {'dateOfUpdate': 'desc'}] search_after_values = ['provider-uuid-123'] @@ -197,7 +195,7 @@ def test_search_with_sort_parameter(self, mock_opensearch_client): search_api_handler(event, self.mock_context) - mock_client_instance.search.assert_called_once_with( + mock_opensearch_client.search.assert_called_once_with( index_name='compact_aslp_providers', body={ 'query': {'match_all': {}}, @@ -207,7 +205,7 @@ def test_search_with_sort_parameter(self, mock_opensearch_client): }, ) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_after_without_sort_returns_400(self, mock_opensearch_client): """Test that search_after without sort raises an error.""" from handlers.search import search_api_handler @@ -242,7 +240,7 @@ def test_invalid_request_body_returns_400(self): body = json.loads(response['body']) self.assertIn('Invalid request', body['message']) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_returns_sanitized_providers(self, mock_opensearch_client): """Test that provider records are sanitized through ProviderGeneralResponseSchema.""" from handlers.search import search_api_handler @@ -288,7 +286,7 @@ def test_search_returns_sanitized_providers(self, mock_opensearch_client): body, ) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_response_includes_last_sort_for_pagination(self, mock_opensearch_client): """Test that lastSort is included in response for search_after pagination.""" from handlers.search import search_api_handler @@ -317,21 +315,21 @@ def test_search_response_includes_last_sort_for_pagination(self, mock_opensearch self.assertIn('lastSort', body) self.assertEqual(['provider-uuid-123', '2024-01-15T10:30:00+00:00'], body['lastSort']) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_search_uses_correct_index_for_compact(self, mock_opensearch_client): """Test that the correct index name is used based on the compact parameter.""" from handlers.search import search_api_handler - mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._when_testing_mock_opensearch_client(mock_opensearch_client) # Test with different compacts for compact in ['aslp', 'octp', 'coun']: - mock_client_instance.reset_mock() + mock_opensearch_client.reset_mock() event = self._create_api_event(compact, body={'query': {'match_all': {}}}) search_api_handler(event, self.mock_context) - call_args = mock_client_instance.search.call_args + call_args = mock_opensearch_client.search.call_args self.assertEqual(f'compact_{compact}_providers', call_args.kwargs['index_name']) def test_missing_scopes_returns_403(self): @@ -436,35 +434,18 @@ def test_query_with_nested_index_key_returns_400(self): self.assertIn('Cross-index queries are not allowed', body['message']) self.assertIn("'index'", body['message']) - @patch('opensearch_client.OpenSearch') + @patch('handlers.search.opensearch_client') def test_opensearch_request_error_returns_400_with_error_message(self, mock_opensearch_client): """Test that OpenSearch RequestError with status 400 returns error message to caller.""" from handlers.search import search_api_handler - # Configure the mock internal OpenSearch client to raise a RequestError - mock_internal_client = Mock() - mock_opensearch_client.return_value = mock_internal_client - # Create a RequestError with realistic OpenSearch error structure error_reason = ( - 'Text fields are not optimised for operations that require per-document field data ' + 'Invalid search query: Text fields are not optimised for operations that require per-document field data ' 'like aggregations and sorting, so these operations are disabled by default. ' 'Please use a keyword field instead.' ) - error_info = { - 'error': { - 'root_cause': [ - { - 'type': 'illegal_argument_exception', - 'reason': error_reason, - } - ], - 'type': 'search_phase_execution_exception', - 'reason': 'all shards failed', - }, - 'status': 400, - } - mock_internal_client.search.side_effect = RequestError(400, 'search_phase_execution_exception', error_info) + mock_opensearch_client.search.side_effect = CCInvalidRequestException(error_reason) event = self._create_api_event( 'aslp', @@ -478,12 +459,9 @@ def test_opensearch_request_error_returns_400_with_error_message(self, mock_open self.assertEqual(400, response['statusCode']) body = json.loads(response['body']) - self.assertEqual( - f'Invalid search query: {error_reason}', - body['message'], - ) + self.assertEqual(error_reason, body['message']) - @patch('handlers.search.OpenSearchClient') + @patch('handlers.search.opensearch_client') def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_client): """Test that a provider with a compact field that doesn't match the path parameter returns 400.""" from handlers.search import search_api_handler diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index b20c7a1f2..0500acab0 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -108,7 +108,7 @@ def test_search_calls_internal_client_with_expected_arguments(self): result = client.search(index_name=index_name, body=query_body) - mock_internal_client.search.assert_called_once_with(index=index_name, body=query_body, timeout=20) + mock_internal_client.search.assert_called_once_with(index=index_name, body=query_body) self.assertEqual(expected_response, result) def test_search_raises_cc_invalid_request_exception_on_400_request_error(self): @@ -225,7 +225,7 @@ def test_bulk_index_calls_internal_client_with_expected_arguments(self): {'index': {'_id': 'provider-2'}}, {'providerId': 'provider-2', 'givenName': 'Jane', 'familyName': 'Smith'}, ] - mock_internal_client.bulk.assert_called_once_with(body=expected_actions, index=index_name, timeout=30) + mock_internal_client.bulk.assert_called_once_with(body=expected_actions, index=index_name) self.assertEqual(expected_response, result) def test_bulk_index_uses_custom_id_field(self): @@ -247,7 +247,7 @@ def test_bulk_index_uses_custom_id_field(self): {'index': {'_id': 'custom-2'}}, {'customId': 'custom-2', 'name': 'Document 2'}, ] - mock_internal_client.bulk.assert_called_once_with(body=expected_actions, index=index_name, timeout=30) + mock_internal_client.bulk.assert_called_once_with(body=expected_actions, index=index_name) def test_bulk_index_returns_early_for_empty_documents(self): """Test that bulk_index returns early without calling the internal client for empty documents.""" From 1898a216586e4f7117afa3619195cb629a636587 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 15:39:35 -0600 Subject: [PATCH 124/137] update requirements to latest --- .../cognito-backup/requirements-dev.txt | 8 ++++---- .../python/cognito-backup/requirements.txt | 6 +++--- .../python/common/requirements-dev.txt | 20 +++++++++---------- .../lambdas/python/common/requirements.txt | 6 +++--- .../requirements-dev.txt | 6 +++--- .../custom-resources/requirements-dev.txt | 6 +++--- .../python/data-events/requirements-dev.txt | 6 +++--- .../disaster-recovery/requirements-dev.txt | 6 +++--- .../provider-data-v1/requirements-dev.txt | 8 ++++---- .../python/search/requirements-dev.txt | 6 +++--- .../lambdas/python/search/requirements.txt | 2 +- .../staff-user-pre-token/requirements-dev.txt | 6 +++--- .../python/staff-users/requirements-dev.txt | 10 +++++----- backend/compact-connect/requirements-dev.txt | 8 ++++---- backend/compact-connect/requirements.txt | 8 ++++---- 15 files changed, 56 insertions(+), 56 deletions(-) diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt index c2f0c39df..184c65492 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements-dev.txt @@ -6,11 +6,11 @@ # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements-dev.in -boto3==1.42.4 +boto3==1.42.11 # via # -r lambdas/python/cognito-backup/requirements-dev.in # moto -botocore==1.42.4 +botocore==1.42.11 # via # -r lambdas/python/cognito-backup/requirements-dev.in # boto3 @@ -37,7 +37,7 @@ jmespath==1.0.1 # aws-lambda-powertools # boto3 # botocore -joserfc==1.5.0 +joserfc==1.6.0 # via moto markupsafe==3.0.3 # via @@ -77,7 +77,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # requests diff --git a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt index 15c38b047..b9089934d 100644 --- a/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt +++ b/backend/compact-connect/lambdas/python/cognito-backup/requirements.txt @@ -6,9 +6,9 @@ # aws-lambda-powertools==3.23.0 # via -r lambdas/python/cognito-backup/requirements.in -boto3==1.42.4 +boto3==1.42.11 # via -r lambdas/python/cognito-backup/requirements.in -botocore==1.42.4 +botocore==1.42.11 # via # -r lambdas/python/cognito-backup/requirements.in # boto3 @@ -26,5 +26,5 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.6.1 +urllib3==2.6.2 # via botocore diff --git a/backend/compact-connect/lambdas/python/common/requirements-dev.txt b/backend/compact-connect/lambdas/python/common/requirements-dev.txt index f3001523e..a59a86d41 100644 --- a/backend/compact-connect/lambdas/python/common/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/common/requirements-dev.txt @@ -16,27 +16,27 @@ aws-sam-translator==1.105.0 # via cfn-lint aws-xray-sdk==2.15.0 # via moto -boto3==1.42.4 +boto3==1.42.11 # via # aws-sam-translator # moto -boto3-stubs[full]==1.42.4 +boto3-stubs[full]==1.42.11 # via -r lambdas/python/common/requirements-dev.in -boto3-stubs-full==1.42.4 +boto3-stubs-full==1.42.10 # via boto3-stubs -botocore==1.42.4 +botocore==1.42.11 # via # aws-xray-sdk # boto3 # moto # s3transfer -botocore-stubs==1.42.4 +botocore-stubs==1.42.11 # via boto3-stubs certifi==2025.11.12 # via requests cffi==2.0.0 # via cryptography -cfn-lint==1.42.0 +cfn-lint==1.43.0 # via moto charset-normalizer==3.4.4 # via requests @@ -59,7 +59,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joserfc==1.5.0 +joserfc==1.6.0 # via moto jsonpatch==1.33 # via cfn-lint @@ -150,7 +150,7 @@ six==1.17.0 # rfc3339-validator sympy==1.14.0 # via cfn-lint -types-awscrt==0.29.2 +types-awscrt==0.30.0 # via botocore-stubs types-s3transfer==0.16.0 # via boto3-stubs @@ -163,9 +163,9 @@ typing-extensions==4.15.0 # typing-inspection typing-inspection==0.4.2 # via pydantic -tzdata==2025.2 +tzdata==2025.3 # via faker -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/common/requirements.txt b/backend/compact-connect/lambdas/python/common/requirements.txt index 886da2a81..632ab69b1 100644 --- a/backend/compact-connect/lambdas/python/common/requirements.txt +++ b/backend/compact-connect/lambdas/python/common/requirements.txt @@ -10,9 +10,9 @@ argon2-cffi-bindings==25.1.0 # via argon2-cffi aws-lambda-powertools==3.23.0 # via -r lambdas/python/common/requirements.in -boto3==1.42.4 +boto3==1.42.11 # via -r lambdas/python/common/requirements.in -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # s3transfer @@ -49,7 +49,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via aws-lambda-powertools -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # requests diff --git a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt index d6a37154b..1f5e19463 100644 --- a/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/compact-configuration/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/compact-configuration/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt index 0a5be4023..a751a4574 100644 --- a/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/custom-resources/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/custom-resources/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt index 6d028c569..a31fe9b7a 100644 --- a/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/data-events/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/data-events/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt index 716b90bfa..834cf4a1c 100644 --- a/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/disaster-recovery/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/disaster-recovery/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt index c801c6500..178f7bf50 100644 --- a/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/provider-data-v1/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -60,9 +60,9 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -tzdata==2025.2 +tzdata==2025.3 # via faker -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/search/requirements-dev.txt b/backend/compact-connect/lambdas/python/search/requirements-dev.txt index 6a8162bab..1d8dccf31 100644 --- a/backend/compact-connect/lambdas/python/search/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/search/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/search/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -56,7 +56,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/search/requirements.txt b/backend/compact-connect/lambdas/python/search/requirements.txt index 6ff061bc8..2352dd4f0 100644 --- a/backend/compact-connect/lambdas/python/search/requirements.txt +++ b/backend/compact-connect/lambdas/python/search/requirements.txt @@ -30,7 +30,7 @@ six==1.17.0 # via python-dateutil typing-extensions==4.15.0 # via grpcio -urllib3==2.6.1 +urllib3==2.6.2 # via # opensearch-py # requests diff --git a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt index 2401d3526..b4c5e0e2c 100644 --- a/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-user-pre-token/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -58,7 +58,7 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt index f022d6c90..5d98ff0c5 100644 --- a/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/staff-users/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras lambdas/python/staff-users/requirements-dev.in # -boto3==1.42.4 +boto3==1.42.11 # via moto -botocore==1.42.4 +botocore==1.42.11 # via # boto3 # moto @@ -33,7 +33,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joserfc==1.5.0 +joserfc==1.6.0 # via moto markupsafe==3.0.3 # via @@ -64,9 +64,9 @@ s3transfer==0.16.0 # via boto3 six==1.17.0 # via python-dateutil -tzdata==2025.2 +tzdata==2025.3 # via faker -urllib3==2.6.1 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/requirements-dev.txt b/backend/compact-connect/requirements-dev.txt index 060dbd80e..c3230bdc7 100644 --- a/backend/compact-connect/requirements-dev.txt +++ b/backend/compact-connect/requirements-dev.txt @@ -28,7 +28,7 @@ defusedxml==0.7.1 # via py-serializable faker==37.12.0 # via -r requirements-dev.in -filelock==3.20.0 +filelock==3.20.1 # via cachecontrol idna==3.11 # via requests @@ -88,7 +88,7 @@ requests==2.32.5 # pip-audit rich==14.2.0 # via pip-audit -ruff==0.14.8 +ruff==0.14.9 # via -r requirements-dev.in sortedcontainers==2.4.0 # via cyclonedx-python-lib @@ -96,9 +96,9 @@ tomli==2.3.0 # via pip-audit tomli-w==1.2.0 # via pip-audit -tzdata==2025.2 +tzdata==2025.3 # via faker -urllib3==2.6.1 +urllib3==2.6.2 # via requests wheel==0.45.1 # via pip-tools diff --git a/backend/compact-connect/requirements.txt b/backend/compact-connect/requirements.txt index 8eb6df186..070ada8b7 100644 --- a/backend/compact-connect/requirements.txt +++ b/backend/compact-connect/requirements.txt @@ -12,11 +12,11 @@ aws-cdk-asset-awscli-v1==2.2.242 # via aws-cdk-lib aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib -aws-cdk-aws-lambda-python-alpha==2.232.1a0 +aws-cdk-aws-lambda-python-alpha==2.232.2a0 # via -r requirements.in aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.232.1 +aws-cdk-lib==2.232.2 # via # -r requirements.in # aws-cdk-aws-lambda-python-alpha @@ -25,7 +25,7 @@ cattrs==25.3.0 # via jsii cdk-nag==2.37.55 # via -r requirements.in -constructs==10.4.3 +constructs==10.4.4 # via # -r requirements.in # aws-cdk-aws-lambda-python-alpha @@ -33,7 +33,7 @@ constructs==10.4.3 # cdk-nag importlib-resources==6.5.2 # via jsii -jsii==1.120.0 +jsii==1.121.0 # via # aws-cdk-asset-awscli-v1 # aws-cdk-asset-node-proxy-agent-v6 From d9b16c64f111d4e6506d9faa493bf720e08fb2c1 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 15:52:59 -0600 Subject: [PATCH 125/137] update multi-account folder requirements to latest --- backend/multi-account/backups/requirements-dev.txt | 4 ++-- backend/multi-account/backups/requirements.txt | 2 +- backend/multi-account/control-tower/requirements-dev.txt | 2 +- backend/multi-account/control-tower/requirements.txt | 2 +- backend/multi-account/log-aggregation/requirements.txt | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/multi-account/backups/requirements-dev.txt b/backend/multi-account/backups/requirements-dev.txt index 9a922a328..5680962c6 100644 --- a/backend/multi-account/backups/requirements-dev.txt +++ b/backend/multi-account/backups/requirements-dev.txt @@ -4,9 +4,9 @@ # # pip-compile --no-emit-index-url --no-strip-extras backups/requirements-dev.in # -boto3==1.42.8 +boto3==1.42.11 # via moto -botocore==1.42.8 +botocore==1.42.11 # via # boto3 # moto diff --git a/backend/multi-account/backups/requirements.txt b/backend/multi-account/backups/requirements.txt index 8dfcf26f0..b681e8aac 100644 --- a/backend/multi-account/backups/requirements.txt +++ b/backend/multi-account/backups/requirements.txt @@ -14,7 +14,7 @@ aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.232.1 +aws-cdk-lib==2.232.2 # via -r backups/requirements.in cattrs==25.3.0 # via jsii diff --git a/backend/multi-account/control-tower/requirements-dev.txt b/backend/multi-account/control-tower/requirements-dev.txt index 81ca07257..62d01ed4f 100644 --- a/backend/multi-account/control-tower/requirements-dev.txt +++ b/backend/multi-account/control-tower/requirements-dev.txt @@ -26,7 +26,7 @@ cyclonedx-python-lib==11.6.0 # via pip-audit defusedxml==0.7.1 # via py-serializable -filelock==3.20.0 +filelock==3.20.1 # via cachecontrol idna==3.11 # via requests diff --git a/backend/multi-account/control-tower/requirements.txt b/backend/multi-account/control-tower/requirements.txt index daaa7bb30..b85f9da0d 100644 --- a/backend/multi-account/control-tower/requirements.txt +++ b/backend/multi-account/control-tower/requirements.txt @@ -14,7 +14,7 @@ aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.232.1 +aws-cdk-lib==2.232.2 # via # -r control-tower/requirements.in # cdk-nag diff --git a/backend/multi-account/log-aggregation/requirements.txt b/backend/multi-account/log-aggregation/requirements.txt index c29c4a7b6..c722156d4 100644 --- a/backend/multi-account/log-aggregation/requirements.txt +++ b/backend/multi-account/log-aggregation/requirements.txt @@ -14,7 +14,7 @@ aws-cdk-asset-node-proxy-agent-v6==2.1.0 # via aws-cdk-lib aws-cdk-cloud-assembly-schema==48.20.0 # via aws-cdk-lib -aws-cdk-lib==2.232.1 +aws-cdk-lib==2.232.2 # via -r log-aggregation/requirements.in cattrs==25.3.0 # via jsii From 0f9b1ef016ebdf32040670e962e229ba3fc23e68 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 15:59:46 -0600 Subject: [PATCH 126/137] update purchases folder requirements to latest --- .../python/purchases/requirements-dev.txt | 16 ++++++++-------- .../lambdas/python/purchases/requirements.txt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt index f4ac929f7..162fd97ae 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt @@ -12,11 +12,11 @@ aws-lambda-powertools==3.23.0 # via -r lambdas/python/purchases/requirements-dev.in boolean-py==5.0 # via license-expression -boto3==1.42.3 +boto3==1.42.11 # via # -r lambdas/python/purchases/requirements-dev.in # moto -botocore==1.42.3 +botocore==1.42.11 # via # boto3 # moto @@ -37,7 +37,7 @@ charset-normalizer==3.4.4 # via requests click==8.3.1 # via pip-tools -coverage[toml]==7.12.0 +coverage[toml]==7.13.0 # via # -r lambdas/python/purchases/requirements-dev.in # pytest-cov @@ -53,7 +53,7 @@ docker==7.1.0 # via moto faker==37.12.0 # via -r lambdas/python/purchases/requirements-dev.in -filelock==3.20.0 +filelock==3.20.1 # via cachecontrol idna==3.11 # via requests @@ -121,7 +121,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==9.0.1 +pytest==9.0.2 # via # -r lambdas/python/purchases/requirements-dev.in # pytest-cov @@ -147,7 +147,7 @@ responses==0.25.8 # via moto rich==14.2.0 # via pip-audit -ruff==0.14.8 +ruff==0.14.9 # via -r lambdas/python/purchases/requirements-dev.in s3transfer==0.16.0 # via boto3 @@ -163,9 +163,9 @@ typing-extensions==4.15.0 # via # aws-lambda-powertools # cyclonedx-python-lib -tzdata==2025.2 +tzdata==2025.3 # via faker -urllib3==2.6.0 +urllib3==2.6.2 # via # botocore # docker diff --git a/backend/compact-connect/lambdas/python/purchases/requirements.txt b/backend/compact-connect/lambdas/python/purchases/requirements.txt index 786959633..4bbd97e5b 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements.txt +++ b/backend/compact-connect/lambdas/python/purchases/requirements.txt @@ -18,5 +18,5 @@ pyxb-x==1.2.6.3 # via authorizenet requests==2.32.5 # via authorizenet -urllib3==2.6.0 +urllib3==2.6.2 # via requests From dbfb6216100cbe0602c21f91f22e39519f0277a7 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 16 Dec 2025 16:17:51 -0600 Subject: [PATCH 127/137] Add comment about purpose of logging request body --- .../lambdas/python/search/handlers/search.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index bc208db7e..34a68276c 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -131,7 +131,11 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus logger.error( 'Provider compact field does not match path parameter', # This case is most likely the result of abuse or misconfiguration. - # We log the request body for triaging purposes + # We log the request body for triaging purposes. Although the request body + # may contain PII, devops support will need to view the full query that allowed + # the user to attempt the attack and what that user was attempting to extract. + # The only personnel who have access to view the logs are the same that have + # access to read records from the database. request_body=body, provider_id=source.get('providerId'), provider_compact=sanitized_provider.get('compact'), @@ -251,7 +255,11 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu logger.error( 'Privilege compact field does not match path parameter', # This case is most likely the result of abuse or misconfiguration. - # We log the request body for triaging purposes + # We log the request body for triaging purposes. Although the request body + # may contain PII, devops support will need to view the full query that allowed + # the user to attempt the attack and what that user was attempting to extract. + # The only personnel who have access to view the logs are the same that have + # access to read records from the database. request_body=body, provider_id=provider.get('providerId'), privilege_id=flattened_privilege.get('privilegeId'), From e1f60550f2e92bd7d3ab4bde9e764f7805a4cbbf Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 22 Dec 2025 12:38:10 -0700 Subject: [PATCH 128/137] Initial PR feedback --- backend/compact-connect/docs/design/README.md | 12 ++++++------ .../stacks/search_api_stack/v1_api/api.py | 2 +- .../search_api_stack/v1_api/privilege_search.py | 2 +- .../search_api_stack/v1_api/provider_search.py | 2 +- .../tests/app/test_search_persistent_stack.py | 4 +++- backend/compact-connect/tests/app/test_vpc.py | 13 ------------- 6 files changed, 12 insertions(+), 23 deletions(-) diff --git a/backend/compact-connect/docs/design/README.md b/backend/compact-connect/docs/design/README.md index 1bb5e620d..f0efea724 100644 --- a/backend/compact-connect/docs/design/README.md +++ b/backend/compact-connect/docs/design/README.md @@ -681,10 +681,11 @@ be processed and stored in the transaction history table. ## Advanced Data Search [Back to top](#backend-design) -To support advanced search capability of providers and privilege records, this project leverages +To support advanced search capabilities for provider and privilege records, this project leverages [AWS OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/what-is.html). -Provider data from the provider DynamoDB table is indexed within an OpenSearch Domain (Cluster), which is then -queryable by staff users through the Search API (search.compactconnect.org). The OpenSearch resources are deployed within a Virtual Private Cloud (VPC) to provide a layer of network security. +Provider data from the provider DynamoDB table is indexed into an OpenSearch Domain (Cluster), enabling staff users to perform complex searches through the Search API (search.compactconnect.org). + +The OpenSearch resources are deployed within a Virtual Private Cloud (VPC) to provide network-level security and restrict outside access. Unlike DynamoDB, which is a fully managed and serverless AWS service that does not require (and does not support) VPC deployment, OpenSearch domains have data nodes that must be managed. Placing the OpenSearch domain in a VPC allows us to tightly control which resources and users can access it, reducing exposure to external threats. ### Architecture Overview ![Advanced Search Diagram](./advanced-provider-search.pdf) @@ -726,14 +727,13 @@ POST /v1/compacts/{compact}/providers/search Returns provider records matching the query. Response includes the full provider document with licenses, privileges, and military affiliations. -#### Privilege Search +#### Privilege CSV Export ``` POST /v1/compacts/{compact}/privileges/export ``` Returns flattened privilege records. This endpoint queries the same provider index but extracts and flattens -privileges, combining privilege data with license data to provide a denormalized view suitable for privilege-focused -reports and exports. +privileges, combining privilege data with license data to provide a denormalized list of objects which are then exported to a CSV file for downloading. ### Document Indexing diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py index 54c156f1c..e4be20e89 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/api.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/api.py @@ -10,7 +10,7 @@ class V1Api: - """v1 of the State API""" + """v1 of the Search API""" def __init__( self, diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py index 258cf2396..d37dcc9b2 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/privilege_search.py @@ -11,7 +11,7 @@ class PrivilegeSearch: """ - Endpoint for searching privileges in the OpenSearch domain. + Endpoint related to privilege searching in the OpenSearch domain. """ def __init__( diff --git a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py index adba15aaf..1cdf36ce9 100644 --- a/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py +++ b/backend/compact-connect/stacks/search_api_stack/v1_api/provider_search.py @@ -11,7 +11,7 @@ class ProviderSearch: """ - These endpoints are used by state IT systems to view provider records + Endpoint related to provider searching in the OpenSearch domain. """ def __init__( diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index 417791bda..f19583abf 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -262,7 +262,9 @@ def test_sandbox_uses_expected_private_subnet(self): For non-prod single-node deployments, OpenSearch must use exactly one subnet. We explicitly select privateSubnet1 (CIDR 10.0.0.0/20) to ensure deterministic - placement across deployments. + placement across deployments, since the related lambda functions will also be + deployed within that same subnet, and we want to ensure that can communicate with + one another. This test verifies that OpenSearch references the specific subnet we expect, not just any arbitrary subnet from the VPC. diff --git a/backend/compact-connect/tests/app/test_vpc.py b/backend/compact-connect/tests/app/test_vpc.py index b268a7d9e..4c59c0e90 100644 --- a/backend/compact-connect/tests/app/test_vpc.py +++ b/backend/compact-connect/tests/app/test_vpc.py @@ -43,19 +43,6 @@ def test_vpc_configuration(self): }, ) - def test_subnets_configuration(self): - """ - Test that subnets are created across multiple availability zones. - """ - vpc_stack = self.app.sandbox_backend_stage.vpc_stack - vpc_template = Template.from_stack(vpc_stack) - - # Verify at least 3 subnets are created (one per AZ, max 3 AZs) - # The actual number depends on the region's available AZs - subnet_resources = vpc_template.find_resources('AWS::EC2::Subnet') - subnet_count = len(subnet_resources) - self.assertEqual(subnet_count, 3, 'The VPC should have 3 subnets for OpenSearch high availability') - def test_no_internet_gateway(self): """ Test that no Internet Gateway is created, as we're using VPC endpoints for AWS service access. From 62d13cbee846778a7412c405cef369922296b01c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 22 Dec 2025 13:59:45 -0700 Subject: [PATCH 129/137] PR feedback - redact request body in log --- .../lambdas/python/search/handlers/search.py | 37 +++++++++++++------ .../tests/function/test_search_providers.py | 19 +++++++++- .../tests/unit/test_opensearch_client.py | 2 - 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 34a68276c..6cb8e08f2 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -131,12 +131,9 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus logger.error( 'Provider compact field does not match path parameter', # This case is most likely the result of abuse or misconfiguration. - # We log the request body for triaging purposes. Although the request body - # may contain PII, devops support will need to view the full query that allowed - # the user to attempt the attack and what that user was attempting to extract. - # The only personnel who have access to view the logs are the same that have - # access to read records from the database. - request_body=body, + # We log the request body for triaging purposes. We redact the leaf values + # from the request body to obscure PII. + request_body=_redact_leaf_values(body), provider_id=source.get('providerId'), provider_compact=sanitized_provider.get('compact'), path_compact=compact, @@ -255,12 +252,9 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu logger.error( 'Privilege compact field does not match path parameter', # This case is most likely the result of abuse or misconfiguration. - # We log the request body for triaging purposes. Although the request body - # may contain PII, devops support will need to view the full query that allowed - # the user to attempt the attack and what that user was attempting to extract. - # The only personnel who have access to view the logs are the same that have - # access to read records from the database. - request_body=body, + # We log the request body for triaging purposes. We redact the leaf values + # from the request body to obscure PII. + request_body=_redact_leaf_values(body), provider_id=provider.get('providerId'), privilege_id=flattened_privilege.get('privilegeId'), privilege_compact=sanitized_privilege.get('compact'), @@ -367,6 +361,25 @@ def _get_caller_user_id(event: dict) -> str: raise CCInternalException('Could not determine caller id for privilege report export') from e +def _redact_leaf_values(data: dict | list | str | int | bool | None) -> dict | list | str: + """ + Recursively redact all leaf field values in a data structure. + + This function preserves the structure of nested dictionaries + and lists while replacing all leaf values with "". + + :param data: The data structure to redact (dict, list, or primitive value) + :return: A copy of the data structure with all leaf values redacted + """ + if isinstance(data, dict): + return {key: _redact_leaf_values(value) for key, value in data.items()} + elif isinstance(data, list): + return [_redact_leaf_values(item) for item in data] + else: + # Primitive value (str, int, float, bool, None) - this is a leaf, redact it + return '' + + def _build_opensearch_search_body(body: dict, size_override: int) -> dict: """ Build the OpenSearch search body from the validated request. diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 08d3622a3..bedd14ac3 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -95,6 +95,14 @@ def _create_mock_provider_hit( 'jurisdictionUploadedLicenseStatus': 'active', 'jurisdictionUploadedCompactEligibility': 'eligible', 'birthMonthDay': '06-15', + # adding a couple of fields that are not recognized in the + # ProviderGeneralResponseSchema. Although these are not currently + # stored in OpenSearch, this mock data ensures we are sanitizing + # these private fields by the search serialization logic + 'someNewField': 'somePrivateValue', + 'ssnLastFour': '1234', + 'emailAddress': 'someemail@address.com', + 'dateOfBirth': '1984-12-11' }, } if sort_values: @@ -496,8 +504,17 @@ def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_clie } self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + custom_query = { + 'bool': { + 'must': [ + {'match': {'givenName': 'John'}}, + {'term': {'ssnLastFour': 1234}}, + ] + } + } + # Request for 'aslp' compact but provider has 'octp' compact - event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + event = self._create_api_event('aslp', body={'query': custom_query}) response = search_api_handler(event, self.mock_context) diff --git a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py index 0500acab0..62e5eb630 100644 --- a/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/compact-connect/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -217,8 +217,6 @@ def test_bulk_index_calls_internal_client_with_expected_arguments(self): result = client.bulk_index(index_name=index_name, documents=documents) - # Verify that the bulk method is called with the index in the URL parameter - # and NOT in the action metadata (for security compliance) expected_actions = [ {'index': {'_id': 'provider-1'}}, {'providerId': 'provider-1', 'givenName': 'John', 'familyName': 'Doe'}, From 3f4cc88a96237bb020107e6b820211a44de96340 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 22 Dec 2025 15:20:59 -0700 Subject: [PATCH 130/137] formatting/linter --- .../lambdas/python/search/handlers/search.py | 10 +++++----- .../search/tests/function/test_search_providers.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 6cb8e08f2..6ee1483e5 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -365,7 +365,7 @@ def _redact_leaf_values(data: dict | list | str | int | bool | None) -> dict | l """ Recursively redact all leaf field values in a data structure. - This function preserves the structure of nested dictionaries + This function preserves the structure of nested dictionaries and lists while replacing all leaf values with "". :param data: The data structure to redact (dict, list, or primitive value) @@ -373,11 +373,11 @@ def _redact_leaf_values(data: dict | list | str | int | bool | None) -> dict | l """ if isinstance(data, dict): return {key: _redact_leaf_values(value) for key, value in data.items()} - elif isinstance(data, list): + if isinstance(data, list): return [_redact_leaf_values(item) for item in data] - else: - # Primitive value (str, int, float, bool, None) - this is a leaf, redact it - return '' + + # Primitive value (str, int, float, bool, None) - this is a leaf, redact it + return '' def _build_opensearch_search_body(body: dict, size_override: int) -> dict: diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index bedd14ac3..b275bac73 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -102,7 +102,7 @@ def _create_mock_provider_hit( 'someNewField': 'somePrivateValue', 'ssnLastFour': '1234', 'emailAddress': 'someemail@address.com', - 'dateOfBirth': '1984-12-11' + 'dateOfBirth': '1984-12-11', }, } if sort_values: From 68a2b25a16b5b43a85074aa84e70a0a2a9e6521f Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Mon, 22 Dec 2025 15:28:00 -0700 Subject: [PATCH 131/137] update purchase dev dependency --- .../lambdas/python/purchases/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt index 162fd97ae..d74008ce4 100644 --- a/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt +++ b/backend/compact-connect/lambdas/python/purchases/requirements-dev.txt @@ -74,7 +74,7 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug -marshmallow==3.26.1 +marshmallow==3.26.2 # via -r lambdas/python/purchases/requirements-dev.in mdurl==0.1.2 # via markdown-it-py From 2fa48338906e8da96b9736e841bd8dae03ff696c Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 23 Dec 2025 11:03:10 -0700 Subject: [PATCH 132/137] PR feedback - docs/comments --- .../lambdas/python/search/handlers/search.py | 4 ++++ .../tests/function/test_search_privileges.py | 15 +++++++++++++-- .../tests/function/test_search_providers.py | 15 +++++++-------- .../tests/app/test_search_persistent_stack.py | 4 ++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 6ee1483e5..74d0d9259 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -172,6 +172,8 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu If the query includes a nested query on privileges with `inner_hits`, only the matched privileges will be returned. Otherwise, all privileges for matching providers are returned. + See https://docs.opensearch.org/latest/search-plugins/searching-data/inner-hits/ for more information + about inner_hits. Example nested query with inner_hits: { @@ -228,6 +230,8 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu provider = hit.get('_source', {}) # Check if inner_hits are present for privileges # If so, use only the matched privileges; otherwise, use all privileges + # see https://docs.opensearch.org/latest/search-plugins/searching-data/inner-hits/ for more information + # about inner_hits. inner_hits = hit.get('inner_hits', {}) privileges_inner_hits = inner_hits.get('privileges', {}).get('hits', {}).get('hits', []) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index fc807c41b..2d8cd299e 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -283,7 +283,10 @@ def test_privilege_export_skips_provider_without_privileges_returns_404(self, mo @patch('handlers.search.opensearch_client') def test_privilege_export_with_multiple_inner_hits_exports_all_matched(self, mock_opensearch_client): - """Test that when inner_hits contains multiple matches, all are exported to CSV.""" + """Test that when inner_hits contains multiple matches, all are exported to CSV. + see https://docs.opensearch.org/latest/search-plugins/searching-data/inner-hits/ for more information + about inner_hits. + """ from handlers.search import search_api_handler provider_id = '00000000-0000-0000-0000-000000000001' @@ -812,8 +815,16 @@ def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_cli } self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) + # Use match_all query to simulate a bad actor attempting the broadest possible search + # across the entire OpenSearch domain. While the handler restricts searches to a single + # index based on the path parameter, this query represents an attempt to retrieve + # all providers without any filtering, which could expose providers from other compacts + # if data integrity issues exist (e.g., misconfigured index aliases or data corruption). + # or if a future feature allows cross-index searches that we are not aware of yet. + custom_query = {'match_all': {}} + # Request for 'aslp' compact but privilege has 'octp' compact - event = self._create_api_event('aslp', body={'query': {'match_all': {}}}) + event = self._create_api_event('aslp', body={'query': custom_query}) response = search_api_handler(event, self.mock_context) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index b275bac73..0c3114f74 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -504,14 +504,13 @@ def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_clie } self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) - custom_query = { - 'bool': { - 'must': [ - {'match': {'givenName': 'John'}}, - {'term': {'ssnLastFour': 1234}}, - ] - } - } + # Use match_all query to simulate a bad actor attempting the broadest possible search + # across the entire OpenSearch domain. While the handler restricts searches to a single + # index based on the path parameter, this query represents an attempt to retrieve + # all providers without any filtering, which could expose providers from other compacts + # if data integrity issues exist (e.g., misconfigured index aliases or data corruption). + # or if a future feature allows cross-index searches that we are not aware of yet. + custom_query = {'match_all': {}} # Request for 'aslp' compact but provider has 'octp' compact event = self._create_api_event('aslp', body={'query': custom_query}) diff --git a/backend/compact-connect/tests/app/test_search_persistent_stack.py b/backend/compact-connect/tests/app/test_search_persistent_stack.py index f19583abf..1b57da7c2 100644 --- a/backend/compact-connect/tests/app/test_search_persistent_stack.py +++ b/backend/compact-connect/tests/app/test_search_persistent_stack.py @@ -417,6 +417,10 @@ def test_prod_index_shard_configuration(self): }, ) + # Note that the prod alarm tests specifically check for the + # differences we configure for our production environment as opposed + # to the non-prod environments. If all the sandbox alarms are properly + # configured, they are configured for prod as well, so we don't retest that here. def test_prod_storage_threshold_alarm(self): """ Test that production storage alarm threshold is set to 50% of 25GB volume (12800 MB). From 2987a08ebaa964742e9cbcf3c42e30d7478ed9f9 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 23 Dec 2025 11:49:59 -0700 Subject: [PATCH 133/137] PR feedback - remove unused field from test setup --- .../search/tests/function/test_populate_provider_documents.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py index b0fd319d5..932f79cb3 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -334,7 +334,6 @@ def test_returns_pagination_info_when_bulk_indexing_fails_after_retries(self, mo # Build the resume event from the first result resume_event = { 'startingCompact': result['resumeFrom']['startingCompact'], - 'startingLastKey': result['resumeFrom']['startingLastKey'], } # Run the second invocation From 1f49ca83fa26f9184d193be83f4676a97b8d1e85 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 23 Dec 2025 11:50:32 -0700 Subject: [PATCH 134/137] PR feedback - add note of race condition when indexing records --- backend/compact-connect/docs/design/README.md | 9 +++++++++ .../handlers/populate_provider_documents.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/backend/compact-connect/docs/design/README.md b/backend/compact-connect/docs/design/README.md index f0efea724..1a64eadfb 100644 --- a/backend/compact-connect/docs/design/README.md +++ b/backend/compact-connect/docs/design/README.md @@ -760,6 +760,15 @@ The function: } ``` +**Race Condition Consideration**: A potential race condition can occur when running this function while provider data is being actively updated: + +1. The `populate_provider_documents` Lambda function queries the current data from DynamoDB for a provider +2. A change is made in DynamoDB for that same provider +3. The DynamoDB stream handler queries the data and indexes the change into OpenSearch after the ~30 second delay of sitting in SQS +4. The `populate_provider_documents` Lambda function finally indexes the stale data into OpenSearch, overwriting the change indexed by the DynamoDB stream handler + +For this reason, it is recommended that this process be run during a period of low traffic. Given that it is a one-time process to initially populate the table, the risk is low and if needed, the Lambda function can be run again to synchronize all the provider documents. + #### Updates via DynamoDB Streams To keep the OpenSearch index synchronized with changes in the provider DynamoDB table, the system uses DynamoDB Streams to capture all modifications made to provide records (see [AWS documentation](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html)). This ensures that provider documents in OpenSearch are updated automatically whenever records are created, modified, or deleted in the provider table. diff --git a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py index a107d5a69..855016fec 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/compact-connect/lambdas/python/search/handlers/populate_provider_documents.py @@ -18,6 +18,21 @@ "startingCompact": "aslp", "startingLastKey": {"pk": "...", "sk": "..."} } + +Race Condition Consideration: +A potential race condition can occur when running this function while provider +data is being actively updated: +1. This Lambda queries the current data from DynamoDB for a provider +2. A change is made in DynamoDB for that same provider +3. The DynamoDB stream handler queries the data and indexes the change into + OpenSearch after the ~30 second delay of sitting in SQS +4. This Lambda finally indexes the stale data into OpenSearch, overwriting + the change indexed by the DynamoDB stream handler + +For this reason, it is recommended that this process be run during a period of +low traffic. Given that it is a one-time process to initially populate the +table, the risk is low and if needed, this Lambda function can be run again to +synchronize all the provider documents. """ from aws_lambda_powertools.utilities.typing import LambdaContext From 150d165b464d4c734e534465aba783018d408a47 Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 23 Dec 2025 13:45:59 -0700 Subject: [PATCH 135/137] PR feedback - clarify test comment --- .../search/tests/function/test_search_providers.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 0c3114f74..43128e2e7 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -504,12 +504,13 @@ def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_clie } self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) - # Use match_all query to simulate a bad actor attempting the broadest possible search - # across the entire OpenSearch domain. While the handler restricts searches to a single - # index based on the path parameter, this query represents an attempt to retrieve - # all providers without any filtering, which could expose providers from other compacts - # if data integrity issues exist (e.g., misconfigured index aliases or data corruption). - # or if a future feature allows cross-index searches that we are not aware of yet. + # Currently, with our safeguards in place, it is not possible for a bad actor to reach across + # indices when searching. This may change in the future with new OpenSearch features that are added + # over time. Because we don't have a valid query to trigger this branch of logic, we're just using a + # generic query here in place of some future query that can get past our safeguards and search provider + # data across compact indices. The mock above is returning a provider from a different compact to + # trigger the branch of logic where we catch this discrepancy and fail with an error log and 400 + # response custom_query = {'match_all': {}} # Request for 'aslp' compact but provider has 'octp' compact From abd12288594a3838c42eeb8d8b368bbe5734137d Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 23 Dec 2025 13:48:06 -0700 Subject: [PATCH 136/137] update other test comment --- .../search/tests/function/test_search_privileges.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 2d8cd299e..67033642c 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -815,12 +815,13 @@ def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_cli } self._when_testing_mock_opensearch_client(mock_opensearch_client, search_response=search_response) - # Use match_all query to simulate a bad actor attempting the broadest possible search - # across the entire OpenSearch domain. While the handler restricts searches to a single - # index based on the path parameter, this query represents an attempt to retrieve - # all providers without any filtering, which could expose providers from other compacts - # if data integrity issues exist (e.g., misconfigured index aliases or data corruption). - # or if a future feature allows cross-index searches that we are not aware of yet. + # Currently, with our safeguards in place, it is not possible for a bad actor to reach across + # indices when searching. This may change in the future with new OpenSearch features that are added + # over time. Because we don't have a valid query to trigger this branch of logic, we're just using a + # generic query here in place of some future query that can get past our safeguards and search provider + # data across compact indices. The mock above is returning a provider from a different compact to + # trigger the branch of logic where we catch this discrepancy and fail with an error log and 400 + # response custom_query = {'match_all': {}} # Request for 'aslp' compact but privilege has 'octp' compact From eb63ee809e35fc4a59ae285d9de5ea8cf95d82de Mon Sep 17 00:00:00 2001 From: Landon Shumway Date: Tue, 23 Dec 2025 14:12:43 -0700 Subject: [PATCH 137/137] filter mismatched records from search results rather than returning a 400 which would notify a bad actor that a match was made with their query, we will filter the record from the results. --- .../lambdas/python/search/handlers/search.py | 9 ++++++--- .../search/tests/function/test_search_privileges.py | 13 ++++++------- .../search/tests/function/test_search_providers.py | 13 +++++++------ 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/backend/compact-connect/lambdas/python/search/handlers/search.py b/backend/compact-connect/lambdas/python/search/handlers/search.py index 74d0d9259..bdc5bed74 100644 --- a/backend/compact-connect/lambdas/python/search/handlers/search.py +++ b/backend/compact-connect/lambdas/python/search/handlers/search.py @@ -138,7 +138,9 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus provider_compact=sanitized_provider.get('compact'), path_compact=compact, ) - raise CCInvalidRequestException('Invalid request body') + # do not include the provider in the results + total['value'] -= 1 + continue sanitized_providers.append(sanitized_provider) # Track the sort values from the last hit for search_after pagination last_sort = hit.get('sort') @@ -150,7 +152,7 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus errors=e.messages, ) - # Build response following OpenSearch DSL structure + # Build response response_body = { 'providers': sanitized_providers, 'total': total, @@ -264,7 +266,8 @@ def _export_privileges(event: dict, context: LambdaContext): # noqa: ARG001 unu privilege_compact=sanitized_privilege.get('compact'), path_compact=compact, ) - raise CCInvalidRequestException('Invalid request body') + # do not include the privilege in the results + continue flattened_privileges.append(sanitized_privilege) except ValidationError as e: logger.error( diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py index 67033642c..3ada04c72 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_privileges.py @@ -268,7 +268,6 @@ def test_privilege_export_skips_provider_without_privileges_returns_404(self, mo body = json.loads(response['body']) # Verify response contains error message - self.assertIn('message', body) self.assertEqual('The search parameters did not match any privileges.', body['message']) # Verify no CSV file was uploaded to S3 @@ -743,8 +742,8 @@ def test_export_query_with_nested_index_key_returns_400(self): self.assertIn("'index'", body['message']) @patch('handlers.search.opensearch_client') - def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_client): - """Test that a privilege with a compact field that doesn't match the path parameter returns 400.""" + def test_privilege_with_mismatched_compact_is_filtered_from_response(self, mock_opensearch_client): + """Test that a privilege with a compact field that doesn't match the path parameter is filtered from results.""" from handlers.search import search_api_handler provider_id = '00000000-0000-0000-0000-000000000001' @@ -820,8 +819,8 @@ def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_cli # over time. Because we don't have a valid query to trigger this branch of logic, we're just using a # generic query here in place of some future query that can get past our safeguards and search provider # data across compact indices. The mock above is returning a provider from a different compact to - # trigger the branch of logic where we catch this discrepancy and fail with an error log and 400 - # response + # trigger the branch of logic where we catch this discrepancy, log the error so an alert fires, and + # filter the document from the response custom_query = {'match_all': {}} # Request for 'aslp' compact but privilege has 'octp' compact @@ -829,6 +828,6 @@ def test_privilege_with_mismatched_compact_returns_400(self, mock_opensearch_cli response = search_api_handler(event, self.mock_context) - self.assertEqual(400, response['statusCode']) + self.assertEqual(404, response['statusCode']) body = json.loads(response['body']) - self.assertEqual('Invalid request body', body['message']) + self.assertEqual('The search parameters did not match any privileges.', body['message']) diff --git a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py index 43128e2e7..63860110b 100644 --- a/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/compact-connect/lambdas/python/search/tests/function/test_search_providers.py @@ -470,8 +470,8 @@ def test_opensearch_request_error_returns_400_with_error_message(self, mock_open self.assertEqual(error_reason, body['message']) @patch('handlers.search.opensearch_client') - def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_client): - """Test that a provider with a compact field that doesn't match the path parameter returns 400.""" + def test_provider_with_mismatched_compact_is_filtered_from_response(self, mock_opensearch_client): + """Test that a provider with a compact field that doesn't match the path parameter is filtered from results.""" from handlers.search import search_api_handler # Create a provider hit with a different compact than the path parameter @@ -509,8 +509,8 @@ def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_clie # over time. Because we don't have a valid query to trigger this branch of logic, we're just using a # generic query here in place of some future query that can get past our safeguards and search provider # data across compact indices. The mock above is returning a provider from a different compact to - # trigger the branch of logic where we catch this discrepancy and fail with an error log and 400 - # response + # trigger the branch of logic where we catch this discrepancy, log the error so an alert fires, and + # filter the document from the response custom_query = {'match_all': {}} # Request for 'aslp' compact but provider has 'octp' compact @@ -518,6 +518,7 @@ def test_provider_with_mismatched_compact_returns_400(self, mock_opensearch_clie response = search_api_handler(event, self.mock_context) - self.assertEqual(400, response['statusCode']) + self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - self.assertEqual('Invalid request body', body['message']) + # should be empty list with total value of 0 + self.assertEqual({'providers': [], 'total': {'relation': 'eq', 'value': 0}}, body)