diff --git a/backend/cosmetology-app/bin/run_python_tests.py b/backend/cosmetology-app/bin/run_python_tests.py index 2d35ad977..22270123c 100755 --- a/backend/cosmetology-app/bin/run_python_tests.py +++ b/backend/cosmetology-app/bin/run_python_tests.py @@ -30,6 +30,7 @@ 'lambdas/python/disaster-recovery', 'lambdas/python/migration', 'lambdas/python/provider-data-v1', + 'lambdas/python/search', 'lambdas/python/staff-user-pre-token', 'lambdas/python/staff-users', '.', # CDK tests diff --git a/backend/cosmetology-app/docs/internal/api-specification/latest-oas30.json b/backend/cosmetology-app/docs/internal/api-specification/latest-oas30.json index 9f8978af4..c69959e20 100644 --- a/backend/cosmetology-app/docs/internal/api-specification/latest-oas30.json +++ b/backend/cosmetology-app/docs/internal/api-specification/latest-oas30.json @@ -2533,7 +2533,6 @@ "jurisdictionAdverseActionsNotificationEmails", "jurisdictionName", "jurisdictionOperationsTeamEmails", - "jurisdictionSummaryReportNotificationEmails", "licenseeRegistrationEnabled", "postalAbbreviation" ], @@ -2573,14 +2572,6 @@ "jurisdictionName": { "type": "string", "description": "The name of the jurisdiction" - }, - "jurisdictionSummaryReportNotificationEmails": { - "type": "array", - "description": "List of email addresses for summary report notifications", - "items": { - "type": "string", - "format": "email" - } } } }, @@ -2745,7 +2736,6 @@ "required": [ "jurisdictionAdverseActionsNotificationEmails", "jurisdictionOperationsTeamEmails", - "jurisdictionSummaryReportNotificationEmails", "licenseeRegistrationEnabled" ], "type": "object", @@ -2775,17 +2765,6 @@ "licenseeRegistrationEnabled": { "type": "boolean", "description": "Denotes whether licensee registration is enabled" - }, - "jurisdictionSummaryReportNotificationEmails": { - "maxItems": 10, - "minItems": 1, - "uniqueItems": true, - "type": "array", - "description": "List of email addresses for summary report notifications", - "items": { - "type": "string", - "format": "email" - } } }, "additionalProperties": false @@ -4443,7 +4422,6 @@ "compactAdverseActionsNotificationEmails", "compactName", "compactOperationsTeamEmails", - "compactSummaryReportNotificationEmails", "configuredStates", "licenseeRegistrationEnabled" ], @@ -4482,14 +4460,6 @@ } } }, - "compactSummaryReportNotificationEmails": { - "type": "array", - "description": "List of email addresses for summary report notifications", - "items": { - "type": "string", - "format": "email" - } - }, "compactAdverseActionsNotificationEmails": { "type": "array", "description": "List of email addresses for adverse actions notifications", @@ -4524,7 +4494,6 @@ "required": [ "compactAdverseActionsNotificationEmails", "compactOperationsTeamEmails", - "compactSummaryReportNotificationEmails", "configuredStates", "licenseeRegistrationEnabled" ], @@ -4564,17 +4533,6 @@ "additionalProperties": false } }, - "compactSummaryReportNotificationEmails": { - "maxItems": 10, - "minItems": 1, - "uniqueItems": true, - "type": "array", - "description": "List of email addresses for summary report notifications", - "items": { - "type": "string", - "format": "email" - } - }, "compactAdverseActionsNotificationEmails": { "maxItems": 10, "minItems": 1, diff --git a/backend/cosmetology-app/docs/internal/postman/postman-collection.json b/backend/cosmetology-app/docs/internal/postman/postman-collection.json index 6c6647cff..006731546 100644 --- a/backend/cosmetology-app/docs/internal/postman/postman-collection.json +++ b/backend/cosmetology-app/docs/internal/postman/postman-collection.json @@ -444,7 +444,7 @@ "response": [ { "_postman_previewlanguage": "json", - "body": "{\n \"compactAbbr\": \"\",\n \"compactAdverseActionsNotificationEmails\": [\n \"\",\n \"\"\n ],\n \"compactName\": \"\",\n \"compactOperationsTeamEmails\": [\n \"\",\n \"\"\n ],\n \"compactSummaryReportNotificationEmails\": [\n \"\",\n \"\"\n ],\n \"configuredStates\": [\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"wa\"\n },\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"va\"\n }\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}", + "body": "{\n \"compactAbbr\": \"\",\n \"compactAdverseActionsNotificationEmails\": [\n \"\",\n \"\"\n ],\n \"compactName\": \"\",\n \"compactOperationsTeamEmails\": [\n \"\",\n \"\"\n ],\n \"configuredStates\": [\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"wa\"\n },\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"va\"\n }\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}", "code": 200, "cookie": [], "header": [ @@ -505,7 +505,7 @@ "language": "json" } }, - "raw": "{\n \"compactAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"compactOperationsTeamEmails\": [\n \"\"\n ],\n \"compactSummaryReportNotificationEmails\": [\n \"\"\n ],\n \"configuredStates\": [\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"wa\"\n },\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"ks\"\n }\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" + "raw": "{\n \"compactAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"compactOperationsTeamEmails\": [\n \"\"\n ],\n \"configuredStates\": [\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"wa\"\n },\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"ks\"\n }\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" }, "description": {}, "header": [ @@ -567,7 +567,7 @@ "language": "json" } }, - "raw": "{\n \"compactAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"compactOperationsTeamEmails\": [\n \"\"\n ],\n \"compactSummaryReportNotificationEmails\": [\n \"\"\n ],\n \"configuredStates\": [\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"wa\"\n },\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"ks\"\n }\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" + "raw": "{\n \"compactAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"compactOperationsTeamEmails\": [\n \"\"\n ],\n \"configuredStates\": [\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"wa\"\n },\n {\n \"isLive\": \"\",\n \"postalAbbreviation\": \"ks\"\n }\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" }, "header": [ { @@ -760,7 +760,7 @@ "response": [ { "_postman_previewlanguage": "json", - "body": "{\n \"compact\": \"cosm\",\n \"jurisdictionAdverseActionsNotificationEmails\": [\n \"\",\n \"\"\n ],\n \"jurisdictionName\": \"\",\n \"jurisdictionOperationsTeamEmails\": [\n \"\",\n \"\"\n ],\n \"jurisdictionSummaryReportNotificationEmails\": [\n \"\",\n \"\"\n ],\n \"licenseeRegistrationEnabled\": \"\",\n \"postalAbbreviation\": \"\"\n}", + "body": "{\n \"compact\": \"cosm\",\n \"jurisdictionAdverseActionsNotificationEmails\": [\n \"\",\n \"\"\n ],\n \"jurisdictionName\": \"\",\n \"jurisdictionOperationsTeamEmails\": [\n \"\",\n \"\"\n ],\n \"licenseeRegistrationEnabled\": \"\",\n \"postalAbbreviation\": \"\"\n}", "code": 200, "cookie": [], "header": [ @@ -823,7 +823,7 @@ "language": "json" } }, - "raw": "{\n \"jurisdictionAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"jurisdictionOperationsTeamEmails\": [\n \"\"\n ],\n \"jurisdictionSummaryReportNotificationEmails\": [\n \"\"\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" + "raw": "{\n \"jurisdictionAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"jurisdictionOperationsTeamEmails\": [\n \"\"\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" }, "description": {}, "header": [ @@ -897,7 +897,7 @@ "language": "json" } }, - "raw": "{\n \"jurisdictionAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"jurisdictionOperationsTeamEmails\": [\n \"\"\n ],\n \"jurisdictionSummaryReportNotificationEmails\": [\n \"\"\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" + "raw": "{\n \"jurisdictionAdverseActionsNotificationEmails\": [\n \"\"\n ],\n \"jurisdictionOperationsTeamEmails\": [\n \"\"\n ],\n \"licenseeRegistrationEnabled\": \"\"\n}" }, "header": [ { diff --git a/backend/cosmetology-app/lambdas/nodejs/email-notification-service/README.md b/backend/cosmetology-app/lambdas/nodejs/email-notification-service/README.md index 92294d2cd..bfaa95acc 100644 --- a/backend/cosmetology-app/lambdas/nodejs/email-notification-service/README.md +++ b/backend/cosmetology-app/lambdas/nodejs/email-notification-service/README.md @@ -11,10 +11,8 @@ The lambda is intended to be invoked directly, rather than through an API endpoi recipientType: // must be one of the following | 'COMPACT_OPERATIONS_TEAM' // compactOperationsTeamEmails | 'COMPACT_ADVERSE_ACTIONS' // compactAdverseActionsNotificationEmails - | 'COMPACT_SUMMARY_REPORT' // compactSummaryReportNotificationEmails | 'JURISDICTION_OPERATIONS_TEAM' // jurisdictionOperationsTeamEmails | 'JURISDICTION_ADVERSE_ACTIONS' // jurisdictionAdverseActionsNotificationEmails - | 'JURISDICTION_SUMMARY_REPORT' // jurisdictionSummaryReportNotificationEmails | 'SPECIFIC'; // specificEmails provided in payload compact: string; // Compact identifier jurisdiction?: string; // Optional jurisdiction identifier, must be specified if sending to a Jurisdiction based email list diff --git a/backend/cosmetology-app/lambdas/nodejs/lib/models/compact.ts b/backend/cosmetology-app/lambdas/nodejs/lib/models/compact.ts index 29a0112e4..0c6c75d90 100644 --- a/backend/cosmetology-app/lambdas/nodejs/lib/models/compact.ts +++ b/backend/cosmetology-app/lambdas/nodejs/lib/models/compact.ts @@ -11,7 +11,6 @@ export interface Compact { compactAbbr: string; compactName: string; compactOperationsTeamEmails: string[]; - compactSummaryReportNotificationEmails: string[]; dateOfUpdate: string; type: string; } diff --git a/backend/cosmetology-app/lambdas/nodejs/lib/models/email-notification-service-event.ts b/backend/cosmetology-app/lambdas/nodejs/lib/models/email-notification-service-event.ts index 3352861b3..f4f73c20b 100644 --- a/backend/cosmetology-app/lambdas/nodejs/lib/models/email-notification-service-event.ts +++ b/backend/cosmetology-app/lambdas/nodejs/lib/models/email-notification-service-event.ts @@ -1,10 +1,8 @@ export type RecipientType = | 'COMPACT_OPERATIONS_TEAM' | 'COMPACT_ADVERSE_ACTIONS' - | 'COMPACT_SUMMARY_REPORT' | 'JURISDICTION_OPERATIONS_TEAM' | 'JURISDICTION_ADVERSE_ACTIONS' - | 'JURISDICTION_SUMMARY_REPORT' | 'SPECIFIC'; export interface EmailNotificationEvent { diff --git a/backend/cosmetology-app/lambdas/nodejs/lib/models/jurisdiction.ts b/backend/cosmetology-app/lambdas/nodejs/lib/models/jurisdiction.ts index 90b142188..a83df1eec 100644 --- a/backend/cosmetology-app/lambdas/nodejs/lib/models/jurisdiction.ts +++ b/backend/cosmetology-app/lambdas/nodejs/lib/models/jurisdiction.ts @@ -7,5 +7,4 @@ export interface IJurisdiction { compact: string; jurisdictionOperationsTeamEmails: string[]; jurisdictionAdverseActionsNotificationEmails: string[]; - jurisdictionSummaryReportNotificationEmails: string[]; } diff --git a/backend/cosmetology-app/lambdas/nodejs/tests/email-notification-service.test.ts b/backend/cosmetology-app/lambdas/nodejs/tests/email-notification-service.test.ts index cd6e72ebb..0307f907a 100644 --- a/backend/cosmetology-app/lambdas/nodejs/tests/email-notification-service.test.ts +++ b/backend/cosmetology-app/lambdas/nodejs/tests/email-notification-service.test.ts @@ -33,7 +33,6 @@ const SAMPLE_COMPACT_CONFIGURATION = { 'compactAbbr': { S: 'cosm' }, 'compactName': { S: 'Audiology and Speech Language Pathology' }, 'compactOperationsTeamEmails': { L: [{ S: 'operations@example.com' }]}, - 'compactSummaryReportNotificationEmails': { L: [{ S: 'summary@example.com' }]}, 'dateOfUpdate': { S: '2024-12-10T19:27:28+00:00' }, 'type': { S: 'compact' } }; @@ -42,7 +41,6 @@ const SAMPLE_JURISDICTION_CONFIGURATION = { 'pk': { S: 'cosm#CONFIGURATION' }, 'sk': { S: 'cosm#JURISDICTION#oh' }, 'jurisdictionName': { S: 'Ohio' }, - 'jurisdictionSummaryReportNotificationEmails': { L: [{ S: 'ohio@example.com' }]}, 'type': { S: 'jurisdiction' } }; diff --git a/backend/cosmetology-app/lambdas/nodejs/tests/lib/compact-configuration-client.test.ts b/backend/cosmetology-app/lambdas/nodejs/tests/lib/compact-configuration-client.test.ts index ed511eafd..2b3e5844e 100644 --- a/backend/cosmetology-app/lambdas/nodejs/tests/lib/compact-configuration-client.test.ts +++ b/backend/cosmetology-app/lambdas/nodejs/tests/lib/compact-configuration-client.test.ts @@ -17,7 +17,6 @@ const SAMPLE_COMPACT_CONFIGURATION = { 'compactAbbr': { S: 'cosm' }, 'compactName': { S: 'Audiology and Speech Language Pathology' }, 'compactOperationsTeamEmails': { L: [{ S: 'operations@example.com' }]}, - 'compactSummaryReportNotificationEmails': { L: [{ S: 'summary@example.com' }]}, 'dateOfUpdate': { S: '2024-12-10T19:27:28+00:00' }, 'type': { S: 'compact' } }; @@ -77,7 +76,6 @@ describe('CompactConfigurationClient', () => { compactAbbr: 'cosm', compactName: 'Audiology and Speech Language Pathology', compactOperationsTeamEmails: ['operations@example.com'], - compactSummaryReportNotificationEmails: ['summary@example.com'], dateOfUpdate: '2024-12-10T19:27:28+00:00', type: 'compact' }); diff --git a/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/encumbrance-notification-service.test.ts b/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/encumbrance-notification-service.test.ts index b4ec71e68..4f6336fbc 100644 --- a/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/encumbrance-notification-service.test.ts +++ b/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/encumbrance-notification-service.test.ts @@ -22,7 +22,6 @@ const SAMPLE_COMPACT_CONFIG: Compact = { compactAbbr: 'cosm', compactName: 'Audiology and Speech Language Pathology', compactOperationsTeamEmails: ['operations@example.com'], - compactSummaryReportNotificationEmails: ['summary@example.com'], dateOfUpdate: '2024-12-10T19:27:28+00:00', type: 'compact' }; @@ -34,8 +33,7 @@ const SAMPLE_JURISDICTION_CONFIG = { postalAbbreviation: 'oh', compact: 'cosm', jurisdictionOperationsTeamEmails: ['oh-ops@example.com'], - jurisdictionAdverseActionsNotificationEmails: ['oh-adverse@example.com'], - jurisdictionSummaryReportNotificationEmails: ['oh-summary@example.com'] + jurisdictionAdverseActionsNotificationEmails: ['oh-adverse@example.com'] }; const asSESClient = (mock: ReturnType) => diff --git a/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/investigation-notification-service.test.ts b/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/investigation-notification-service.test.ts index bfcd333ea..deb28c72a 100644 --- a/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/investigation-notification-service.test.ts +++ b/backend/cosmetology-app/lambdas/nodejs/tests/lib/email/investigation-notification-service.test.ts @@ -22,7 +22,6 @@ const SAMPLE_COMPACT_CONFIG: Compact = { compactAbbr: 'cosm', compactName: 'Audiology and Speech Language Pathology', compactOperationsTeamEmails: ['operations@example.com'], - compactSummaryReportNotificationEmails: ['summary@example.com'], dateOfUpdate: '2024-12-10T19:27:28+00:00', type: 'compact' }; @@ -34,8 +33,7 @@ const SAMPLE_JURISDICTION_CONFIG = { postalAbbreviation: 'oh', compact: 'cosm', jurisdictionOperationsTeamEmails: ['oh-ops@example.com'], - jurisdictionAdverseActionsNotificationEmails: ['oh-adverse@example.com'], - jurisdictionSummaryReportNotificationEmails: ['oh-summary@example.com'] + jurisdictionAdverseActionsNotificationEmails: ['oh-adverse@example.com'] }; const asSESClient = (mock: ReturnType) => diff --git a/backend/cosmetology-app/lambdas/nodejs/tests/lib/jurisdiction-client.test.ts b/backend/cosmetology-app/lambdas/nodejs/tests/lib/jurisdiction-client.test.ts index 9b420ec1b..9ae718d47 100644 --- a/backend/cosmetology-app/lambdas/nodejs/tests/lib/jurisdiction-client.test.ts +++ b/backend/cosmetology-app/lambdas/nodejs/tests/lib/jurisdiction-client.test.ts @@ -34,9 +34,6 @@ const SAMPLE_JURISDICTION_ITEMS = [ } ] }, - 'jurisdictionSummaryReportNotificationEmails': { - 'L': [] - }, 'jurisprudenceRequirements': { 'M': { 'required': { @@ -77,9 +74,6 @@ const SAMPLE_JURISDICTION_ITEMS = [ } ] }, - 'jurisdictionSummaryReportNotificationEmails': { - 'L': [] - }, 'jurisprudenceRequirements': { 'M': { 'required': { diff --git a/backend/cosmetology-app/lambdas/nodejs/tests/sample-records.ts b/backend/cosmetology-app/lambdas/nodejs/tests/sample-records.ts index 724521ec1..db901627d 100644 --- a/backend/cosmetology-app/lambdas/nodejs/tests/sample-records.ts +++ b/backend/cosmetology-app/lambdas/nodejs/tests/sample-records.ts @@ -256,9 +256,6 @@ export const SAMPLE_JURISDICTION_CONFIGURATION = { } ] }, - 'jurisdictionSummaryReportNotificationEmails': { - 'L': [] - }, 'jurisprudenceRequirements': { 'M': { 'required': { @@ -283,7 +280,6 @@ export const SAMPLE_UNMARSHALLED_JURISDICTION_CONFIGURATION = { 'jurisdictionName': 'Ohio', 'jurisdictionOperationsTeamEmails': [ 'justin@inspiringapps.com' ], - 'jurisdictionSummaryReportNotificationEmails': [], 'jurisprudenceRequirements': { 'required': true }, @@ -304,7 +300,6 @@ export const SAMPLE_COMPACT_CONFIGURATION = { 'compactAbbr': { 'S': 'cosm' }, 'compactName': { 'S': 'Audiology and Speech Language Pathology' }, 'compactOperationsTeamEmails': { 'L': [{ 'S': 'compact-ops@example.com' }]}, - 'compactSummaryReportNotificationEmails': { 'L': [{ 'S': 'summary@example.com' }]}, 'dateOfUpdate': { 'S': '2024-12-10T19:27:28+00:00' }, 'type': { 'S': 'compact' } }; @@ -320,7 +315,6 @@ export const SAMPLE_UNMARSHALLED_COMPACT_CONFIGURATION = { 'compactAbbr': 'cosm', 'compactName': 'Audiology and Speech Language Pathology', 'compactOperationsTeamEmails': ['compact-ops@example.com'], - 'compactSummaryReportNotificationEmails': ['summary@example.com'], 'dateOfUpdate': '2024-12-10T19:27:28+00:00', 'type': 'compact' }; diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/provider_record_util.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/provider_record_util.py index accd24ec0..5765afc9b 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/provider_record_util.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/provider_record_util.py @@ -424,15 +424,23 @@ def find_best_license_in_current_known_licenses( ) return sorted_licenses[0] - def generate_privileges_for_provider(self) -> list[dict]: + def generate_privileges_for_provider(self, include_inactive_privileges: bool = False) -> list[dict]: """ Generate privilege dicts at runtime for all eligible license types this provider holds. For each license type, the home license is chosen from all licenses of that type: the license renewed most recently (when dateOfRenewal is present), otherwise the license with the most recent date of issuance. - Privileges are generated for that type only if the chosen home license is compact-eligible. - For each such type, one privilege is generated per active compact jurisdiction + By default, privileges are generated only when the chosen home license is compact-eligible. + + When include_inactive_privileges is True, privileges are also generated for ineligible home + licenses and are marked inactive. This is primarily used when indexing to OpenSearch so that adverse + actions and investigations remain searchable even when a license is ineligible. + + For each qualifying type, one privilege is generated per active compact jurisdiction (excluding the home jurisdiction). + + :param include_inactive_privileges: When True, generate privileges for ineligible home licenses + and mark them inactive instead of omitting them entirely. """ if not self._license_records: return [] @@ -458,14 +466,18 @@ def generate_privileges_for_provider(self) -> list[dict]: reverse=True, ) most_recent_license = sorted_licenses[0] - # If the most recently renewed/issued license is not compact eligible, - # we will not generate privileges for it - if most_recent_license.compactEligibility != CompactEligibilityStatus.ELIGIBLE: + if ( + not include_inactive_privileges + and most_recent_license.compactEligibility != CompactEligibilityStatus.ELIGIBLE + ): + logger.debug('skipping inactive license', + license_jurisdiction=most_recent_license.jurisdiction, license_type=most_recent_license.licenseType) continue most_recent_licenses_for_each_type.append(most_recent_license) result: list[dict] = [] for most_recent_license in most_recent_licenses_for_each_type: + is_eligible = most_recent_license.compactEligibility == CompactEligibilityStatus.ELIGIBLE home_jurisdiction = most_recent_license.jurisdiction.lower() license_type_abbr = most_recent_license.licenseTypeAbbreviation @@ -486,12 +498,10 @@ def generate_privileges_for_provider(self) -> list[dict]: 'licenseJurisdiction': home_jurisdiction, 'licenseType': most_recent_license.licenseType, 'dateOfExpiration': most_recent_license.dateOfExpiration, - # the only way a privilege under this model shows inactive is if - # there has been an encumbrance set by a state admin that has not been - # lifted. If the license itself is inactive or ineligible for whatever reason, we don't - # return any associated privilege objects + # A privilege is inactive if the home license is ineligible, or if a state admin + # has set an encumbrance that has not been lifted. 'status': ActiveInactiveStatus.ACTIVE.value - if not privilege_unlifted + if is_eligible and not privilege_unlifted else ActiveInactiveStatus.INACTIVE.value, 'adverseActions': [aa.to_dict() for aa in privilege_aa], 'investigations': [inv.to_dict() for inv in inv_records], @@ -594,3 +604,68 @@ def generate_api_response_object(self) -> dict: provider['privileges'] = privileges return provider + + def generate_opensearch_documents(self) -> list[dict]: + """ + Generate one OpenSearch document per license for this provider. + + Each document contains the full provider-level fields, a single license in the `licenses` + array, and privileges only if that license is the home license for its type. This enables + 1:1 mapping between OpenSearch documents and license records for native pagination. + + Privileges are always included for home license documents — including when the license is + ineligible — so that adverse actions and investigations remain linked to privilege records. + Privileges for ineligible home licenses carry status 'inactive'. + + :return: A list of dicts, each representing a single-license OpenSearch document. + Empty list if the provider has no licenses. + """ + if not self._license_records: + return [] + + provider_dict = self.get_provider_record().to_dict() + all_privileges = self.generate_privileges_for_provider(include_inactive_privileges=True) + + # Determine the home license for each license type using the same sort logic + # as generate_privileges_for_provider, so privilege assignment is consistent. + by_type: dict[str, list] = {} + for lic in self._license_records: + by_type.setdefault(lic.licenseType, []).append(lic) + + home_licenses: set[tuple[str, str]] = set() + for _lt, licenses in by_type.items(): + sorted_licenses = sorted( + licenses, + key=ProviderRecordUtility._license_sort_key, # noqa: SLF001 + reverse=True, + ) + home = sorted_licenses[0] + home_licenses.add((home.jurisdiction.lower(), home.licenseType)) + + documents = [] + for license_record in self._license_records: + license_dict = license_record.to_dict() + license_dict['adverseActions'] = [ + rec.to_dict() + for rec in self.get_adverse_action_records_for_license( + license_record.jurisdiction, license_record.licenseTypeAbbreviation + ) + ] + license_dict['investigations'] = [ + rec.to_dict() + for rec in self.get_investigation_records_for_license( + license_record.jurisdiction, license_record.licenseTypeAbbreviation + ) + ] + + is_home = (license_record.jurisdiction.lower(), license_record.licenseType) in home_licenses + license_privileges = ( + [p for p in all_privileges if p['licenseType'] == license_record.licenseType] if is_home else [] + ) + + doc = dict(provider_dict) + doc['licenses'] = [license_dict] + doc['privileges'] = license_privileges + documents.append(doc) + + return documents diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/__init__.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/__init__.py index 3314de6c4..78af488bf 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/__init__.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/__init__.py @@ -29,10 +29,6 @@ def compact_operations_team_emails(self) -> list[str] | None: def compact_adverse_actions_notification_emails(self) -> list[str] | None: return self.get('compactAdverseActionsNotificationEmails') - @property - def compact_summary_report_notification_emails(self) -> list[str] | None: - return self.get('compactSummaryReportNotificationEmails') - @property def licensee_registration_enabled(self): return self.get('licenseeRegistrationEnabled', False) @@ -67,10 +63,6 @@ def compactOperationsTeamEmails(self) -> list[str]: def compactAdverseActionsNotificationEmails(self) -> list[str]: return self._data.get('compactAdverseActionsNotificationEmails', []) - @property - def compactSummaryReportNotificationEmails(self) -> list[str]: - return self._data.get('compactSummaryReportNotificationEmails', []) - @property def licenseeRegistrationEnabled(self) -> bool: return self._data.get('licenseeRegistrationEnabled', False) diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/api.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/api.py index fe1847420..61743f8f7 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/api.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/api.py @@ -21,11 +21,6 @@ class CompactConfigurationResponseSchema(ForgivingSchema): required=True, allow_none=False, ) - compactSummaryReportNotificationEmails = List( - Email(required=True, allow_none=False), - required=True, - allow_none=False, - ) licenseeRegistrationEnabled = Boolean(required=True, allow_none=False) configuredStates = List(Nested(ConfiguredStateSchema()), required=True, allow_none=False) @@ -39,9 +34,6 @@ class PutCompactConfigurationRequestSchema(Schema): compactAdverseActionsNotificationEmails = List( Email(required=True, allow_none=False), required=True, allow_none=False, validate=Length(min=1) ) - compactSummaryReportNotificationEmails = List( - Email(required=True, allow_none=False), required=True, allow_none=False, validate=Length(min=1) - ) licenseeRegistrationEnabled = Boolean(required=True, allow_none=False) configuredStates = List(Nested(ConfiguredStateSchema()), required=True, allow_none=False) diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/record.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/record.py index bec69f9c1..db3e19fdf 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/record.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/compact/record.py @@ -27,11 +27,6 @@ class CompactRecordSchema(BaseRecordSchema): required=True, allow_none=False, ) - compactSummaryReportNotificationEmails = List( - String(required=True, allow_none=False), - required=True, - allow_none=False, - ) licenseeRegistrationEnabled = Boolean(required=True, allow_none=False) # List of states that have submitted configurations and their live status configuredStates = List(Nested(ConfiguredStateSchema()), required=True, allow_none=False) diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/__init__.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/__init__.py index 6a59f2db5..aca2eebce 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/__init__.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/__init__.py @@ -37,10 +37,6 @@ def jurisdictionOperationsTeamEmails(self) -> list[str]: def jurisdictionAdverseActionsNotificationEmails(self) -> list[str]: return self._data.get('jurisdictionAdverseActionsNotificationEmails', []) - @property - def jurisdictionSummaryReportNotificationEmails(self) -> list[str]: - return self._data.get('jurisdictionSummaryReportNotificationEmails', []) - @property def licenseeRegistrationEnabled(self) -> bool: return self._data.get('licenseeRegistrationEnabled', False) diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/api.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/api.py index 54033c541..a1181b6c8 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/api.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/api.py @@ -44,11 +44,6 @@ class CompactJurisdictionConfigurationResponseSchema(ForgivingSchema): required=True, allow_none=False, ) - jurisdictionSummaryReportNotificationEmails = List( - Email(required=True, allow_none=False), - required=True, - allow_none=False, - ) licenseeRegistrationEnabled = Boolean(required=True, allow_none=False) @@ -65,6 +60,3 @@ class PutCompactJurisdictionConfigurationRequestSchema(Schema): jurisdictionAdverseActionsNotificationEmails = List( Email(required=True, allow_none=False), required=True, allow_none=False, validate=Length(min=1) ) - jurisdictionSummaryReportNotificationEmails = List( - Email(required=True, allow_none=False), required=True, allow_none=False, validate=Length(min=1) - ) diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/record.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/record.py index eb42183a0..4e2342323 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/record.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/jurisdiction/record.py @@ -24,11 +24,6 @@ class JurisdictionRecordSchema(BaseRecordSchema): required=True, allow_none=False, ) - jurisdictionSummaryReportNotificationEmails = List( - Email(required=True, allow_none=False), - required=True, - allow_none=False, - ) licenseeRegistrationEnabled = Boolean(required=True, allow_none=False) # Generated fields diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/license/api.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/license/api.py index de42a9d7f..0e4db7d86 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/license/api.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/license/api.py @@ -228,3 +228,18 @@ class LicenseReadPrivateResponseSchema(LicenseExpirationStatusMixin, ForgivingSc # these fields are specific to the read private role dateOfBirth = Raw(required=False, allow_none=False) ssnLastFour = String(required=False, allow_none=False, validate=Length(equal=4)) + + +class LicenseOpenSearchDocumentSchema(LicenseGeneralResponseSchema): + """ + License object fields for OpenSearch document indexing. + + Extends LicenseGeneralResponseSchema with the dateOfBirth field to enable + authorized staff users to search providers by date of birth. This schema + is used only for indexing into OpenSearch, not for API responses. + + Serialization direction: + Python -> load() -> OpenSearch document + """ + + dateOfBirth = Raw(required=False, allow_none=False) diff --git a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/provider/api.py b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/provider/api.py index fd823febb..11f6176cf 100644 --- a/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/provider/api.py +++ b/backend/cosmetology-app/lambdas/python/common/cc_common/data_model/schema/provider/api.py @@ -1,5 +1,5 @@ # ruff: noqa: N801, N815, ARG002 invalid-name unused-argument -from marshmallow import ValidationError, validates_schema +from marshmallow import ValidationError, post_load, validates_schema from marshmallow.fields import Integer, List, Nested, Raw, String from marshmallow.validate import Length, Range, Regexp @@ -14,6 +14,7 @@ ) from cc_common.data_model.schema.license.api import ( LicenseGeneralResponseSchema, + LicenseOpenSearchDocumentSchema, LicenseReadPrivateResponseSchema, ) from cc_common.data_model.schema.privilege.api import ( @@ -159,6 +160,21 @@ class ProviderGeneralResponseSchema(ForgivingSchema): privileges = List(Nested(PrivilegeGeneralResponseSchema(), required=False, allow_none=False)) +class ProviderOpenSearchDocumentSchema(ProviderGeneralResponseSchema): + """ + Provider object fields for OpenSearch document indexing. + + Extends ProviderGeneralResponseSchema with license objects that include dateOfBirth, + enabling authorized staff users to search providers by date of birth. This schema + is used only for indexing into OpenSearch, not for API responses. + + Serialization direction: + Python -> load() -> OpenSearch document + """ + + licenses = List(Nested(LicenseOpenSearchDocumentSchema(), required=False, allow_none=False)) + + class ProviderPublicResponseSchema(ForgivingSchema): """ Provider object fields that are sanitized for the public lookup endpoints. @@ -192,6 +208,27 @@ class ProviderPublicResponseSchema(ForgivingSchema): # Note the lack of `licenses` here: we do not return license data for public endpoints +class PublicLicenseSearchResponseSchema(ForgivingSchema): + """ + License object fields returned by the public query providers endpoint (OpenSearch-backed). + Used to sanitize license records extracted from inner_hits; jurisdiction is renamed to licenseJurisdiction. + """ + + providerId = Raw(required=True, allow_none=False) + givenName = String(required=True, allow_none=False, validate=Length(1, 100)) + familyName = String(required=True, allow_none=False, validate=Length(1, 100)) + jurisdiction = String(required=False, allow_none=False, load_only=True) # OpenSearch uses jurisdiction + licenseJurisdiction = String(required=False, allow_none=False, load_default=None) + compact = Compact(required=True, allow_none=False) + licenseNumber = String(required=True, allow_none=False, validate=Length(1, 100)) + + @post_load + def rename_jurisdiction_to_license_jurisdiction(self, data, **kwargs): + if 'jurisdiction' in data: + data['licenseJurisdiction'] = data.pop('jurisdiction') + return data + + class QueryProvidersRequestSchema(CCRequestSchema): """ Schema for query providers requests. @@ -212,6 +249,7 @@ class QuerySchema(CCRequestSchema): jurisdiction = Jurisdiction(required=False, allow_none=False) givenName = String(required=False, allow_none=False, validate=Length(min=1, max=100)) familyName = String(required=False, allow_none=False, validate=Length(min=1, max=100)) + licenseNumber = String(required=False, allow_none=False, validate=Length(min=1, max=100)) class PaginationSchema(ForgivingSchema): """ @@ -219,7 +257,7 @@ class PaginationSchema(ForgivingSchema): """ lastKey = String(required=False, allow_none=False, validate=Length(min=1, max=1024)) - pageSize = Integer(required=False, allow_none=False) + pageSize = Integer(required=False, allow_none=False, validate=Range(min=5, max=100)) class SortingSchema(ForgivingSchema): """ diff --git a/backend/cosmetology-app/lambdas/python/common/common_test/test_data_generator.py b/backend/cosmetology-app/lambdas/python/common/common_test/test_data_generator.py index 981e071f7..5eb9dc04d 100644 --- a/backend/cosmetology-app/lambdas/python/common/common_test/test_data_generator.py +++ b/backend/cosmetology-app/lambdas/python/common/common_test/test_data_generator.py @@ -349,7 +349,6 @@ def generate_default_compact_configuration(value_overrides: dict | None = None) 'compactName': 'Cosmetology', 'compactOperationsTeamEmails': ['ops@example.com'], 'compactAdverseActionsNotificationEmails': ['adverse@example.com'], - 'compactSummaryReportNotificationEmails': ['summary@example.com'], 'licenseeRegistrationEnabled': True, 'configuredStates': [], } @@ -388,7 +387,6 @@ def generate_default_jurisdiction_configuration( 'jurisdictionName': 'Kentucky', 'jurisdictionOperationsTeamEmails': ['state-ops@example.com'], 'jurisdictionAdverseActionsNotificationEmails': ['state-adverse@example.com'], - 'jurisdictionSummaryReportNotificationEmails': ['state-summary@example.com'], 'licenseeRegistrationEnabled': True, } if value_overrides: diff --git a/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/compact.json b/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/compact.json index 6fc396768..da238515c 100644 --- a/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/compact.json +++ b/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/compact.json @@ -6,7 +6,6 @@ "compactName": "Cosmetology", "compactOperationsTeamEmails": [""], "compactAdverseActionsNotificationEmails": [""], - "compactSummaryReportNotificationEmails": [""], "licenseeRegistrationEnabled": true, "configuredStates": [ { diff --git a/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/jurisdiction.json b/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/jurisdiction.json index b4b274c34..80a72f445 100644 --- a/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/jurisdiction.json +++ b/backend/cosmetology-app/lambdas/python/common/tests/resources/dynamo/jurisdiction.json @@ -7,7 +7,6 @@ "postalAbbreviation": "oh", "jurisdictionOperationsTeamEmails": ["some-operations-team@test.com"], "jurisdictionAdverseActionsNotificationEmails": ["some-adverse-actions-notification-team@test.com"], - "jurisdictionSummaryReportNotificationEmails": ["some-summary-report-notification-team@test.com"], "licenseeRegistrationEnabled": true, "dateOfUpdate": "2024-10-04T12:34:56+00:00" } diff --git a/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_license.py b/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_license.py index 9bac4adbc..c4414718c 100644 --- a/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_license.py +++ b/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_license.py @@ -303,6 +303,79 @@ def test_compact_eligible_with_inactive_license_not_allowed(self): LicenseIngestSchema().load({'compact': 'cosm', 'jurisdiction': 'oh', **license_record}) +class TestLicenseOpenSearchDocumentSchema(TstLambdas): + """Tests for LicenseOpenSearchDocumentSchema which extends LicenseGeneralResponseSchema with dateOfBirth.""" + + def _make_license_data(self, *, license_status='active', date_of_expiration='2100-01-01'): + """Create valid license data including dateOfBirth for testing.""" + return { + 'providerId': 'a4182428-d061-701c-82e5-a3d1d547d797', + 'type': 'license', + 'dateOfUpdate': '2024-01-01T00:00:00+00:00', + 'compact': 'cosm', + 'jurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'licenseStatus': license_status, + 'jurisdictionUploadedLicenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2024-01-01', + 'dateOfExpiration': date_of_expiration, + 'homeAddressStreet1': '123 Main St', + 'homeAddressCity': 'Columbus', + 'homeAddressState': 'OH', + 'homeAddressPostalCode': '43215', + 'licenseNumber': 'LIC12345', + 'dateOfBirth': '1985-06-06', + } + + def test_includes_date_of_birth(self): + """LicenseOpenSearchDocumentSchema should include dateOfBirth in the loaded output.""" + from cc_common.data_model.schema.license.api import LicenseOpenSearchDocumentSchema + + license_data = self._make_license_data() + result = LicenseOpenSearchDocumentSchema().load(license_data) + + self.assertEqual('1985-06-06', result['dateOfBirth']) + + def test_retains_all_general_response_fields(self): + """LicenseOpenSearchDocumentSchema should retain all fields from LicenseGeneralResponseSchema.""" + from cc_common.data_model.schema.license.api import LicenseOpenSearchDocumentSchema + + license_data = self._make_license_data() + result = LicenseOpenSearchDocumentSchema().load(license_data) + + for field in [ + 'providerId', 'type', 'dateOfUpdate', 'compact', 'jurisdiction', + 'licenseType', 'licenseStatus', 'licenseNumber', 'givenName', 'familyName', + 'dateOfIssuance', 'dateOfExpiration', 'homeAddressStreet1', 'homeAddressCity', + 'homeAddressState', 'homeAddressPostalCode', + ]: + self.assertIn(field, result, f'Expected field {field} to be in loaded result') + + def test_expired_license_status_corrected_to_inactive(self): + """LicenseOpenSearchDocumentSchema should inherit expiration status correction from LicenseExpirationStatusMixin.""" + from cc_common.data_model.schema.license.api import LicenseOpenSearchDocumentSchema + + license_data = self._make_license_data(license_status='active', date_of_expiration='2020-01-01') + result = LicenseOpenSearchDocumentSchema().load(license_data) + + self.assertEqual('inactive', result['licenseStatus']) + + def test_strips_fields_not_in_schema(self): + """LicenseOpenSearchDocumentSchema should strip fields not defined in the schema (ForgivingSchema behavior).""" + from cc_common.data_model.schema.license.api import LicenseOpenSearchDocumentSchema + + license_data = self._make_license_data() + license_data['ssnLastFour'] = '1234' + + result = LicenseOpenSearchDocumentSchema().load(license_data) + + self.assertNotIn('ssnLastFour', result) + + class TestLicenseGeneralResponseSchemaExpirationCheck(TstLambdas): """ Tests for the LicenseExpirationStatusMixin applied to LicenseGeneralResponseSchema. diff --git a/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_provider.py b/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_provider.py index 54f1cef87..2fb2b77cd 100644 --- a/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_provider.py +++ b/backend/cosmetology-app/lambdas/python/common/tests/unit/test_data_model/test_schema/test_provider.py @@ -7,6 +7,99 @@ from tests import TstLambdas +class TestProviderOpenSearchDocumentSchema(TstLambdas): + """Tests for ProviderOpenSearchDocumentSchema which extends ProviderGeneralResponseSchema + with dateOfBirth on nested license objects.""" + + def _make_provider_data_with_license(self): + """Create valid provider data with a nested license that includes dateOfBirth.""" + return { + 'providerId': 'a4182428-d061-701c-82e5-a3d1d547d797', + 'type': 'provider', + 'dateOfUpdate': '2024-07-08T23:59:59+00:00', + 'compact': 'cosm', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfExpiration': '2100-01-01', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-06', + 'licenses': [ + { + 'providerId': 'a4182428-d061-701c-82e5-a3d1d547d797', + 'type': 'license', + 'dateOfUpdate': '2024-06-06T12:59:59+00:00', + 'compact': 'cosm', + 'jurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'licenseStatus': 'active', + 'jurisdictionUploadedLicenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'licenseNumber': 'LIC12345', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2024-01-01', + 'dateOfExpiration': '2100-01-01', + 'homeAddressStreet1': '123 Main St', + 'homeAddressCity': 'Columbus', + 'homeAddressState': 'OH', + 'homeAddressPostalCode': '43215', + 'dateOfBirth': '1985-06-06', + } + ], + 'privileges': [], + } + + def test_license_includes_date_of_birth(self): + """ProviderOpenSearchDocumentSchema should include dateOfBirth in nested license objects.""" + from cc_common.data_model.schema.provider.api import ProviderOpenSearchDocumentSchema + + data = self._make_provider_data_with_license() + result = ProviderOpenSearchDocumentSchema().load(data) + + self.assertEqual(1, len(result['licenses'])) + self.assertEqual('1985-06-06', result['licenses'][0]['dateOfBirth']) + + def test_top_level_fields_match_general_response(self): + """ProviderOpenSearchDocumentSchema should retain all top-level fields from ProviderGeneralResponseSchema.""" + from cc_common.data_model.schema.provider.api import ProviderOpenSearchDocumentSchema + + data = self._make_provider_data_with_license() + result = ProviderOpenSearchDocumentSchema().load(data) + + for field in [ + 'providerId', 'type', 'dateOfUpdate', 'compact', 'licenseJurisdiction', + 'licenseStatus', 'compactEligibility', 'givenName', 'familyName', + 'dateOfExpiration', 'birthMonthDay', + ]: + self.assertIn(field, result, f'Expected field {field} to be in loaded result') + + def test_does_not_include_private_fields_at_top_level(self): + """ProviderOpenSearchDocumentSchema should NOT include top-level private fields like dateOfBirth or ssnLastFour.""" + from cc_common.data_model.schema.provider.api import ProviderOpenSearchDocumentSchema + + data = self._make_provider_data_with_license() + data['dateOfBirth'] = '1985-06-06' + data['ssnLastFour'] = '1234' + result = ProviderOpenSearchDocumentSchema().load(data) + + self.assertNotIn('dateOfBirth', result) + self.assertNotIn('ssnLastFour', result) + + def test_general_response_schema_does_not_include_date_of_birth_in_licenses(self): + """ProviderGeneralResponseSchema should NOT include dateOfBirth in license objects (baseline comparison).""" + from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema + + data = self._make_provider_data_with_license() + result = ProviderGeneralResponseSchema().load(data) + + self.assertNotIn('dateOfBirth', result['licenses'][0]) + + class TestProviderRecordSchema(TstLambdas): def test_serde(self): """Test round-trip deserialization/serialization""" diff --git a/backend/cosmetology-app/lambdas/python/common/tests/unit/test_provider_record_util.py b/backend/cosmetology-app/lambdas/python/common/tests/unit/test_provider_record_util.py index 53721760a..5245d4977 100644 --- a/backend/cosmetology-app/lambdas/python/common/tests/unit/test_provider_record_util.py +++ b/backend/cosmetology-app/lambdas/python/common/tests/unit/test_provider_record_util.py @@ -1,5 +1,6 @@ from datetime import date -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, ANY +from uuid import UUID from tests import TstLambdas @@ -503,3 +504,501 @@ def test_find_best_license_complex_scenario(self): best_license = ProviderRecordUtility.find_best_license(licenses) self.assertEqual(best_license['dateOfIssuance'], '2024-03-01') self.assertEqual(best_license['compactEligibility'], CompactEligibilityStatus.INELIGIBLE) + + +@patch('cc_common.config._Config.expiration_resolution_date', date(2025, 6, 1)) +class TestGenerateOpenSearchDocuments(TstLambdas): + """Tests for ProviderUserRecords.generate_opensearch_documents().""" + + def _make_provider_records(self, provider_overrides=None, license_overrides_list=None, extra_records=None): + """Build list of provider + license (and optional other) records as dicts for ProviderUserRecords.""" + from common_test.test_data_generator import TestDataGenerator + + if license_overrides_list is None: + license_overrides_list = [] + + provider = TestDataGenerator.generate_default_provider(provider_overrides or {}) + provider_record = provider.serialize_to_database_record() + records = [provider_record] + for overrides in license_overrides_list: + lic = TestDataGenerator.generate_default_license(overrides) + records.append(lic.serialize_to_database_record()) + if extra_records: + records.extend(extra_records) + return records + + def _patch_config_for_privilege_generation(self, live_compact_jurisdictions=None): + if live_compact_jurisdictions is None: + live_compact_jurisdictions = {'cosm': ['al', 'ky', 'oh']} + mock_config = MagicMock() + mock_config.live_compact_jurisdictions = live_compact_jurisdictions + mock_config.license_type_abbreviations = {'cosm': {'cosmetologist': 'cos', 'esthetician': 'esth'}} + return patch('cc_common.data_model.provider_record_util.config', mock_config) + + def test_single_license_returns_one_document(self): + """Provider with one license produces exactly one OpenSearch document.""" + from cc_common.data_model.provider_record_util import ProviderUserRecords + + records = self._make_provider_records( + license_overrides_list=[ + { + 'jurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'dateOfExpiration': date(2026, 4, 4), + } + ] + ) + with self._patch_config_for_privilege_generation(): + pur = ProviderUserRecords(records) + docs = pur.generate_opensearch_documents() + + self.assertEqual([{'birthMonthDay': '06-06', + 'compact': 'cosm', + 'compactEligibility': 'ineligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2025, 4, 4), + 'dateOfUpdate': ANY, + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'licenses': [{'adverseActions': [], + 'compact': 'cosm', + 'compactEligibility': 'eligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2026, 4, 4), + 'dateOfIssuance': date(2010, 6, 6), + 'dateOfRenewal': date(2020, 4, 4), + 'dateOfUpdate': ANY, + 'emailAddress': 'björk@example.com', + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'homeAddressCity': 'Columbus', + 'homeAddressPostalCode': '43004', + 'homeAddressState': 'oh', + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'investigations': [], + 'jurisdiction': 'oh', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseNumber': 'A0608337260', + 'licenseStatus': 'active', + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseType': 'cosmetologist', + 'middleName': 'Gunnar', + 'phoneNumber': '+13213214321', + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'license'}], + 'middleName': 'Gunnar', + 'privileges': [{'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'al', + 'licenseJurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'active', + 'type': 'privilege'}, + {'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'active', + 'type': 'privilege'}], + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'provider'}], docs) + + def test_two_licenses_different_types_returns_two_documents(self): + """Provider with two licenses of different types produces two documents. + The second license is also ineligible, so its associated privileges should be inactive. + """ + from cc_common.data_model.provider_record_util import ProviderUserRecords + from cc_common.data_model.schema.common import CompactEligibilityStatus + + records = self._make_provider_records( + license_overrides_list=[ + { + 'jurisdiction': 'al', + 'licenseType': 'cosmetologist', + 'dateOfExpiration': date(2026, 4, 4), + 'jurisdictionUploadedCompactEligibility': CompactEligibilityStatus.ELIGIBLE, + }, + { + 'jurisdiction': 'oh', + 'licenseType': 'esthetician', + 'dateOfExpiration': date(2026, 4, 4), + # jurisdictionUploadedCompactEligibility is ineligible, so the privileges should be inactive + 'jurisdictionUploadedCompactEligibility': CompactEligibilityStatus.INELIGIBLE, + }, + ] + ) + with self._patch_config_for_privilege_generation(): + pur = ProviderUserRecords(records) + docs = pur.generate_opensearch_documents() + + self.assertEqual([{'birthMonthDay': '06-06', + 'compact': 'cosm', + 'compactEligibility': 'ineligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2025, 4, 4), + 'dateOfUpdate': ANY, + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'licenses': [{'adverseActions': [], + 'compact': 'cosm', + 'compactEligibility': 'eligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2026, 4, 4), + 'dateOfIssuance': date(2010, 6, 6), + 'dateOfRenewal': date(2020, 4, 4), + 'dateOfUpdate': ANY, + 'emailAddress': 'björk@example.com', + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'homeAddressCity': 'Columbus', + 'homeAddressPostalCode': '43004', + 'homeAddressState': 'oh', + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'investigations': [], + 'jurisdiction': 'al', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseNumber': 'A0608337260', + 'licenseStatus': 'active', + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseType': 'cosmetologist', + 'middleName': 'Gunnar', + 'phoneNumber': '+13213214321', + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'license'}], + 'middleName': 'Gunnar', + 'privileges': [{'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'al', + 'licenseType': 'cosmetologist', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'active', + 'type': 'privilege'}, + {'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'oh', + 'licenseJurisdiction': 'al', + 'licenseType': 'cosmetologist', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'active', + 'type': 'privilege'}], + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'provider'}, + {'birthMonthDay': '06-06', + 'compact': 'cosm', + 'compactEligibility': 'ineligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2025, 4, 4), + 'dateOfUpdate': ANY, + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'licenses': [{'adverseActions': [], + 'compact': 'cosm', + 'compactEligibility': 'ineligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2026, 4, 4), + 'dateOfIssuance': date(2010, 6, 6), + 'dateOfRenewal': date(2020, 4, 4), + 'dateOfUpdate': ANY, + 'emailAddress': 'björk@example.com', + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'homeAddressCity': 'Columbus', + 'homeAddressPostalCode': '43004', + 'homeAddressState': 'oh', + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'investigations': [], + 'jurisdiction': 'oh', + 'jurisdictionUploadedCompactEligibility': 'ineligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseNumber': 'A0608337260', + 'licenseStatus': 'active', + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseType': 'esthetician', + 'middleName': 'Gunnar', + 'phoneNumber': '+13213214321', + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'license'}], + 'middleName': 'Gunnar', + # these privileges are inactive due to the home state license being ineligible + 'privileges': [{'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'al', + 'licenseJurisdiction': 'oh', + 'licenseType': 'esthetician', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'inactive', + 'type': 'privilege'}, + {'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'esthetician', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'inactive', + 'type': 'privilege'}], + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'provider'}], docs) + + def test_privileges_assigned_only_to_home_license_document(self): + """Privileges are only on the document whose license is the home license for its type.""" + from cc_common.data_model.provider_record_util import ProviderUserRecords + from cc_common.data_model.schema.common import CompactEligibilityStatus + + records = self._make_provider_records( + license_overrides_list=[ + { + 'jurisdiction': 'al', + 'licenseType': 'cosmetologist', + 'dateOfExpiration': date(2026, 4, 4), + 'compactEligibility': CompactEligibilityStatus.ELIGIBLE, + 'dateOfIssuance': date(2023, 1, 1), + }, + { + 'jurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'dateOfExpiration': date(2026, 4, 4), + 'compactEligibility': CompactEligibilityStatus.ELIGIBLE, + # this license was issues more recently, so it should have the privileges associated with it. + 'dateOfIssuance': date(2024, 6, 1), + }, + ] + ) + with self._patch_config_for_privilege_generation(): + pur = ProviderUserRecords(records) + docs = pur.generate_opensearch_documents() + + self.assertEqual([{'birthMonthDay': '06-06', + 'compact': 'cosm', + 'compactEligibility': 'ineligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2025, 4, 4), + 'dateOfUpdate': ANY, + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'licenses': [{'adverseActions': [], + 'compact': 'cosm', + 'compactEligibility': 'eligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2026, 4, 4), + 'dateOfIssuance': date(2023, 1, 1), + 'dateOfRenewal': date(2020, 4, 4), + 'dateOfUpdate': ANY, + 'emailAddress': 'björk@example.com', + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'homeAddressCity': 'Columbus', + 'homeAddressPostalCode': '43004', + 'homeAddressState': 'oh', + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'investigations': [], + 'jurisdiction': 'al', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseNumber': 'A0608337260', + 'licenseStatus': 'active', + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseType': 'cosmetologist', + 'middleName': 'Gunnar', + 'phoneNumber': '+13213214321', + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'license'}], + 'middleName': 'Gunnar', + 'privileges': [], + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'provider'}, + {'birthMonthDay': '06-06', + 'compact': 'cosm', + 'compactEligibility': 'ineligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2025, 4, 4), + 'dateOfUpdate': ANY, + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'licenses': [{'adverseActions': [], + 'compact': 'cosm', + 'compactEligibility': 'eligible', + 'dateOfBirth': date(1985, 6, 6), + 'dateOfExpiration': date(2026, 4, 4), + 'dateOfIssuance': date(2024, 6, 1), + 'dateOfRenewal': date(2020, 4, 4), + 'dateOfUpdate': ANY, + 'emailAddress': 'björk@example.com', + 'familyName': 'Guðmundsdóttir', + 'givenName': 'Björk', + 'homeAddressCity': 'Columbus', + 'homeAddressPostalCode': '43004', + 'homeAddressState': 'oh', + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'investigations': [], + 'jurisdiction': 'oh', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'licenseNumber': 'A0608337260', + 'licenseStatus': 'active', + 'licenseStatusName': 'DEFINITELY_A_HUMAN', + 'licenseType': 'cosmetologist', + 'middleName': 'Gunnar', + 'phoneNumber': '+13213214321', + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'license'}], + 'middleName': 'Gunnar', + 'privileges': [{'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'al', + 'licenseJurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'active', + 'type': 'privilege'}, + {'administratorSetStatus': 'active', + 'adverseActions': [], + 'compact': 'cosm', + 'dateOfExpiration': date(2026, 4, 4), + 'investigations': [], + 'jurisdiction': 'ky', + 'licenseJurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'providerId': '89a6377e-c3a5-40e5-bca5-317ec854c570', + 'status': 'active', + 'type': 'privilege'}], + 'providerId': UUID('89a6377e-c3a5-40e5-bca5-317ec854c570'), + 'ssnLastFour': '1234', + 'type': 'provider'}], docs) + + def test_multiple_types_privileges_on_correct_home_licenses(self): + """With two license types, each type's home license gets its own privileges.""" + from cc_common.data_model.provider_record_util import ProviderUserRecords + from cc_common.data_model.schema.common import CompactEligibilityStatus + + records = self._make_provider_records( + license_overrides_list=[ + { + 'jurisdiction': 'al', + 'licenseType': 'cosmetologist', + 'dateOfExpiration': date(2026, 4, 4), + 'compactEligibility': CompactEligibilityStatus.ELIGIBLE, + }, + { + 'jurisdiction': 'oh', + 'licenseType': 'esthetician', + 'dateOfExpiration': date(2026, 4, 4), + 'compactEligibility': CompactEligibilityStatus.ELIGIBLE, + }, + ] + ) + with self._patch_config_for_privilege_generation(): + pur = ProviderUserRecords(records) + docs = pur.generate_opensearch_documents() + + self.assertEqual(2, len(docs)) + al_doc = next(d for d in docs if d['licenses'][0]['jurisdiction'] == 'al') + oh_doc = next(d for d in docs if d['licenses'][0]['jurisdiction'] == 'oh') + # cosmetologist home is al -> al_doc gets cosmetologist privileges + cos_privs = [p for p in al_doc['privileges'] if p['licenseType'] == 'cosmetologist'] + self.assertGreater(len(cos_privs), 0) + # esthetician home is oh -> oh_doc gets esthetician privileges + esth_privs = [p for p in oh_doc['privileges'] if p['licenseType'] == 'esthetician'] + self.assertGreater(len(esth_privs), 0) + + def test_license_adverse_actions_included(self): + """Each document includes adverse actions specific to its license.""" + from cc_common.data_model.provider_record_util import ProviderUserRecords + from cc_common.data_model.schema.common import CompactEligibilityStatus + + records = self._make_provider_records( + license_overrides_list=[ + { + 'jurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'dateOfExpiration': date(2026, 4, 4), + 'compactEligibility': CompactEligibilityStatus.ELIGIBLE, + } + ], + extra_records=[ + self.test_data_generator.generate_default_adverse_action( + value_overrides={ + 'jurisdiction': 'oh', + 'actionAgainst': 'license', + 'licenseTypeAbbreviation': 'cos', + } + ).serialize_to_database_record() + ], + ) + with self._patch_config_for_privilege_generation(): + pur = ProviderUserRecords(records) + docs = pur.generate_opensearch_documents() + + self.assertEqual(1, len(docs)) + self.assertEqual(1, len(docs[0]['licenses'][0]['adverseActions'])) + + def test_no_licenses_returns_empty_list(self): + """Provider with no license records produces an empty list.""" + from cc_common.data_model.provider_record_util import ProviderUserRecords + + records = self._make_provider_records() + with self._patch_config_for_privilege_generation(): + pur = ProviderUserRecords(records) + docs = pur.generate_opensearch_documents() + + self.assertEqual([], docs) diff --git a/backend/cosmetology-app/lambdas/python/compact-configuration/handlers/compact_configuration.py b/backend/cosmetology-app/lambdas/python/compact-configuration/handlers/compact_configuration.py index 6efcf858a..71b380e99 100644 --- a/backend/cosmetology-app/lambdas/python/compact-configuration/handlers/compact_configuration.py +++ b/backend/cosmetology-app/lambdas/python/compact-configuration/handlers/compact_configuration.py @@ -186,7 +186,6 @@ def _get_staff_users_compact_configuration(event: dict, context: LambdaContext): 'licenseeRegistrationEnabled': False, 'compactOperationsTeamEmails': [], 'compactAdverseActionsNotificationEmails': [], - 'compactSummaryReportNotificationEmails': [], 'configuredStates': [], } ).to_dict() @@ -356,7 +355,6 @@ def _get_staff_users_jurisdiction_configuration(event: dict, context: LambdaCont }, 'jurisdictionOperationsTeamEmails': [], 'jurisdictionAdverseActionsNotificationEmails': [], - 'jurisdictionSummaryReportNotificationEmails': [], 'licenseeRegistrationEnabled': False, } ).to_dict() diff --git a/backend/cosmetology-app/lambdas/python/compact-configuration/tests/function/test_compact_configuration.py b/backend/cosmetology-app/lambdas/python/compact-configuration/tests/function/test_compact_configuration.py index 98231ba26..fefb50523 100644 --- a/backend/cosmetology-app/lambdas/python/compact-configuration/tests/function/test_compact_configuration.py +++ b/backend/cosmetology-app/lambdas/python/compact-configuration/tests/function/test_compact_configuration.py @@ -308,7 +308,6 @@ def _when_testing_put_compact_configuration_with_existing_configuration(self): 'licenseeRegistrationEnabled': compact_config.licenseeRegistrationEnabled, 'compactOperationsTeamEmails': compact_config.compactOperationsTeamEmails, 'compactAdverseActionsNotificationEmails': compact_config.compactAdverseActionsNotificationEmails, - 'compactSummaryReportNotificationEmails': compact_config.compactSummaryReportNotificationEmails, 'configuredStates': compact_config.configuredStates, }, cls=ResponseEncoder, @@ -333,7 +332,6 @@ def _when_testing_put_compact_configuration(self): 'licenseeRegistrationEnabled': compact_config.licenseeRegistrationEnabled, 'compactOperationsTeamEmails': compact_config.compactOperationsTeamEmails, 'compactAdverseActionsNotificationEmails': compact_config.compactAdverseActionsNotificationEmails, - 'compactSummaryReportNotificationEmails': compact_config.compactSummaryReportNotificationEmails, 'configuredStates': compact_config.configuredStates, }, cls=ResponseEncoder, @@ -387,7 +385,6 @@ def test_get_compact_configuration_returns_empty_compact_configuration_if_no_con 'compactName': 'Cosmetology', 'compactOperationsTeamEmails': [], 'compactAdverseActionsNotificationEmails': [], - 'compactSummaryReportNotificationEmails': [], 'licenseeRegistrationEnabled': False, 'configuredStates': [], }, @@ -636,7 +633,6 @@ def _when_testing_put_jurisdiction_configuration(self, create_compact=True): { 'jurisdictionOperationsTeamEmails': jurisdiction_config.jurisdictionOperationsTeamEmails, 'jurisdictionAdverseActionsNotificationEmails': jurisdiction_config.jurisdictionAdverseActionsNotificationEmails, - 'jurisdictionSummaryReportNotificationEmails': jurisdiction_config.jurisdictionSummaryReportNotificationEmails, 'licenseeRegistrationEnabled': jurisdiction_config.licenseeRegistrationEnabled, }, cls=ResponseEncoder, @@ -711,7 +707,6 @@ def test_get_jurisdiction_configuration_returns_empty_jurisdiction_configuration 'jurisdictionAdverseActionsNotificationEmails': [], 'jurisdictionName': 'Kentucky', 'jurisdictionOperationsTeamEmails': [], - 'jurisdictionSummaryReportNotificationEmails': [], 'licenseeRegistrationEnabled': False, 'postalAbbreviation': 'ky', }, @@ -746,7 +741,6 @@ def test_get_jurisdiction_configuration_returns_configuration_if_exists(self): 'postalAbbreviation': test_jurisdiction_config.postalAbbreviation, 'jurisdictionOperationsTeamEmails': test_jurisdiction_config.jurisdictionOperationsTeamEmails, 'jurisdictionAdverseActionsNotificationEmails': test_jurisdiction_config.jurisdictionAdverseActionsNotificationEmails, - 'jurisdictionSummaryReportNotificationEmails': test_jurisdiction_config.jurisdictionSummaryReportNotificationEmails, 'licenseeRegistrationEnabled': test_jurisdiction_config.licenseeRegistrationEnabled, }, response_body, diff --git a/backend/cosmetology-app/lambdas/python/search/handlers/manage_opensearch_indices.py b/backend/cosmetology-app/lambdas/python/search/handlers/manage_opensearch_indices.py index 9950038d3..0c606ba4d 100644 --- a/backend/cosmetology-app/lambdas/python/search/handlers/manage_opensearch_indices.py +++ b/backend/cosmetology-app/lambdas/python/search/handlers/manage_opensearch_indices.py @@ -257,6 +257,7 @@ def _get_provider_index_mapping(self, number_of_shards: int, number_of_replicas: 'dateOfIssuance': {'type': 'date'}, 'dateOfRenewal': {'type': 'date'}, 'dateOfExpiration': {'type': 'date'}, + 'dateOfBirth': {'type': 'date'}, 'homeAddressStreet1': {'type': 'text'}, 'homeAddressStreet2': {'type': 'text'}, 'homeAddressCity': { diff --git a/backend/cosmetology-app/lambdas/python/search/handlers/populate_provider_documents.py b/backend/cosmetology-app/lambdas/python/search/handlers/populate_provider_documents.py index b044185b9..b4904f24b 100644 --- a/backend/cosmetology-app/lambdas/python/search/handlers/populate_provider_documents.py +++ b/backend/cosmetology-app/lambdas/python/search/handlers/populate_provider_documents.py @@ -40,7 +40,7 @@ from cc_common.exceptions import CCInternalException from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from utils import generate_provider_opensearch_document +from utils import generate_provider_opensearch_documents # Batch size for DynamoDB pagination DYNAMODB_PAGE_SIZE = 1000 @@ -77,7 +77,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Track statistics stats = { 'total_providers_processed': 0, - 'total_providers_indexed': 0, + 'total_licenses_indexed': 0, 'total_providers_failed': 0, 'compacts_processed': [], 'errors': [], @@ -110,7 +110,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): documents_to_index = [] compact_stats = { 'providers_processed': 0, - 'providers_indexed': 0, + 'licenses_indexed': 0, 'providers_failed': 0, } @@ -151,7 +151,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Update stats for current compact stats['total_providers_processed'] += compact_stats['providers_processed'] - stats['total_providers_indexed'] += compact_stats['providers_indexed'] + stats['total_licenses_indexed'] += compact_stats['licenses_indexed'] stats['total_providers_failed'] += compact_stats['providers_failed'] if compact_stats['providers_processed'] > 0: stats['compacts_processed'].append( @@ -171,7 +171,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): logger.info( 'Returning for pagination', total_providers_processed=stats['total_providers_processed'], - total_providers_indexed=stats['total_providers_indexed'], + total_licenses_indexed=stats['total_licenses_indexed'], resume_from=stats['resumeFrom'], ) @@ -214,9 +214,8 @@ def populate_provider_documents(event: dict, context: LambdaContext): continue try: - # Use the shared utility to process the provider - serializable_document = generate_provider_opensearch_document(compact, provider_id) - documents_to_index.append(serializable_document) + serializable_documents = generate_provider_opensearch_documents(compact, provider_id) + documents_to_index.extend(serializable_documents) except ValidationError as e: logger.warning( @@ -259,7 +258,7 @@ def populate_provider_documents(event: dict, context: LambdaContext): # Update overall stats stats['total_providers_processed'] += compact_stats['providers_processed'] - stats['total_providers_indexed'] += compact_stats['providers_indexed'] + stats['total_licenses_indexed'] += compact_stats['licenses_indexed'] stats['total_providers_failed'] += compact_stats['providers_failed'] stats['compacts_processed'].append( { @@ -272,14 +271,14 @@ def populate_provider_documents(event: dict, context: LambdaContext): 'Completed processing compact', compact=compact, providers_processed=compact_stats['providers_processed'], - providers_indexed=compact_stats['providers_indexed'], + licenses_indexed=compact_stats['licenses_indexed'], providers_failed=compact_stats['providers_failed'], ) logger.info( 'Completed populating provider documents', total_providers_processed=stats['total_providers_processed'], - total_providers_indexed=stats['total_providers_indexed'], + total_licenses_indexed=stats['total_licenses_indexed'], total_providers_failed=stats['total_providers_failed'], ) @@ -292,7 +291,7 @@ def _index_records_and_track_stats( index_name = f'compact_{compact}_providers' if documents_to_index: failed_ids = _bulk_index_documents(opensearch_client, index_name, documents_to_index) - compact_stats['providers_indexed'] += len(documents_to_index) - len(failed_ids) + compact_stats['licenses_indexed'] += len(documents_to_index) - len(failed_ids) if failed_ids: compact_stats['providers_failed'] += len(failed_ids) logger.warning( @@ -325,7 +324,7 @@ def _build_error_response( # Update stats for current compact stats['total_providers_processed'] += compact_stats['providers_processed'] - stats['total_providers_indexed'] += compact_stats['providers_indexed'] + stats['total_licenses_indexed'] += compact_stats['licenses_indexed'] stats['total_providers_failed'] += compact_stats['providers_failed'] if compact_stats['providers_processed'] > 0: stats['compacts_processed'].append( @@ -365,7 +364,7 @@ def _bulk_index_documents(opensearch_client: OpenSearchClient, index_name: str, return set() # This will raise CCInternalException if all retries fail - response = opensearch_client.bulk_index(index_name=index_name, documents=documents) + response = opensearch_client.bulk_index(index_name=index_name, documents=documents, id_field='documentId') # Check for errors in the bulk response (individual document failures, not connection issues) if response.get('errors'): diff --git a/backend/cosmetology-app/lambdas/python/search/handlers/provider_update_ingest.py b/backend/cosmetology-app/lambdas/python/search/handlers/provider_update_ingest.py index 442dfe4d9..cec5f36a7 100644 --- a/backend/cosmetology-app/lambdas/python/search/handlers/provider_update_ingest.py +++ b/backend/cosmetology-app/lambdas/python/search/handlers/provider_update_ingest.py @@ -7,6 +7,11 @@ compact, and bulk indexes the sanitized provider documents into the appropriate OpenSearch indices. +The handler classifies events by their DynamoDB eventName: +- INSERT/MODIFY: Generate one document per license and upsert via composite documentId +- REMOVE: Delete all documents for the provider, then re-check DynamoDB and re-index + any remaining license documents + The handler uses the @sqs_batch_handler decorator which passes all SQS messages to the handler at once, enabling batch processing and deduplication. The handler returns batchItemFailures directly for partial success handling. @@ -18,7 +23,7 @@ from cc_common.utils import sqs_batch_handler from marshmallow import ValidationError from opensearch_client import OpenSearchClient -from utils import generate_provider_opensearch_document +from utils import generate_provider_opensearch_documents # Instantiate the OpenSearch client outside of the handler to cache connection between invocations opensearch_client = OpenSearchClient(timeout=30) @@ -30,10 +35,10 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: Process DynamoDB stream events from SQS and index provider documents into OpenSearch. This function: - 1. Creates a set for each compact to deduplicate provider IDs - 2. Extracts compact and providerId from each stream record (old or new image) - 3. Processes each unique provider by compact using the shared utility - 4. Bulk indexes the documents into the appropriate OpenSearch index + 1. Classifies events by eventName (REMOVE vs INSERT/MODIFY) + 2. Deduplicates provider IDs per compact + 3. For INSERT/MODIFY: generates one document per license and bulk upserts + 4. For REMOVE: deletes all docs for the provider, re-checks DynamoDB, re-indexes remaining :param records: List of SQS records, each containing 'messageId' and 'body' (DynamoDB stream record) :return: Response with batch item failures for partial success handling @@ -44,13 +49,13 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: logger.info('Processing SQS batch with DynamoDB stream records', record_count=len(records)) - # Create a set for each compact to deduplicate provider IDs - providers_by_compact: dict[str, set[str]] = {compact: set() for compact in config.compacts} + # Track providers to update and delete separately per compact + providers_to_update: dict[str, set[str]] = {compact: set() for compact in config.compacts} + providers_to_delete: dict[str, set[str]] = {compact: set() for compact in config.compacts} # Track which message IDs correspond to which compact/provider for failure reporting record_mapping: dict[str, tuple[str, str]] = {} # message_id -> (compact, provider_id) - # Extract compact and providerId from each record for record in records: message_id = record['messageId'] # The body contains the DynamoDB stream record sent via EventBridge Pipe @@ -64,7 +69,6 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: continue # Extract compact and providerId from the DynamoDB image - # The format is {'S': 'value'} for string attributes deserialized_image = TypeDeserializer().deserialize(value={'M': image}) compact = deserialized_image.get('compact') provider_id = deserialized_image.get('providerId') @@ -78,30 +82,39 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: ) continue - # Add to the appropriate compact's set to dedup provider ids - if compact in providers_by_compact: - providers_by_compact[compact].add(provider_id) - record_mapping[message_id] = (compact, provider_id) - else: + if compact not in providers_to_update: logger.warning('Unknown compact in record', compact=compact, provider_id=provider_id) + continue + + record_mapping[message_id] = (compact, provider_id) + + is_remove_event = stream_record.get('eventName') == 'REMOVE' + if is_remove_event: + providers_to_delete[compact].add(provider_id) + else: + providers_to_update[compact].add(provider_id) - # Process providers and bulk index by compact batch_item_failures = [] failed_providers: dict[str, set] = {compact: set() for compact in config.compacts} - for compact, provider_ids in providers_by_compact.items(): + # --- Process INSERT/MODIFY events --- + for compact, provider_ids in providers_to_update.items(): + # Exclude providers that are also in the delete set (REMOVE takes precedence) + provider_ids = provider_ids - providers_to_delete[compact] + + if not provider_ids: + continue + index_name = f'compact_{compact}_providers' - logger.info('Processing providers for compact', compact=compact, provider_count=len(provider_ids)) + logger.info('Processing providers for update', compact=compact, provider_count=len(provider_ids)) documents_to_index = [] - providers_to_delete = [] # Provider IDs that no longer exist and need to be deleted from the index for provider_id in provider_ids: try: - document = generate_provider_opensearch_document(compact, provider_id) - documents_to_index.append(document) + docs = generate_provider_opensearch_documents(compact, provider_id) + documents_to_index.extend(docs) except CCNotFoundException as e: - # if no provider records are found, the provider needs to be deleted from the index logger.warning( 'No provider records found. This may occur if a license upload rollback was performed or if records' ' were manually deleted. Will delete provider document from index.', @@ -109,7 +122,7 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: compact=compact, error=str(e), ) - providers_to_delete.append(provider_id) + providers_to_delete[compact].add(provider_id) except ValidationError as e: logger.warning( 'Failed to process provider for indexing', @@ -119,31 +132,25 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: ) failed_providers[compact].add(provider_id) - if failed_providers[compact]: - logger.warning( - 'Some providers failed serialization', - compact=compact, - failed_provider_ids=failed_providers[compact], - successful_count=len(documents_to_index), - ) - - # Bulk index the documents if documents_to_index: try: - response = opensearch_client.bulk_index(index_name=index_name, documents=documents_to_index) + response = opensearch_client.bulk_index( + index_name=index_name, documents=documents_to_index, id_field='documentId' + ) - # Check for individual document failures if response.get('errors'): for item in response.get('items', []): index_result = item.get('index', {}) if index_result.get('error'): - doc_id = index_result.get('_id') + doc_id = index_result.get('_id', '') + provider_id = doc_id.split('#')[0] if '#' in doc_id else doc_id logger.error( 'Document indexing failed', - provider_id=doc_id, + document_id=doc_id, + provider_id=provider_id, error=index_result.get('error'), ) - failed_providers[compact].add(doc_id) + failed_providers[compact].add(provider_id) logger.info( 'Bulk indexed documents', @@ -159,39 +166,79 @@ def provider_update_ingest_handler(records: list[dict]) -> dict: document_count=len(documents_to_index), error=str(e), ) - # Mark all providers in this compact as failed - document_provider_ids = [document['providerId'] for document in documents_to_index] - for provider_id in document_provider_ids: - failed_providers[compact].add(provider_id) + for doc in documents_to_index: + failed_providers[compact].add(doc['providerId']) + + # --- Process REMOVE events --- + for compact, provider_ids in providers_to_delete.items(): + if not provider_ids: + continue - # Bulk delete providers that no longer exist - if providers_to_delete: + index_name = f'compact_{compact}_providers' + logger.info('Processing providers for delete', compact=compact, provider_count=len(provider_ids)) + + for provider_id in provider_ids: try: - failed_provider_ids = opensearch_client.bulk_delete( - index_name=index_name, document_ids=providers_to_delete + result = opensearch_client.delete_provider_documents( + index_name=index_name, + provider_id=provider_id, ) - failed_providers[compact].update(failed_provider_ids) - logger.info( - 'Bulk deleted documents', + 'Deleted provider documents from index', index_name=index_name, - document_count=len(providers_to_delete), - failed_provider_ids=list(failed_provider_ids), + provider_id=provider_id, + deleted_count=result.get('deleted', 0), + ) + except CCInternalException as e: + logger.error( + 'Failed to delete provider documents from index', + index_name=index_name, + provider_id=provider_id, + error=str(e), + ) + failed_providers[compact].add(provider_id) + continue + + # Re-check DynamoDB -- the REMOVE may have been for a single record while + # the provider still has other records remaining. + try: + docs = generate_provider_opensearch_documents(compact, provider_id) + if docs: + response = opensearch_client.bulk_index( + index_name=index_name, documents=docs, id_field='documentId' + ) + logger.info( + 'Re-indexed remaining documents after delete', + index_name=index_name, + provider_id=provider_id, + document_count=len(docs), + ) + if response.get('errors'): + for item in response.get('items', []): + index_result = item.get('index', {}) + if index_result.get('error'): + logger.error( + 'Document re-indexing failed after delete', + document_id=index_result.get('_id'), + error=index_result.get('error'), + ) + failed_providers[compact].add(provider_id) + except CCNotFoundException: + logger.info( + 'Provider no longer exists after REMOVE event, delete is complete', + provider_id=provider_id, + compact=compact, ) except CCInternalException as e: - # All deletes for this compact failed logger.error( - 'Failed to bulk delete documents after retries', + 'Failed to re-index remaining documents after delete', index_name=index_name, - document_count=len(providers_to_delete), + provider_id=provider_id, error=str(e), ) - # Mark all providers to delete as failed - for provider_id in providers_to_delete: - failed_providers[compact].add(provider_id) + failed_providers[compact].add(provider_id) - # Build batch item failures response for failed providers - # Map back from failed providers to their SQS message IDs + # Build batch item failures response for message_id, (compact, provider_id) in record_mapping.items(): if provider_id in failed_providers[compact]: logger.info( diff --git a/backend/cosmetology-app/lambdas/python/search/handlers/public_search.py b/backend/cosmetology-app/lambdas/python/search/handlers/public_search.py new file mode 100644 index 000000000..940a6dc65 --- /dev/null +++ b/backend/cosmetology-app/lambdas/python/search/handlers/public_search.py @@ -0,0 +1,182 @@ +import json +from base64 import b64decode, b64encode + +from aws_lambda_powertools.utilities.typing import LambdaContext +from cc_common.config import config, logger +from cc_common.data_model.schema.provider.api import ( + PublicLicenseSearchResponseSchema, + QueryProvidersRequestSchema, +) +from cc_common.exceptions import CCInvalidRequestException +from cc_common.utils import api_handler +from marshmallow import ValidationError +from opensearch_client import OpenSearchClient + +# Instantiate the OpenSearch client outside the handler to cache the connection between invocations +# Set timeout to 20 seconds to give API gateway time to respond with response +opensearch_client = OpenSearchClient(timeout=25) + + +@api_handler +def public_search_api_handler(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument + """ + Public query providers endpoint (no auth). + Translates structured query (licenseNumber, familyName, givenName, jurisdiction) into OpenSearch + nested query and returns license-level results with existing pagination schema. + + Indexing is one OpenSearch document per license; each hit maps to one license row. + """ + http_method = event.get('httpMethod') + resource_path = event.get('resource') + if (http_method, resource_path) != ('POST', '/v1/public/compacts/{compact}/providers/query'): + raise CCInvalidRequestException(f'Unsupported method or resource: {http_method} {resource_path}') + + return _public_query_licenses(event, context) + + +def _public_query_licenses(event: dict, context: LambdaContext): # noqa: ARG001 unused-argument + compact = event['pathParameters']['compact'] + body = _parse_and_validate_public_query_body(event) + query_obj = body.get('query', {}) + pagination = body.get('pagination') or {} + page_size = pagination.get('pageSize') or config.default_page_size + + cursor = _decode_public_cursor(pagination.get('lastKey')) + search_body = _build_public_license_search_body(compact=compact, body=body, cursor=cursor) + index_name = f'compact_{compact}_providers' + + logger.info('Executing public license search', compact=compact, index_name=index_name) + response = opensearch_client.search(index_name=index_name, body=search_body) + + hits = response.get('hits', {}).get('hits', []) + license_schema = PublicLicenseSearchResponseSchema() + providers = [] + + for hit in hits: + source = hit.get('_source', {}) + provider_id = source.get('providerId') + if source.get('compact') != compact: + logger.warning( + 'Provider compact does not match path, skipping', + provider_id=provider_id, + path_compact=compact, + ) + continue + licenses = source.get('licenses') or [] + if not licenses: + logger.warning('OpenSearch hit has no licenses array', provider_id=provider_id) + continue + license_fields = licenses[0].copy() + license_fields['providerId'] = source['providerId'] + license_fields['compact'] = source['compact'] + license_fields['givenName'] = source['givenName'] + license_fields['familyName'] = source['familyName'] + try: + sanitized = license_schema.load(license_fields) + sanitized.pop('jurisdiction', None) + providers.append(sanitized) + except ValidationError as e: + logger.error( + 'Failed to sanitize license record', + provider_id=provider_id, + errors=e.messages, + ) + + last_sort = hits[-1].get('sort') if hits else None + # Full page from OpenSearch => may have more results; use last hit's sort values for search_after + last_key = None + if last_sort is not None and len(hits) >= page_size: + last_key = _encode_public_cursor(last_sort) + + return { + 'providers': providers, + 'pagination': { + 'pageSize': page_size, + 'lastKey': last_key, + 'prevLastKey': pagination.get('lastKey'), + }, + 'query': query_obj, + } + + +def _parse_and_validate_public_query_body(event: dict) -> dict: + try: + schema = QueryProvidersRequestSchema() + raw_body = event.get('body') or '{}' + body = schema.loads(raw_body) + except ValidationError as e: + logger.warning('Invalid public query request body', errors=e.messages) + raise CCInvalidRequestException(f'Invalid request: {e.messages}') from e + + query = body.get('query', {}) + if query.get('givenName') and not query.get('familyName'): + raise CCInvalidRequestException('familyName is required if givenName is provided') + + if not any((query.get('licenseNumber'), query.get('jurisdiction'), query.get('familyName'))): + raise CCInvalidRequestException('At least one of licenseNumber, jurisdiction, or familyName must be provided') + + return body + + +def _decode_public_cursor(last_key: str | None) -> dict | None: + """ + Decode and validate the public cursor (base64 JSON with search_after list). + Raises CCInvalidRequestException if lastKey is present but invalid. + """ + if not last_key: + return None + try: + decoded = json.loads(b64decode(last_key).decode('utf-8')) + except Exception as e: + raise CCInvalidRequestException('Invalid lastKey') from e + if not isinstance(decoded, dict): + raise CCInvalidRequestException('Invalid lastKey') + search_after = decoded.get('search_after') + if not isinstance(search_after, list) or len(search_after) == 0: + raise CCInvalidRequestException('Invalid lastKey') + return {'search_after': search_after} + + +def _encode_public_cursor(search_after: list) -> str: + payload = {'search_after': search_after} + return b64encode(json.dumps(payload).encode('utf-8')).decode('utf-8') + + +def _build_public_license_search_body(*, compact: str, body: dict, cursor: dict | None = None) -> dict: + query_obj = body.get('query', {}) + pagination = body.get('pagination') or {} + page_size = pagination.get('pageSize') or config.default_page_size + + search_after = cursor.get('search_after') if cursor else None + + nested_must = [] + if query_obj.get('licenseNumber'): + nested_must.append({'term': {'licenses.licenseNumber': query_obj['licenseNumber']}}) + if query_obj.get('jurisdiction'): + nested_must.append({'term': {'licenses.jurisdiction': query_obj['jurisdiction'].lower()}}) + if query_obj.get('familyName'): + nested_must.append({'match': {'licenses.familyName': query_obj['familyName']}}) + if query_obj.get('givenName'): + nested_must.append({'match': {'licenses.givenName': query_obj['givenName']}}) + + nested_query = {'nested': {'path': 'licenses', 'query': {'bool': {'must': nested_must}}}} + + must = [ + {'term': {'compact': compact}}, + nested_query, + ] + + search_body = { + 'query': {'bool': {'must': must}}, + 'size': page_size, + 'sort': [ + {'familyName.keyword': 'asc'}, + {'givenName.keyword': 'asc'}, + {'providerId': 'asc'}, + {'_id': 'asc'}, + ], + } + if search_after is not None: + search_body['search_after'] = search_after + + return search_body diff --git a/backend/cosmetology-app/lambdas/python/search/handlers/search.py b/backend/cosmetology-app/lambdas/python/search/handlers/search.py index 41f3f3df8..ddb6c4cd6 100644 --- a/backend/cosmetology-app/lambdas/python/search/handlers/search.py +++ b/backend/cosmetology-app/lambdas/python/search/handlers/search.py @@ -1,3 +1,5 @@ +from re import match + from aws_lambda_powertools.utilities.typing import LambdaContext from cc_common.config import logger from cc_common.data_model.schema.common import CCPermissionsAction @@ -6,7 +8,7 @@ SearchProvidersRequestSchema, ) from cc_common.exceptions import CCInvalidRequestException -from cc_common.utils import api_handler, authorize_compact_level_only_action +from cc_common.utils import api_handler, authorize_compact_level_only_action, get_event_scopes from marshmallow import ValidationError from opensearch_client import OpenSearchClient @@ -61,6 +63,9 @@ def _search_providers(event: dict, context: LambdaContext): # noqa: ARG001 unus # Parse and validate the request body using the schema body = _parse_and_validate_request_body(event) + # If the query references dateOfBirth, verify the caller has readPrivate permission + _validate_date_of_birth_permission(body.get('query', {}), compact, get_event_scopes(event)) + # Build the OpenSearch search body search_body = _build_opensearch_search_body(body, size_override=MAX_PROVIDER_PAGE_SIZE) @@ -194,3 +199,56 @@ def _build_opensearch_search_body(body: dict, size_override: int) -> dict: raise CCInvalidRequestException('sort is required when using search_after pagination') return search_body + + +def _query_references_field(obj, field_name: str) -> bool: + """ + Recursively check if any key in the query DSL references the given field name. + + :param obj: The object to check (dict, list, or scalar) + :param field_name: The field name to search for in dict keys + :return: True if the field name is found in any key + """ + if isinstance(obj, dict): + for key, value in obj.items(): + if field_name in key: + return True + if _query_references_field(value, field_name): + return True + elif isinstance(obj, list): + for item in obj: + if _query_references_field(item, field_name): + return True + return False + + +def _caller_has_read_private_scope(compact: str, scopes: set[str]) -> bool: + """ + Check if the caller has readPrivate permission at either compact or jurisdiction level. + + :param compact: The compact abbreviation + :param scopes: The caller's scopes + :return: True if the caller has readPrivate permission + """ + action = CCPermissionsAction.READ_PRIVATE + + if f'{compact}/{action}' in scopes: + return True + + jurisdiction_scope_pattern = rf'.+/{compact}\.{action}$' + return any(match(jurisdiction_scope_pattern, scope) for scope in scopes) + + +def _validate_date_of_birth_permission(query: dict, compact: str, scopes: set[str]) -> None: + """ + Validate that the caller has readPrivate permission if the query references dateOfBirth. + + :param query: The OpenSearch query body + :param compact: The compact abbreviation + :param scopes: The caller's scopes + :raises CCInvalidRequestException: If dateOfBirth is in the query and the caller lacks readPrivate permission + """ + if _query_references_field(query, 'dateOfBirth') and not _caller_has_read_private_scope(compact, scopes): + raise CCInvalidRequestException( + 'Searching by dateOfBirth requires readPrivate permission' + ) diff --git a/backend/cosmetology-app/lambdas/python/search/opensearch_client.py b/backend/cosmetology-app/lambdas/python/search/opensearch_client.py index 0fc2214b0..166c4c740 100644 --- a/backend/cosmetology-app/lambdas/python/search/opensearch_client.py +++ b/backend/cosmetology-app/lambdas/python/search/opensearch_client.py @@ -208,6 +208,21 @@ def _extract_opensearch_error_reason(e: RequestError) -> str: ) return str(e.error) + def delete_provider_documents(self, index_name: str, provider_id: str) -> dict: + """ + Delete all OpenSearch documents for a given provider from the specified index. + + :param index_name: The name of the index to delete from + :param provider_id: The provider ID whose documents should be deleted + :return: The delete_by_query response from OpenSearch (includes 'deleted' count) + :raises CCInternalException: If all retry attempts fail + """ + query = {'term': {'providerId': provider_id}} + return self._execute_with_retry( + operation=lambda: self._client.delete_by_query(index=index_name, body={'query': query}), + operation_name=f'delete_provider_documents({index_name})', + ) + def bulk_index(self, index_name: str, documents: list[dict], id_field: str = 'providerId') -> dict: """ Bulk index multiple documents into the specified index. diff --git a/backend/cosmetology-app/lambdas/python/search/tests/__init__.py b/backend/cosmetology-app/lambdas/python/search/tests/__init__.py index 1d47cbe55..e7218acab 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/__init__.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/__init__.py @@ -19,6 +19,7 @@ def setUpClass(cls): 'ENVIRONMENT_NAME': 'test', 'COMPACTS': '["cosm"]', 'PROVIDER_TABLE_NAME': 'provider-table', + 'COMPACT_CONFIGURATION_TABLE_NAME': 'compact-config-table', 'PROV_DATE_OF_UPDATE_INDEX_NAME': 'providerDateOfUpdate', 'PROV_FAM_GIV_MID_INDEX_NAME': 'providerFamGivMid', 'LICENSE_GSI_NAME': 'licenseGSI', diff --git a/backend/cosmetology-app/lambdas/python/search/tests/function/__init__.py b/backend/cosmetology-app/lambdas/python/search/tests/function/__init__.py index 3d3f139f9..6bbe23817 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/function/__init__.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/function/__init__.py @@ -26,6 +26,7 @@ def setUp(self): # noqa: N801 invalid-name def build_resources(self): self.create_provider_table() + self.create_compact_configuration_table() self.create_export_results_bucket() def delete_resources(self): @@ -33,6 +34,7 @@ def delete_resources(self): # must delete all objects in the bucket before deleting the bucket self._bucket.objects.delete() self._bucket.delete() + self._compact_configuration_table.delete() def create_export_results_bucket(self): """Create the mock S3 bucket for export results""" @@ -91,3 +93,18 @@ def create_provider_table(self): }, ], ) + + def create_compact_configuration_table(self): + """Create the compact configuration table for testing.""" + self._compact_configuration_table = boto3.resource('dynamodb').create_table( + AttributeDefinitions=[ + {'AttributeName': 'pk', 'AttributeType': 'S'}, + {'AttributeName': 'sk', 'AttributeType': 'S'}, + ], + TableName=os.environ['COMPACT_CONFIGURATION_TABLE_NAME'], + KeySchema=[ + {'AttributeName': 'pk', 'KeyType': 'HASH'}, + {'AttributeName': 'sk', 'KeyType': 'RANGE'}, + ], + BillingMode='PAY_PER_REQUEST', + ) diff --git a/backend/cosmetology-app/lambdas/python/search/tests/function/test_manage_opensearch_indices.py b/backend/cosmetology-app/lambdas/python/search/tests/function/test_manage_opensearch_indices.py index 03d2fabe5..36fde869a 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/function/test_manage_opensearch_indices.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/function/test_manage_opensearch_indices.py @@ -169,6 +169,7 @@ def test_on_create_creates_versioned_indices_and_aliases_for_all_compacts_when_n }, 'compact': {'type': 'keyword'}, 'compactEligibility': {'type': 'keyword'}, + 'dateOfBirth': {'type': 'date'}, 'dateOfExpiration': {'type': 'date'}, 'dateOfIssuance': {'type': 'date'}, 'dateOfRenewal': {'type': 'date'}, diff --git a/backend/cosmetology-app/lambdas/python/search/tests/function/test_populate_provider_documents.py b/backend/cosmetology-app/lambdas/python/search/tests/function/test_populate_provider_documents.py index fcf74ca37..3f1a89233 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/function/test_populate_provider_documents.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/function/test_populate_provider_documents.py @@ -1,6 +1,7 @@ -from unittest.mock import Mock, call +from unittest.mock import Mock, patch from common_test.test_constants import ( + DEFAULT_DATE_OF_BIRTH, DEFAULT_LICENSE_EXPIRATION_DATE, DEFAULT_LICENSE_ISSUANCE_DATE, DEFAULT_LICENSE_RENEWAL_DATE, @@ -75,65 +76,85 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_inde if not bulk_index_response: bulk_index_response = {'items': [], 'errors': False} - # Create a mock instance that will be returned by the OpenSearchClient constructor mock_client_instance = Mock() mock_opensearch_client.return_value = mock_client_instance mock_client_instance.bulk_index.return_value = bulk_index_response return mock_client_instance - def _generate_expected_call_for_document(self, compact): - # Use timezone(timedelta(0), '+0000') to match how the code creates UTC timezone - return call( - index_name=f'compact_{compact}_providers', - documents=[ + def _generate_expected_document(self, compact): + provider_id = test_provider_id_mapping[compact] + license_type = test_license_type_mapping[compact] + return { + 'providerId': provider_id, + 'type': 'provider', + 'dateOfUpdate': DEFAULT_PROVIDER_UPDATE_DATETIME, + 'compact': compact, + 'licenseJurisdiction': 'oh', + 'licenseStatus': 'inactive', + 'compactEligibility': 'ineligible', + 'givenName': f'test{compact}GivenName', + 'middleName': 'Gunnar', + 'familyName': f'test{compact}FamilyName', + 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'birthMonthDay': '06-06', + 'documentId': f'{provider_id}#oh#{license_type}', + 'licenses': [ { - 'providerId': test_provider_id_mapping[compact], - 'type': 'provider', - 'dateOfUpdate': DEFAULT_PROVIDER_UPDATE_DATETIME, + 'providerId': provider_id, + 'type': 'license', + 'dateOfUpdate': DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, 'compact': compact, - 'licenseJurisdiction': 'oh', - 'currentHomeJurisdiction': 'oh', + 'jurisdiction': 'oh', + 'licenseType': license_type, + 'licenseStatusName': 'DEFINITELY_A_HUMAN', 'licenseStatus': 'inactive', + 'jurisdictionUploadedLicenseStatus': 'active', 'compactEligibility': 'ineligible', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'licenseNumber': 'A0608337260', 'givenName': f'test{compact}GivenName', 'middleName': 'Gunnar', 'familyName': f'test{compact}FamilyName', + 'dateOfIssuance': DEFAULT_LICENSE_ISSUANCE_DATE, + 'dateOfRenewal': DEFAULT_LICENSE_RENEWAL_DATE, 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, - 'jurisdictionUploadedLicenseStatus': 'active', - 'jurisdictionUploadedCompactEligibility': 'eligible', - 'birthMonthDay': '06-06', - 'licenses': [ - { - 'providerId': test_provider_id_mapping[compact], - 'type': 'license', - 'dateOfUpdate': DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, - 'compact': compact, - 'jurisdiction': 'oh', - 'licenseType': test_license_type_mapping[compact], - 'licenseStatusName': 'DEFINITELY_A_HUMAN', - 'licenseStatus': 'inactive', - 'jurisdictionUploadedLicenseStatus': 'active', - 'compactEligibility': 'ineligible', - 'jurisdictionUploadedCompactEligibility': 'eligible', - 'licenseNumber': 'A0608337260', - 'givenName': f'test{compact}GivenName', - 'middleName': 'Gunnar', - 'familyName': f'test{compact}FamilyName', - 'dateOfIssuance': DEFAULT_LICENSE_ISSUANCE_DATE, - 'dateOfRenewal': DEFAULT_LICENSE_RENEWAL_DATE, - 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, - 'homeAddressStreet1': '123 A St.', - 'homeAddressStreet2': 'Apt 321', - 'homeAddressCity': 'Columbus', - 'homeAddressState': 'oh', - 'homeAddressPostalCode': '43004', - 'emailAddress': 'björk@example.com', - 'phoneNumber': '+13213214321', - 'adverseActions': [], - 'investigations': [], - } - ], - 'privileges': [], + 'dateOfBirth': DEFAULT_DATE_OF_BIRTH, + 'homeAddressStreet1': '123 A St.', + 'homeAddressStreet2': 'Apt 321', + 'homeAddressCity': 'Columbus', + 'homeAddressState': 'oh', + 'homeAddressPostalCode': '43004', + 'emailAddress': 'björk@example.com', + 'phoneNumber': '+13213214321', + 'adverseActions': [], + 'investigations': [], } ], - ) + 'privileges': [], + } + + @patch('handlers.populate_provider_documents.OpenSearchClient') + def test_populate_indexes_document_with_document_id(self, mock_opensearch_client): + """Test that populate handler indexes documents with id_field='documentId'.""" + from handlers.populate_provider_documents import populate_provider_documents + + mock_client_instance = self._when_testing_mock_opensearch_client(mock_opensearch_client) + self._put_test_provider_and_license_record_in_dynamodb_table('cosm') + + mock_context = Mock() + mock_context.get_remaining_time_in_millis.return_value = 600000 + + result = populate_provider_documents({}, mock_context) + + self.assertTrue(result['completed']) + self.assertGreaterEqual(mock_client_instance.bulk_index.call_count, 1) + + bulk_index_call = mock_client_instance.bulk_index.call_args + self.assertEqual('compact_cosm_providers', bulk_index_call.kwargs['index_name']) + self.assertEqual('documentId', bulk_index_call.kwargs['id_field']) + + indexed_documents = bulk_index_call.kwargs['documents'] + self.assertEqual(1, len(indexed_documents)) + self.assertEqual(self._generate_expected_document('cosm'), indexed_documents[0]) diff --git a/backend/cosmetology-app/lambdas/python/search/tests/function/test_provider_update_ingest.py b/backend/cosmetology-app/lambdas/python/search/tests/function/test_provider_update_ingest.py index 103a7208e..7cb73bc47 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/function/test_provider_update_ingest.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/function/test_provider_update_ingest.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, patch from common_test.test_constants import ( + DEFAULT_DATE_OF_BIRTH, DEFAULT_LICENSE_EXPIRATION_DATE, DEFAULT_LICENSE_ISSUANCE_DATE, DEFAULT_LICENSE_RENEWAL_DATE, @@ -115,8 +116,8 @@ def _when_testing_mock_opensearch_client(self, mock_opensearch_client, bulk_inde if not bulk_index_response: bulk_index_response = {'items': [], 'errors': False} - # mock_opensearch_client is the patched instance, not the class mock_opensearch_client.bulk_index.return_value = bulk_index_response + mock_opensearch_client.delete_provider_documents.return_value = {'deleted': 0, 'failures': []} return mock_opensearch_client def _generate_expected_document(self, compact: str, provider_id: str = None) -> dict: @@ -124,6 +125,7 @@ def _generate_expected_document(self, compact: str, provider_id: str = None) -> if provider_id is None: provider_id = TEST_PROVIDER_ID_MAPPING[compact] + license_type = TEST_LICENSE_TYPE_MAPPING[compact] return { 'providerId': provider_id, 'type': 'provider', @@ -139,6 +141,7 @@ def _generate_expected_document(self, compact: str, provider_id: str = None) -> 'jurisdictionUploadedLicenseStatus': 'active', 'jurisdictionUploadedCompactEligibility': 'eligible', 'birthMonthDay': '06-06', + 'documentId': f'{provider_id}#oh#{license_type}', 'licenses': [ { 'providerId': provider_id, @@ -146,7 +149,7 @@ def _generate_expected_document(self, compact: str, provider_id: str = None) -> 'dateOfUpdate': DEFAULT_LICENSE_UPDATE_DATE_OF_UPDATE, 'compact': compact, 'jurisdiction': 'oh', - 'licenseType': TEST_LICENSE_TYPE_MAPPING[compact], + 'licenseType': license_type, 'licenseStatusName': 'DEFINITELY_A_HUMAN', 'licenseStatus': 'inactive', 'jurisdictionUploadedLicenseStatus': 'active', @@ -159,6 +162,7 @@ def _generate_expected_document(self, compact: str, provider_id: str = None) -> 'dateOfIssuance': DEFAULT_LICENSE_ISSUANCE_DATE, 'dateOfRenewal': DEFAULT_LICENSE_RENEWAL_DATE, 'dateOfExpiration': DEFAULT_LICENSE_EXPIRATION_DATE, + 'dateOfBirth': DEFAULT_DATE_OF_BIRTH, 'homeAddressStreet1': '123 A St.', 'homeAddressStreet2': 'Apt 321', 'homeAddressCity': 'Columbus', @@ -173,18 +177,50 @@ def _generate_expected_document(self, compact: str, provider_id: str = None) -> 'privileges': [], } + def _create_dynamodb_stream_record_with_old_image_only( + self, compact: str, provider_id: str, sequence_number: str + ) -> dict: + """Create a DynamoDB stream record for REMOVE events (only OldImage, no NewImage).""" + image_data = { + 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, + 'sk': {'S': f'{compact}#PROVIDER'}, + 'compact': {'S': compact}, + 'providerId': {'S': provider_id}, + 'type': {'S': 'provider'}, + 'givenName': {'S': f'test{compact}GivenName'}, + 'familyName': {'S': f'test{compact}FamilyName'}, + } + + return { + 'eventID': f'event-{sequence_number}', + 'eventName': 'REMOVE', + 'eventVersion': '1.1', + 'eventSource': 'aws:dynamodb', + 'awsRegion': 'us-east-1', + 'dynamodb': { + 'ApproximateCreationDateTime': 1234567890, + 'Keys': { + 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, + 'sk': {'S': f'{compact}#PROVIDER'}, + }, + 'OldImage': image_data, + 'SequenceNumber': sequence_number, + 'SizeBytes': 256, + 'StreamViewType': 'NEW_AND_OLD_IMAGES', + }, + 'eventSourceARN': 'arn:aws:dynamodb:us-east-1:123456789012:table/provider-table/stream/1234', + } + + # ---- INSERT/MODIFY path tests ---- + @patch('handlers.provider_update_ingest.opensearch_client') def test_opensearch_client_called_with_expected_parameters(self, mock_opensearch_client): """Test that OpenSearch client is called with expected parameters when indexing a record.""" from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client self._when_testing_mock_opensearch_client(mock_opensearch_client) - - # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('cosm') - # Create an SQS event with DynamoDB stream record in the body event = { 'Records': [ { @@ -200,19 +236,16 @@ def test_opensearch_client_called_with_expected_parameters(self, mock_opensearch ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that bulk_index was called once with expected parameters self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) - # Verify the call arguments call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual('compact_cosm_providers', call_args.kwargs['index_name']) self.assertEqual([self._generate_expected_document('cosm')], call_args.kwargs['documents']) + self.assertEqual('documentId', call_args.kwargs['id_field']) - # Verify no batch item failures self.assertEqual({'batchItemFailures': []}, result) @patch('handlers.provider_update_ingest.opensearch_client') @@ -220,13 +253,9 @@ def test_provider_ids_are_deduped_only_one_document_indexed(self, mock_opensearc """Test that duplicate provider IDs in the batch are deduplicated.""" from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client self._when_testing_mock_opensearch_client(mock_opensearch_client) - - # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('cosm') - # Create multiple SQS records for the SAME provider (simulating multiple updates) event = { 'Records': [ { @@ -265,19 +294,16 @@ def test_provider_ids_are_deduped_only_one_document_indexed(self, mock_opensearc ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that bulk_index was called only once despite 3 records self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) - # Verify only ONE document was indexed (deduplication worked) call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual(1, len(call_args.kwargs['documents'])) self.assertEqual(MOCK_COSM_PROVIDER_ID, call_args.kwargs['documents'][0]['providerId']) + self.assertEqual('documentId', call_args.kwargs['id_field']) - # Verify no batch item failures self.assertEqual({'batchItemFailures': []}, result) @patch('handlers.provider_update_ingest.opensearch_client') @@ -285,7 +311,6 @@ def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_cli """Test that a record that fails validation is returned in batchItemFailures.""" from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client self._when_testing_mock_opensearch_client(mock_opensearch_client) provider = self.test_data_generator.generate_default_provider( @@ -297,11 +322,9 @@ def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_cli } ) serialized_provider = provider.serialize_to_database_record() - # put invalid compact to fail validation serialized_provider['compact'] = 'foo' self.config.provider_table.put_item(Item=serialized_provider) - # Create SQS event with DynamoDB stream record in the body event = { 'Records': [ { @@ -317,11 +340,9 @@ def test_validation_failure_returns_batch_item_failure(self, mock_opensearch_cli ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that the batch item failure is returned with the message ID self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) @@ -330,13 +351,13 @@ def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opens """Test that a record which fails to be indexed by OpenSearch is in batchItemFailures.""" from handlers.provider_update_ingest import provider_update_ingest_handler - # Simulate OpenSearch returning an error for one document + document_id = f'{MOCK_COSM_PROVIDER_ID}#oh#cosmetologist' mock_opensearch_client.bulk_index.return_value = { 'errors': True, 'items': [ { 'index': { - '_id': MOCK_COSM_PROVIDER_ID, + '_id': document_id, '_index': 'compact_cosm_providers', 'status': 400, 'error': { @@ -348,10 +369,8 @@ def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opens ], } - # Create provider and license records in DynamoDB for both compacts self._put_test_provider_and_license_record_in_dynamodb_table('cosm') - # Create SQS events with DynamoDB stream records in the body for both providers event = { 'Records': [ { @@ -367,11 +386,9 @@ def test_opensearch_indexing_failure_returns_batch_item_failure(self, mock_opens ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that only the failed document's message ID is in batchItemFailures self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) @@ -381,14 +398,10 @@ def test_bulk_index_exception_returns_all_batch_item_failures(self, mock_opensea from cc_common.exceptions import CCInternalException from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client to raise an exception mock_opensearch_client.bulk_index.side_effect = CCInternalException('Connection timeout after 5 retries') - # Create provider and license records in DynamoDB for both compacts - self._put_test_provider_and_license_record_in_dynamodb_table('cosm') self._put_test_provider_and_license_record_in_dynamodb_table('cosm') - # Create SQS events with DynamoDB stream records in the body for both providers event = { 'Records': [ { @@ -414,11 +427,9 @@ def test_bulk_index_exception_returns_all_batch_item_failures(self, mock_opensea ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that both records were returned in batch failures self.assertEqual(2, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) self.assertEqual('12346', result['batchItemFailures'][1]['itemIdentifier']) @@ -433,29 +444,17 @@ def test_empty_records_returns_empty_batch_failures(self, mock_opensearch_client mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify empty response self.assertEqual({'batchItemFailures': []}, result) - - # Verify OpenSearch client was never called mock_opensearch_client.bulk_index.assert_not_called() @patch('handlers.provider_update_ingest.opensearch_client') def test_insert_event_without_old_image_indexes_successfully(self, mock_opensearch_client): - """Test that INSERT events (newly created records) without OldImage are processed correctly. - - When a new record is created in DynamoDB, the stream event contains only NewImage - and no OldImage. The handler should extract the compact and providerId from NewImage - and successfully index the document. - """ + """Test that INSERT events (newly created records) without OldImage are processed correctly.""" from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client self._when_testing_mock_opensearch_client(mock_opensearch_client) - - # Create provider and license records in DynamoDB self._put_test_provider_and_license_record_in_dynamodb_table('cosm') - # Create an SQS event with DynamoDB stream record in the body for INSERT (no OldImage) event = { 'Records': [ { @@ -466,79 +465,47 @@ def test_insert_event_without_old_image_indexes_successfully(self, mock_opensear provider_id=MOCK_COSM_PROVIDER_ID, sequence_number='some-sequence-number', event_name='INSERT', - include_old_image=False, # INSERT events don't have OldImage + include_old_image=False, ) ), } ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that bulk_index was called with the correct parameters self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) - # Verify the call arguments call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual('compact_cosm_providers', call_args.kwargs['index_name']) self.assertEqual([self._generate_expected_document('cosm')], call_args.kwargs['documents']) + self.assertEqual('documentId', call_args.kwargs['id_field']) - # Verify no batch item failures for INSERT event - self.assertEqual({'batchItemFailures': []}, result) + # No delete_provider_documents should be called for INSERT events + mock_opensearch_client.delete_provider_documents.assert_not_called() - def _create_dynamodb_stream_record_with_old_image_only( - self, compact: str, provider_id: str, sequence_number: str - ) -> dict: - """Create a DynamoDB stream record for REMOVE events (only OldImage, no NewImage).""" - image_data = { - 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, - 'sk': {'S': f'{compact}#PROVIDER'}, - 'compact': {'S': compact}, - 'providerId': {'S': provider_id}, - 'type': {'S': 'provider'}, - 'givenName': {'S': f'test{compact}GivenName'}, - 'familyName': {'S': f'test{compact}FamilyName'}, - } + self.assertEqual({'batchItemFailures': []}, result) - return { - 'eventID': f'event-{sequence_number}', - 'eventName': 'REMOVE', - 'eventVersion': '1.1', - 'eventSource': 'aws:dynamodb', - 'awsRegion': 'us-east-1', - 'dynamodb': { - 'ApproximateCreationDateTime': 1234567890, - 'Keys': { - 'pk': {'S': f'{compact}#PROVIDER#{provider_id}'}, - 'sk': {'S': f'{compact}#PROVIDER'}, - }, - 'OldImage': image_data, # REMOVE events only have OldImage - 'SequenceNumber': sequence_number, - 'SizeBytes': 256, - 'StreamViewType': 'NEW_AND_OLD_IMAGES', - }, - 'eventSourceARN': 'arn:aws:dynamodb:us-east-1:123456789012:table/provider-table/stream/1234', - } + # ---- REMOVE event path tests ---- @patch('handlers.provider_update_ingest.opensearch_client') - def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opensearch_client): - """Test that REMOVE events (deleted records) with only OldImage are processed correctly. - - When a record is deleted from DynamoDB, the stream event contains only OldImage - and no NewImage. The handler should extract the compact and providerId from OldImage - and still index/update the document (to reflect the latest state of the provider). + def test_remove_event_with_remaining_records_deletes_then_reindexes(self, mock_opensearch_client): + """Test that REMOVE events trigger delete_provider_documents then re-index remaining records. + + When a single record (e.g., a license) is deleted but the provider still has other records + in DynamoDB, the handler should: + 1. Call delete_provider_documents to remove all documents for the provider + 2. Re-check DynamoDB and find the provider still exists + 3. Re-index the remaining license documents """ from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client self._when_testing_mock_opensearch_client(mock_opensearch_client) - # Create provider and license records in DynamoDB + # Provider still exists in DynamoDB with remaining records self._put_test_provider_and_license_record_in_dynamodb_table('cosm') - # Create an SQS event with DynamoDB stream record in the body for REMOVE (only OldImage, no NewImage) event = { 'Records': [ { @@ -554,126 +521,115 @@ def test_remove_event_with_only_old_image_indexes_successfully(self, mock_opense ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that bulk_index was called with the correct parameters - self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) + # delete_provider_documents should be called to remove all existing docs for this provider + mock_opensearch_client.delete_provider_documents.assert_called_once_with( + index_name='compact_cosm_providers', + provider_id=MOCK_COSM_PROVIDER_ID, + ) - # Verify the call arguments + # bulk_index should be called with the remaining documents + self.assertEqual(1, mock_opensearch_client.bulk_index.call_count) call_args = mock_opensearch_client.bulk_index.call_args self.assertEqual('compact_cosm_providers', call_args.kwargs['index_name']) self.assertEqual([self._generate_expected_document('cosm')], call_args.kwargs['documents']) + self.assertEqual('documentId', call_args.kwargs['id_field']) - # Verify no batch item failures for REMOVE event self.assertEqual({'batchItemFailures': []}, result) @patch('handlers.provider_update_ingest.opensearch_client') - def test_provider_deleted_from_index_when_no_records_found(self, mock_opensearch_client): - """Test that when no provider records are found (CCNotFoundException), bulk_delete is called. - - This scenario occurs when a provider is completely removed from the system, - such as during a license upload rollback. The handler should call bulk_delete - to remove the provider document from the OpenSearch index. + def test_remove_event_provider_fully_deleted_no_reindex(self, mock_opensearch_client): + """Test that REMOVE events for a fully deleted provider just delete from OpenSearch. + + When a REMOVE event occurs and the provider no longer exists in DynamoDB at all, + the handler should: + 1. Call delete_provider_documents to remove all documents for the provider + 2. Re-check DynamoDB and find the provider does NOT exist + 3. NOT attempt to re-index """ from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client - mock_opensearch_client.bulk_index.return_value = {'items': [], 'errors': False} - mock_opensearch_client.bulk_delete.return_value = set() # bulk_delete returns a set of failed IDs + self._when_testing_mock_opensearch_client(mock_opensearch_client) - # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted + # Do NOT create any provider records in DynamoDB - provider is fully deleted - # Create an SQS event with DynamoDB stream record in the body for a provider that no longer exists event = { 'Records': [ { 'messageId': '12345', 'body': json.dumps( - self._create_dynamodb_stream_record( + self._create_dynamodb_stream_record_with_old_image_only( compact='cosm', provider_id=MOCK_COSM_PROVIDER_ID, sequence_number='some-sequence-number', - event_name='REMOVE', - include_old_image=False, ) ), } ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that bulk_index was NOT called (no documents to index) - mock_opensearch_client.bulk_index.assert_not_called() + # delete_provider_documents should be called + mock_opensearch_client.delete_provider_documents.assert_called_once_with( + index_name='compact_cosm_providers', + provider_id=MOCK_COSM_PROVIDER_ID, + ) - # Assert that bulk_delete WAS called with the correct parameters - self.assertEqual(1, mock_opensearch_client.bulk_delete.call_count) - call_args = mock_opensearch_client.bulk_delete.call_args - self.assertEqual('compact_cosm_providers', call_args.kwargs['index_name']) - self.assertEqual([MOCK_COSM_PROVIDER_ID], call_args.kwargs['document_ids']) + # bulk_index should NOT be called (provider no longer exists) + mock_opensearch_client.bulk_index.assert_not_called() - # Verify no batch item failures (deletion is expected behavior, not a failure) self.assertEqual({'batchItemFailures': []}, result) @patch('handlers.provider_update_ingest.opensearch_client') - def test_bulk_delete_failure_returns_batch_item_failure(self, mock_opensearch_client): - """Test that when bulk_delete fails, the provider is returned in batchItemFailures.""" + def test_delete_provider_documents_failure_returns_batch_item_failure(self, mock_opensearch_client): + """Test that when delete_provider_documents fails, the provider is returned in batchItemFailures.""" from cc_common.exceptions import CCInternalException from handlers.provider_update_ingest import provider_update_ingest_handler - # Set up mock OpenSearch client - bulk_delete raises exception - mock_opensearch_client.bulk_delete.side_effect = CCInternalException('Connection timeout after 5 retries') - - # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted + mock_opensearch_client.delete_provider_documents.side_effect = CCInternalException( + 'Connection timeout after 5 retries' + ) - # Create an SQS event with DynamoDB stream record in the body for a provider that no longer exists event = { 'Records': [ { 'messageId': '12345', 'body': json.dumps( - self._create_dynamodb_stream_record( + self._create_dynamodb_stream_record_with_old_image_only( compact='cosm', provider_id=MOCK_COSM_PROVIDER_ID, sequence_number='some-sequence-number', - event_name='REMOVE', - include_old_image=False, ) ), } ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Verify that the batch item failure is returned with the message ID self.assertEqual(1, len(result['batchItemFailures'])) self.assertEqual('12345', result['batchItemFailures'][0]['itemIdentifier']) @patch('handlers.provider_update_ingest.opensearch_client') - def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock_opensearch_client): - """Test that when bulk_delete returns 404 (document not found), it is NOT treated as a failure. + def test_cc_not_found_on_non_remove_event_logs_warning_no_reindex(self, mock_opensearch_client): + """Test that CCNotFoundException on a non-REMOVE event logs a warning without re-indexing. - This scenario occurs when a provider document has already been deleted from OpenSearch - (e.g., a previous delete succeeded, or the document never existed in the index). - The 404 response should be ignored since the desired end state (document not in index) - has been achieved. + This is a safety net for race conditions where a MODIFY/INSERT event arrives but the + provider has already been deleted from DynamoDB. The handler should log a warning + and NOT attempt to re-index. """ from handlers.provider_update_ingest import provider_update_ingest_handler - # Simulate OpenSearch bulk delete response when document doesn't exist - # bulk_delete returns a set of failed document IDs, empty set means no failures (404 is ignored) - mock_opensearch_client.bulk_delete.return_value = set() + self._when_testing_mock_opensearch_client(mock_opensearch_client) - # Do NOT create any provider records in DynamoDB - this simulates the provider being deleted + # Do NOT create any provider records in DynamoDB - simulates race condition + # where provider was deleted between event creation and processing - # Create a DynamoDB stream event for a provider that no longer exists event = { 'Records': [ { @@ -683,20 +639,24 @@ def test_bulk_delete_404_not_found_does_not_return_batch_item_failure(self, mock compact='cosm', provider_id=MOCK_COSM_PROVIDER_ID, sequence_number='some-sequence-number', - event_name='REMOVE', - include_old_image=False, + event_name='MODIFY', ) ), } ] } - # Run the handler mock_context = MagicMock() result = provider_update_ingest_handler(event, mock_context) - # Assert that bulk_delete was called - self.assertEqual(1, mock_opensearch_client.bulk_delete.call_count) + # delete_provider_documents should be called to remove documents from OpenSearch + mock_opensearch_client.delete_provider_documents.assert_called_once_with( + index_name='compact_cosm_providers', + provider_id=MOCK_COSM_PROVIDER_ID, + ) + + # No bulk_index should be called (no documents to index) + mock_opensearch_client.bulk_index.assert_not_called() - # Verify NO batch item failures - 404 is not treated as an error + # No batch failures - this is expected behavior for a race condition self.assertEqual({'batchItemFailures': []}, result) diff --git a/backend/cosmetology-app/lambdas/python/search/tests/function/test_public_search_providers.py b/backend/cosmetology-app/lambdas/python/search/tests/function/test_public_search_providers.py new file mode 100644 index 000000000..0e7b666d0 --- /dev/null +++ b/backend/cosmetology-app/lambdas/python/search/tests/function/test_public_search_providers.py @@ -0,0 +1,383 @@ +import json +from unittest.mock import patch + +from moto import mock_aws + +from . import TstFunction + + +@mock_aws +class TestPublicSearchProviders(TstFunction): + """Test suite for public_search_api_handler - public license search via OpenSearch.""" + + def setUp(self): + super().setUp() + + def _create_public_api_event(self, compact: str, body: dict = None) -> dict: + """Create API Gateway event for public query providers (no auth).""" + return { + 'resource': '/v1/public/compacts/{compact}/providers/query', + 'path': f'/v1/public/compacts/{compact}/providers/query', + 'httpMethod': 'POST', + 'headers': {'accept': 'application/json', 'content-type': 'application/json'}, + 'multiValueHeaders': {}, + 'queryStringParameters': None, + 'pathParameters': {'compact': compact}, + 'requestContext': { + 'httpMethod': 'POST', + 'resourcePath': '/v1/public/compacts/{compact}/providers/query', + }, + 'body': json.dumps(body) if body else None, + 'isBase64Encoded': False, + } + + def _create_mock_hit( + self, + provider_id: str = '00000000-0000-0000-0000-000000000001', + compact: str = 'cosm', + jurisdiction: str = 'oh', + license_number: str = 'LN123', + family_name: str = 'Doe', + given_name: str = 'John', + sort_values: list = None, + license_type: str = 'cosmetologist', + ) -> dict: + """Create a mock OpenSearch hit for one document per license.""" + doc_id = f'{provider_id}#{jurisdiction}#{license_type}' + hit = { + '_index': f'compact_{compact}_providers', + '_id': doc_id, + '_source': { + 'providerId': provider_id, + 'compact': compact, + 'givenName': given_name, + 'familyName': family_name, + 'licenses': [ + { + 'jurisdiction': jurisdiction, + 'licenseNumber': license_number, + 'licenseType': license_type, + } + ], + }, + } + if sort_values is not None: + hit['sort'] = sort_values + return hit + + @patch('handlers.public_search.opensearch_client') + def test_license_number_search_builds_nested_query(self, mock_opensearch_client): + """Test that licenseNumber in query builds nested term query on licenses.licenseNumber.""" + from handlers.public_search import public_search_api_handler + + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'hits': []}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'licenseNumber': 'LN999'}, 'pagination': {'pageSize': 10}}, + ) + public_search_api_handler(event, self.mock_context) + call_body = mock_opensearch_client.search.call_args.kwargs['body'] + self.assertIn('query', call_body) + must = call_body['query']['bool']['must'] + nested = next(m for m in must if 'nested' in m) + self.assertEqual('licenses', nested['nested']['path']) + self.assertNotIn('inner_hits', nested['nested']) + inner_must = nested['nested']['query']['bool']['must'] + self.assertIn({'term': {'licenses.licenseNumber': 'LN999'}}, inner_must) + + @patch('handlers.public_search.opensearch_client') + def test_jurisdiction_and_name_search_builds_nested_query(self, mock_opensearch_client): + """Test that jurisdiction and familyName build correct nested query.""" + from handlers.public_search import public_search_api_handler + + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'hits': []}, + } + event = self._create_public_api_event( + 'cosm', + body={ + 'query': {'jurisdiction': 'oh', 'familyName': 'Smith'}, + 'pagination': {'pageSize': 10}, + }, + ) + public_search_api_handler(event, self.mock_context) + call_body = mock_opensearch_client.search.call_args.kwargs['body'] + must = call_body['query']['bool']['must'] + nested = next(m for m in must if 'nested' in m) + self.assertNotIn('inner_hits', nested['nested']) + inner_must = nested['nested']['query']['bool']['must'] + self.assertIn({'term': {'licenses.jurisdiction': 'oh'}}, inner_must) + self.assertTrue(any('licenses.familyName' in str(m) for m in inner_must)) + + @patch('handlers.public_search.opensearch_client') + def test_name_only_search_builds_nested_query(self, mock_opensearch_client): + """Test that familyName only builds nested match on licenses.familyName.""" + from handlers.public_search import public_search_api_handler + + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'hits': []}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'familyName': 'Jones'}, 'pagination': {'pageSize': 10}}, + ) + public_search_api_handler(event, self.mock_context) + call_body = mock_opensearch_client.search.call_args.kwargs['body'] + must = call_body['query']['bool']['must'] + nested = next(m for m in must if 'nested' in m) + self.assertNotIn('inner_hits', nested['nested']) + inner_must = nested['nested']['query']['bool']['must'] + self.assertTrue(any('familyName' in str(m) for m in inner_must)) + + @patch('handlers.public_search.opensearch_client') + def test_sort_includes_id_tiebreaker(self, mock_opensearch_client): + """OpenSearch sort includes _id as the fourth tiebreaker for deterministic pagination.""" + from handlers.public_search import public_search_api_handler + + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'hits': []}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'familyName': 'Doe'}, 'pagination': {'pageSize': 10}}, + ) + public_search_api_handler(event, self.mock_context) + call_body = mock_opensearch_client.search.call_args.kwargs['body'] + sort = call_body['sort'] + self.assertEqual(4, len(sort)) + self.assertEqual({'_id': 'asc'}, sort[3]) + + def test_given_name_without_family_name_returns_400(self): + """Test that givenName without familyName returns 400.""" + from handlers.public_search import public_search_api_handler + + event = self._create_public_api_event( + 'cosm', + body={'query': {'givenName': 'John'}, 'pagination': {'pageSize': 10}}, + ) + response = public_search_api_handler(event, self.mock_context) + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('familyName is required if givenName is provided', body['message']) + + def test_no_search_criteria_returns_400(self): + """Test that at least one of licenseNumber, jurisdiction, or familyName is required.""" + from handlers.public_search import public_search_api_handler + + event = self._create_public_api_event( + 'cosm', + body={'query': {}, 'pagination': {'pageSize': 10}}, + ) + response = public_search_api_handler(event, self.mock_context) + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('At least one of licenseNumber, jurisdiction, or familyName', body['message']) + + @patch('handlers.public_search.opensearch_client') + def test_pagination_page_size_maps_to_size_and_search_after_from_last_key(self, mock_opensearch_client): + """Test that pageSize maps to size and lastKey decodes to search_after.""" + from base64 import b64encode + + from handlers.public_search import public_search_api_handler + + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'hits': []}, + } + last_key_payload = json.dumps( + {'search_after': ['doe', 'jane', 'uuid-123', 'uuid-123#oh#cosmetologist']} + ) + last_key_str = b64encode(last_key_payload.encode('utf-8')).decode('utf-8') + event = self._create_public_api_event( + 'cosm', + body={ + 'query': {'familyName': 'Doe'}, + 'pagination': {'pageSize': 25, 'lastKey': last_key_str}, + }, + ) + public_search_api_handler(event, self.mock_context) + call_body = mock_opensearch_client.search.call_args.kwargs['body'] + self.assertEqual(25, call_body['size']) + self.assertEqual( + ['doe', 'jane', 'uuid-123', 'uuid-123#oh#cosmetologist'], + call_body['search_after'], + ) + + @patch('handlers.public_search.opensearch_client') + def test_response_last_key_encodes_last_hit_sort_when_full_page(self, mock_opensearch_client): + """When OpenSearch returns a full page of hits, lastKey encodes search_after from the last hit.""" + from base64 import b64decode + + from handlers.public_search import public_search_api_handler + + mock_hits_full_page = [] + for i in range(5): + sort_i = [ + 'doe', + 'john', + f'00000000-0000-0000-0000-00000000000{i}', + f'00000000-0000-0000-0000-00000000000{i}#oh#cosmetologist', + ] + mock_hits_full_page.append( + self._create_mock_hit( + provider_id=f'00000000-0000-0000-0000-00000000000{i}', + sort_values=sort_i, + ) + ) + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 10, 'relation': 'eq'}, 'hits': mock_hits_full_page}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'familyName': 'Doe'}, 'pagination': {'pageSize': 5}}, + ) + response = public_search_api_handler(event, self.mock_context) + body = json.loads(response['body']) + self.assertIn('lastKey', body['pagination']) + self.assertIsNotNone(body['pagination']['lastKey']) + decoded = json.loads(b64decode(body['pagination']['lastKey']).decode('utf-8')) + self.assertEqual(decoded['search_after'], mock_hits_full_page[-1]['sort']) + + @patch('handlers.public_search.opensearch_client') + def test_response_last_key_null_when_fewer_hits_than_page_size(self, mock_opensearch_client): + """When hit count is below pageSize, there are no more pages and lastKey is null.""" + from handlers.public_search import public_search_api_handler + + sort_four = [ + 'doe', + 'john', + '00000000-0000-0000-0000-000000000001', + '00000000-0000-0000-0000-000000000001#oh#cosmetologist', + ] + single_hit = self._create_mock_hit(sort_values=sort_four) + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 1, 'relation': 'eq'}, 'hits': [single_hit]}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'familyName': 'Doe'}, 'pagination': {'pageSize': 100}}, + ) + response = public_search_api_handler(event, self.mock_context) + body = json.loads(response['body']) + self.assertIsNone(body['pagination']['lastKey']) + + @patch('handlers.public_search.opensearch_client') + def test_response_contains_only_allowed_license_fields(self, mock_opensearch_client): + """Test that each item in providers has only expected fields.""" + from handlers.public_search import public_search_api_handler + + mock_hit = self._create_mock_hit() + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 1, 'relation': 'eq'}, 'hits': [mock_hit]}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'licenseNumber': 'LN123'}, 'pagination': {'pageSize': 10}}, + ) + response = public_search_api_handler(event, self.mock_context) + body = json.loads(response['body']) + self.assertEqual(len(body['providers']), 1) + provider = body['providers'][0] + allowed = {'providerId', 'givenName', 'familyName', 'licenseJurisdiction', 'compact', 'licenseNumber'} + self.assertEqual(set(provider.keys()), allowed) + self.assertEqual(provider['licenseJurisdiction'], 'oh') + self.assertEqual(provider['licenseNumber'], 'LN123') + + @patch('handlers.public_search.opensearch_client') + def test_compact_mismatch_filtered_out(self, mock_opensearch_client): + """Test that hits with compact != path compact are not included in results.""" + from handlers.public_search import public_search_api_handler + + mock_hit = self._create_mock_hit(compact='other') + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 1, 'relation': 'eq'}, 'hits': [mock_hit]}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'familyName': 'Doe'}, 'pagination': {'pageSize': 10}}, + ) + response = public_search_api_handler(event, self.mock_context) + body = json.loads(response['body']) + self.assertEqual(body['providers'], []) + + def test_invalid_request_body_returns_400(self): + """Test that invalid or missing body returns 400.""" + from handlers.public_search import public_search_api_handler + + event = self._create_public_api_event('cosm', body=None) + event['body'] = 'not valid json' + response = public_search_api_handler(event, self.mock_context) + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('Invalid request', body['message']) + + def test_unsupported_route_returns_400(self): + """Test that wrong method/path returns 400.""" + from handlers.public_search import public_search_api_handler + + event = self._create_public_api_event('cosm', body={'query': {'familyName': 'x'}}) + event['resource'] = '/v1/public/compacts/{compact}/providers/other' + response = public_search_api_handler(event, self.mock_context) + self.assertEqual(400, response['statusCode']) + self.assertIn('Unsupported method or resource', json.loads(response['body'])['message']) + + @patch('handlers.public_search.opensearch_client') + def test_terminal_page_returns_last_key_null(self, mock_opensearch_client): + """When fewer hits than pageSize, lastKey must be null.""" + from handlers.public_search import public_search_api_handler + + mock_hits = [ + self._create_mock_hit( + provider_id='pid-1', + jurisdiction='oh', + license_number='L1', + sort_values=['doe', 'john', 'pid-1', 'pid-1#oh#cosmetologist'], + ), + self._create_mock_hit( + provider_id='pid-1', + jurisdiction='al', + license_number='L2', + sort_values=['doe', 'john', 'pid-1', 'pid-1#al#cosmetologist'], + ), + self._create_mock_hit( + provider_id='pid-2', + jurisdiction='oh', + license_number='L3', + sort_values=['doe', 'john', 'pid-2', 'pid-2#oh#cosmetologist'], + ), + ] + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 3, 'relation': 'eq'}, 'hits': mock_hits}, + } + event = self._create_public_api_event( + 'cosm', + body={'query': {'familyName': 'Doe'}, 'pagination': {'pageSize': 10}}, + ) + response = public_search_api_handler(event, self.mock_context) + body = json.loads(response['body']) + self.assertIsNone(body['pagination']['lastKey'], 'no more results -> lastKey null') + + @patch('handlers.public_search.opensearch_client') + def test_invalid_last_key_format_returns_400(self, mock_opensearch_client): + """Malformed or invalid lastKey must return 400.""" + from base64 import b64encode + + from handlers.public_search import public_search_api_handler + + mock_opensearch_client.search.return_value = { + 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'hits': []}, + } + bad_payload = json.dumps({}) + last_key = b64encode(bad_payload.encode('utf-8')).decode('utf-8') + event = self._create_public_api_event( + 'cosm', + body={ + 'query': {'familyName': 'Doe'}, + 'pagination': {'pageSize': 10, 'lastKey': last_key}, + }, + ) + response = public_search_api_handler(event, self.mock_context) + self.assertEqual(response['statusCode'], 400) + body = json.loads(response['body']) + self.assertIn('lastkey', body['message'].lower()) diff --git a/backend/cosmetology-app/lambdas/python/search/tests/function/test_search_providers.py b/backend/cosmetology-app/lambdas/python/search/tests/function/test_search_providers.py index 0df39642b..b3a322042 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/function/test_search_providers.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/function/test_search_providers.py @@ -76,10 +76,11 @@ def _create_mock_provider_hit( compact: str = 'cosm', sort_values: list = None, ) -> dict: - """Create a mock OpenSearch hit for a provider document.""" + """Create a mock OpenSearch hit for a one-doc-per-license provider document.""" + document_id = f'{provider_id}#oh#cosmetologist' hit = { '_index': f'compact_{compact}_providers', - '_id': provider_id, + '_id': document_id, '_score': 1.0, '_source': { 'providerId': provider_id, @@ -95,14 +96,36 @@ def _create_mock_provider_hit( 'jurisdictionUploadedLicenseStatus': 'active', 'jurisdictionUploadedCompactEligibility': 'eligible', 'birthMonthDay': '06-15', - # adding a couple of fields that are not recognized in the - # ProviderGeneralResponseSchema. Although these are not currently - # stored in OpenSearch, this mock data ensures we are sanitizing - # these private fields by the search serialization logic + 'documentId': document_id, + 'licenses': [ + { + 'providerId': provider_id, + 'type': 'license', + 'compact': compact, + 'jurisdiction': 'oh', + 'licenseType': 'cosmetologist', + 'licenseNumber': 'A0608337260', + 'givenName': 'John', + 'familyName': 'Doe', + 'dateOfIssuance': '2024-01-01', + 'dateOfExpiration': '2025-12-31', + 'licenseStatus': 'active', + 'compactEligibility': 'eligible', + 'jurisdictionUploadedLicenseStatus': 'active', + 'jurisdictionUploadedCompactEligibility': 'eligible', + 'dateOfUpdate': '2024-01-15T10:30:00+00:00', + 'dateOfBirth': '1984-12-11', + 'homeAddressStreet1': '123 Main St', + 'homeAddressCity': 'Columbus', + 'homeAddressState': 'oh', + 'homeAddressPostalCode': '43004', + } + ], + 'privileges': [], + # Fields that should be stripped by ForgivingSchema 'someNewField': 'somePrivateValue', 'ssnLastFour': '1234', 'emailAddress': 'someemail@address.com', - 'dateOfBirth': '1984-12-11', }, } if sort_values: @@ -269,29 +292,30 @@ def test_search_returns_sanitized_providers(self, mock_opensearch_client): self.assertEqual(200, response['statusCode']) body = json.loads(response['body']) - self.assertEqual( - { - 'providers': [ - { - 'birthMonthDay': '06-15', - 'compact': 'cosm', - 'compactEligibility': 'eligible', - 'dateOfExpiration': '2025-12-31', - 'dateOfUpdate': '2024-01-15T10:30:00+00:00', - 'familyName': 'Doe', - 'givenName': 'John', - 'jurisdictionUploadedCompactEligibility': 'eligible', - 'jurisdictionUploadedLicenseStatus': 'active', - 'licenseJurisdiction': 'oh', - 'licenseStatus': 'active', - 'providerId': '00000000-0000-0000-0000-000000000001', - 'type': 'provider', - } - ], - 'total': {'relation': 'eq', 'value': 1}, - }, - body, - ) + providers = body['providers'] + self.assertEqual(1, len(providers)) + provider = providers[0] + # Verify provider-level fields are present and sanitized + self.assertEqual('cosm', provider['compact']) + self.assertEqual('John', provider['givenName']) + self.assertEqual('Doe', provider['familyName']) + self.assertEqual('oh', provider['licenseJurisdiction']) + self.assertEqual('active', provider['licenseStatus']) + self.assertEqual('eligible', provider['compactEligibility']) + self.assertEqual('06-15', provider['birthMonthDay']) + self.assertEqual('00000000-0000-0000-0000-000000000001', provider['providerId']) + # Verify licenses array with one license is present + self.assertEqual(1, len(provider['licenses'])) + self.assertEqual('oh', provider['licenses'][0]['jurisdiction']) + self.assertEqual('cosmetologist', provider['licenses'][0]['licenseType']) + # Verify private fields were stripped + self.assertNotIn('ssnLastFour', provider) + self.assertNotIn('someNewField', provider) + self.assertNotIn('emailAddress', provider) + # Verify documentId was stripped by ForgivingSchema + self.assertNotIn('documentId', provider) + # Verify total + self.assertEqual({'relation': 'eq', 'value': 1}, body['total']) @patch('handlers.search.opensearch_client') def test_search_response_includes_last_sort_for_pagination(self, mock_opensearch_client): @@ -467,3 +491,105 @@ def test_opensearch_request_error_returns_400_with_error_message(self, mock_open self.assertEqual(400, response['statusCode']) body = json.loads(response['body']) self.assertEqual(error_reason, body['message']) + + @patch('handlers.search.opensearch_client') + def test_search_with_date_of_birth_query_allowed_for_compact_level_read_private_scope( + self, mock_opensearch_client + ): + """Test that a query containing dateOfBirth succeeds when the caller has compact-level readPrivate scope.""" + from handlers.search import search_api_handler + + self._when_testing_mock_opensearch_client(mock_opensearch_client) + + query = { + 'nested': { + 'path': 'licenses', + 'query': {'term': {'licenses.dateOfBirth': '1985-06-06'}}, + } + } + event = self._create_api_event( + 'cosm', + body={'query': query}, + scopes_override='openid email cosm/readGeneral cosm/readPrivate', + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + mock_opensearch_client.search.assert_called_once() + + @patch('handlers.search.opensearch_client') + def test_search_with_date_of_birth_query_allowed_for_jurisdiction_level_read_private_scope( + self, mock_opensearch_client + ): + """Test that a query containing dateOfBirth succeeds when the caller has a jurisdiction-level readPrivate scope.""" + from handlers.search import search_api_handler + + self._when_testing_mock_opensearch_client(mock_opensearch_client) + + query = { + 'nested': { + 'path': 'licenses', + 'query': {'term': {'licenses.dateOfBirth': '1985-06-06'}}, + } + } + event = self._create_api_event( + 'cosm', + body={'query': query}, + scopes_override='openid email cosm/readGeneral oh/cosm.readPrivate', + ) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(200, response['statusCode']) + mock_opensearch_client.search.assert_called_once() + + def test_search_with_date_of_birth_query_rejected_without_read_private_scope(self): + """Test that a query containing dateOfBirth returns 400 when the caller only has readGeneral scope.""" + from handlers.search import search_api_handler + + query = { + 'nested': { + 'path': 'licenses', + 'query': {'term': {'licenses.dateOfBirth': '1985-06-06'}}, + } + } + event = self._create_api_event('cosm', body={'query': query}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('dateOfBirth', body['message']) + + def test_search_with_nested_date_of_birth_query_rejected_without_read_private_scope(self): + """Test that deeply nested dateOfBirth references are caught and rejected.""" + from handlers.search import search_api_handler + + query = { + 'bool': { + 'must': [ + {'match': {'givenName': 'John'}}, + { + 'nested': { + 'path': 'licenses', + 'query': { + 'bool': { + 'must': [ + {'term': {'licenses.jurisdiction': 'oh'}}, + {'range': {'licenses.dateOfBirth': {'gte': '1985-01-01'}}}, + ] + } + }, + } + }, + ] + } + } + event = self._create_api_event('cosm', body={'query': query}) + + response = search_api_handler(event, self.mock_context) + + self.assertEqual(400, response['statusCode']) + body = json.loads(response['body']) + self.assertIn('dateOfBirth', body['message']) diff --git a/backend/cosmetology-app/lambdas/python/search/tests/unit/test_opensearch_client.py b/backend/cosmetology-app/lambdas/python/search/tests/unit/test_opensearch_client.py index 62e5eb630..96cfe7730 100644 --- a/backend/cosmetology-app/lambdas/python/search/tests/unit/test_opensearch_client.py +++ b/backend/cosmetology-app/lambdas/python/search/tests/unit/test_opensearch_client.py @@ -496,3 +496,82 @@ def test_cluster_health_raises_after_max_retries(self, mock_sleep): # Verify health was called MAX_RETRY_ATTEMPTS times self.assertEqual(MAX_RETRY_ATTEMPTS, mock_internal_client.cluster.health.call_count) self.assertIn('cluster_health', str(context.exception)) + + +class TestOpenSearchClientDeleteProviderDocuments(TestCase): + """Test suite for OpenSearchClient.delete_provider_documents().""" + + def _create_client_with_mock(self): + """Create an OpenSearchClient with a mocked internal client.""" + with ( + patch('opensearch_client.boto3'), + patch('opensearch_client.config'), + patch('opensearch_client.OpenSearch') as mock_opensearch_class, + ): + mock_internal_client = MagicMock() + mock_opensearch_class.return_value = mock_internal_client + + from opensearch_client import OpenSearchClient + + client = OpenSearchClient() + return client, mock_internal_client + + def test_delete_provider_documents_calls_internal_client_with_expected_arguments(self): + """Test that delete_provider_documents builds the provider query and calls the internal client.""" + client, mock_internal_client = self._create_client_with_mock() + + index_name = 'compact_cosm_providers' + provider_id = 'provider-1' + expected_response = {'deleted': 3, 'failures': []} + mock_internal_client.delete_by_query.return_value = expected_response + + result = client.delete_provider_documents(index_name=index_name, provider_id=provider_id) + + mock_internal_client.delete_by_query.assert_called_once_with( + index=index_name, + body={'query': {'term': {'providerId': provider_id}}}, + ) + self.assertEqual(expected_response, result) + + @patch('opensearch_client.time.sleep') + def test_delete_provider_documents_retries_on_connection_timeout(self, mock_sleep): + """Test that delete_provider_documents retries on ConnectionTimeout.""" + from opensearch_client import INITIAL_BACKOFF_SECONDS + + client, mock_internal_client = self._create_client_with_mock() + + expected_response = {'deleted': 1, 'failures': []} + mock_internal_client.delete_by_query.side_effect = [ + ConnectionTimeout('Connection timed out', 503, 'some error'), + expected_response, + ] + + result = client.delete_provider_documents( + index_name='compact_cosm_providers', + provider_id='provider-1', + ) + + self.assertEqual(2, mock_internal_client.delete_by_query.call_count) + self.assertEqual(1, mock_sleep.call_count) + mock_sleep.assert_called_with(INITIAL_BACKOFF_SECONDS) + self.assertEqual(expected_response, result) + + @patch('opensearch_client.time.sleep') + def test_delete_provider_documents_raises_after_max_retries(self, mock_sleep): + """Test that delete_provider_documents raises CCInternalException after max retries.""" + from opensearch_client import MAX_RETRY_ATTEMPTS + + client, mock_internal_client = self._create_client_with_mock() + + mock_internal_client.delete_by_query.side_effect = ConnectionTimeout( + 'Connection timed out', 503, 'some error' + ) + + with self.assertRaises(CCInternalException) as context: + client.delete_provider_documents( + index_name='compact_cosm_providers', + provider_id='provider-1', + ) + + self.assertEqual(MAX_RETRY_ATTEMPTS, mock_internal_client.delete_by_query.call_count) + self.assertIn('delete_provider_documents', str(context.exception)) diff --git a/backend/cosmetology-app/lambdas/python/search/utils.py b/backend/cosmetology-app/lambdas/python/search/utils.py index 04fd50aa5..f6bdd639c 100644 --- a/backend/cosmetology-app/lambdas/python/search/utils.py +++ b/backend/cosmetology-app/lambdas/python/search/utils.py @@ -9,33 +9,46 @@ import json from cc_common.config import config -from cc_common.data_model.schema.provider.api import ProviderGeneralResponseSchema +from cc_common.data_model.schema.provider.api import ProviderOpenSearchDocumentSchema from cc_common.utils import ResponseEncoder -def generate_provider_opensearch_document(compact: str, provider_id: str) -> dict: +def generate_provider_opensearch_documents(compact: str, provider_id: str) -> list[dict]: """ - Process a single provider and return the sanitized document ready for indexing. + Process a single provider and return a list of sanitized documents ready for indexing. + + Each document corresponds to one license. This is because the Cosmetology compact search returns results by license, + so we need to index one document per license to support native pagination. + + Becuase of this, rather than just using the provider_id as the documentId, + we add a composite documentId that includes the jurisdiction and license type. + This composite documentId is added after sanitization so that bulk_index can use it as the OpenSearch _id. :param compact: The compact abbreviation :param provider_id: The provider ID to process - :return: Sanitized document ready for indexing + :return: List of sanitized documents, each with a composite documentId :raises CCNotFoundException: If the provider is not found :raises ValidationError: If the provider data fails schema validation """ - # Get complete provider records provider_user_records = config.data_client.get_provider_user_records( compact=compact, provider_id=provider_id, consistent_read=True, ) - # Generate API response object with all nested records - api_response = provider_user_records.generate_api_response_object() + raw_documents = provider_user_records.generate_opensearch_documents() + + schema = ProviderOpenSearchDocumentSchema() + result = [] + for raw_doc in raw_documents: + sanitized = schema.load(raw_doc) + serializable = json.loads(json.dumps(sanitized, cls=ResponseEncoder)) + + license_info = serializable['licenses'][0] + jurisdiction = license_info['jurisdiction'] + license_type = license_info['licenseType'] + serializable['documentId'] = f'{provider_id}#{jurisdiction}#{license_type}' - # Sanitize using ProviderGeneralResponseSchema - schema = ProviderGeneralResponseSchema() - sanitized_document = schema.load(api_response) + result.append(serializable) - # Serialize using ResponseEncoder to convert sets to lists and datetime objects to strings - return json.loads(json.dumps(sanitized_document, cls=ResponseEncoder)) + return result diff --git a/backend/cosmetology-app/pipeline/backend_stage.py b/backend/cosmetology-app/pipeline/backend_stage.py index 067ce66d3..982cd1941 100644 --- a/backend/cosmetology-app/pipeline/backend_stage.py +++ b/backend/cosmetology-app/pipeline/backend_stage.py @@ -114,6 +114,18 @@ def __init__( persistent_stack=self.persistent_stack, ) + # Search Persistent Stack - OpenSearch Domain (created before ApiStack for public search wiring) + self.search_persistent_stack = SearchPersistentStack( + self, + 'SearchPersistentStack', + env=environment, + environment_context=environment_context, + standard_tags=standard_tags, + environment_name=environment_name, + vpc_stack=self.vpc_stack, + persistent_stack=self.persistent_stack, + ) + self.api_stack = ApiStack( self, 'APIStack', @@ -123,6 +135,7 @@ def __init__( environment_name=environment_name, persistent_stack=self.persistent_stack, api_lambda_stack=self.api_lambda_stack, + search_persistent_stack=self.search_persistent_stack, ) self.state_api_stack = StateApiStack( @@ -191,18 +204,6 @@ def __init__( standard_tags=standard_tags, ) - # Search Persistent Stack - OpenSearch Domain for advanced provider search - self.search_persistent_stack = SearchPersistentStack( - self, - 'SearchPersistentStack', - env=environment, - environment_context=environment_context, - standard_tags=standard_tags, - environment_name=environment_name, - vpc_stack=self.vpc_stack, - persistent_stack=self.persistent_stack, - ) - self.search_api_stack = SearchApiStack( self, 'SearchAPIStack', diff --git a/backend/cosmetology-app/stacks/api_lambda_stack/public_lookup_api.py b/backend/cosmetology-app/stacks/api_lambda_stack/public_lookup_api.py index c0bad9ccb..ac94680d8 100644 --- a/backend/cosmetology-app/stacks/api_lambda_stack/public_lookup_api.py +++ b/backend/cosmetology-app/stacks/api_lambda_stack/public_lookup_api.py @@ -52,6 +52,11 @@ def __init__( ) api_lambda_stack.log_groups.append(self.query_providers_handler.log_group) + # Dummy export to avoid CDK deadly embrace: public query providers now uses + # SearchPersistentStack.public_handler; this lambda is no longer wired to the API. + # TODO: remove this export (and the lambda above) after the stack is deployed # noqa: FIX002 + stack.export_value(self.query_providers_handler.function_arn) + def _get_provider_handler( self, scope: Construct, diff --git a/backend/cosmetology-app/stacks/api_stack/__init__.py b/backend/cosmetology-app/stacks/api_stack/__init__.py index 9488e7064..b9d56c586 100644 --- a/backend/cosmetology-app/stacks/api_stack/__init__.py +++ b/backend/cosmetology-app/stacks/api_stack/__init__.py @@ -5,6 +5,7 @@ from constructs import Construct from stacks import persistent_stack as ps +from stacks import search_persistent_stack as sps from stacks.api_lambda_stack import ApiLambdaStack from .api import LicenseApi @@ -20,6 +21,7 @@ def __init__( environment_context: dict, persistent_stack: ps.PersistentStack, api_lambda_stack: ApiLambdaStack, + search_persistent_stack: sps.SearchPersistentStack, **kwargs, ): super().__init__( @@ -35,5 +37,6 @@ def __init__( security_profile=security_profile, persistent_stack=persistent_stack, api_lambda_stack=api_lambda_stack, + search_persistent_stack=search_persistent_stack, domain_name=self.api_domain_name, ) diff --git a/backend/cosmetology-app/stacks/api_stack/api.py b/backend/cosmetology-app/stacks/api_stack/api.py index 89c98fc03..92df96209 100644 --- a/backend/cosmetology-app/stacks/api_stack/api.py +++ b/backend/cosmetology-app/stacks/api_stack/api.py @@ -6,6 +6,7 @@ from common_constructs.cc_api import CCApi from stacks import persistent_stack as ps +from stacks import search_persistent_stack as sps from stacks.api_lambda_stack import ApiLambdaStack @@ -17,6 +18,7 @@ def __init__( *, persistent_stack: ps.PersistentStack, api_lambda_stack: ApiLambdaStack, + search_persistent_stack: sps.SearchPersistentStack, **kwargs, ): super().__init__( @@ -33,6 +35,7 @@ def __init__( self.root, persistent_stack=persistent_stack, api_lambda_stack=api_lambda_stack, + search_persistent_stack=search_persistent_stack, ) @cached_property diff --git a/backend/cosmetology-app/stacks/api_stack/v1_api/api.py b/backend/cosmetology-app/stacks/api_stack/v1_api/api.py index ccffb7b83..dfd68c188 100644 --- a/backend/cosmetology-app/stacks/api_stack/v1_api/api.py +++ b/backend/cosmetology-app/stacks/api_stack/v1_api/api.py @@ -4,6 +4,7 @@ from aws_cdk.aws_apigateway import AuthorizationType, IResource, MethodOptions from stacks import persistent_stack as ps +from stacks import search_persistent_stack as sps from stacks.api_lambda_stack import ApiLambdaStack from .api_model import ApiModel @@ -23,6 +24,7 @@ def __init__( root: IResource, persistent_stack: ps.PersistentStack, api_lambda_stack: ApiLambdaStack, + search_persistent_stack: sps.SearchPersistentStack, ): super().__init__() from stacks.api_stack.api import LicenseApi @@ -112,6 +114,7 @@ def __init__( resource=self.public_compacts_compact_providers_resource, api_model=self.api_model, api_lambda_stack=api_lambda_stack, + search_persistent_stack=search_persistent_stack, ) # /v1/compacts diff --git a/backend/cosmetology-app/stacks/api_stack/v1_api/api_model.py b/backend/cosmetology-app/stacks/api_stack/v1_api/api_model.py index d0660d535..ad21f61ea 100644 --- a/backend/cosmetology-app/stacks/api_stack/v1_api/api_model.py +++ b/backend/cosmetology-app/stacks/api_stack/v1_api/api_model.py @@ -110,6 +110,12 @@ def query_providers_request_model(self) -> Model: max_length=100, description='Filter for providers with a family name', ), + 'licenseNumber': JsonSchema( + type=JsonSchemaType.STRING, + min_length=1, + max_length=500, + description='Filter for licenses with a specific license number', + ), }, ), 'pagination': self._pagination_request_schema, @@ -1063,7 +1069,6 @@ def get_compact_configuration_response_model(self) -> Model: 'compactName', 'compactOperationsTeamEmails', 'compactAdverseActionsNotificationEmails', - 'compactSummaryReportNotificationEmails', 'licenseeRegistrationEnabled', 'configuredStates', ], @@ -1082,11 +1087,6 @@ def get_compact_configuration_response_model(self) -> Model: description='List of email addresses for adverse actions notifications', items=JsonSchema(type=JsonSchemaType.STRING, format='email'), ), - 'compactSummaryReportNotificationEmails': JsonSchema( - type=JsonSchemaType.ARRAY, - description='List of email addresses for summary report notifications', - items=JsonSchema(type=JsonSchemaType.STRING, format='email'), - ), 'licenseeRegistrationEnabled': JsonSchema( type=JsonSchemaType.BOOLEAN, description='Denotes whether licensee registration is enabled', @@ -1130,7 +1130,6 @@ def put_compact_request_model(self) -> Model: required=[ 'compactOperationsTeamEmails', 'compactAdverseActionsNotificationEmails', - 'compactSummaryReportNotificationEmails', 'licenseeRegistrationEnabled', 'configuredStates', ], @@ -1151,14 +1150,6 @@ def put_compact_request_model(self) -> Model: unique_items=True, items=JsonSchema(type=JsonSchemaType.STRING, format='email'), ), - 'compactSummaryReportNotificationEmails': JsonSchema( - type=JsonSchemaType.ARRAY, - description='List of email addresses for summary report notifications', - min_items=1, - max_items=10, - unique_items=True, - items=JsonSchema(type=JsonSchemaType.STRING, format='email'), - ), 'licenseeRegistrationEnabled': JsonSchema( type=JsonSchemaType.BOOLEAN, description='Denotes whether licensee registration is enabled', @@ -1207,7 +1198,6 @@ def get_jurisdiction_response_model(self) -> Model: 'postalAbbreviation', 'jurisdictionOperationsTeamEmails', 'jurisdictionAdverseActionsNotificationEmails', - 'jurisdictionSummaryReportNotificationEmails', 'licenseeRegistrationEnabled', ], properties={ @@ -1234,11 +1224,6 @@ def get_jurisdiction_response_model(self) -> Model: description='List of email addresses for adverse actions notifications', items=JsonSchema(type=JsonSchemaType.STRING, format='email'), ), - 'jurisdictionSummaryReportNotificationEmails': JsonSchema( - type=JsonSchemaType.ARRAY, - description='List of email addresses for summary report notifications', - items=JsonSchema(type=JsonSchemaType.STRING, format='email'), - ), 'licenseeRegistrationEnabled': JsonSchema( type=JsonSchemaType.BOOLEAN, description='Denotes whether licensee registration is enabled', @@ -1265,7 +1250,6 @@ def put_jurisdiction_request_model(self) -> Model: required=[ 'jurisdictionOperationsTeamEmails', 'jurisdictionAdverseActionsNotificationEmails', - 'jurisdictionSummaryReportNotificationEmails', 'licenseeRegistrationEnabled', ], properties={ @@ -1285,14 +1269,6 @@ def put_jurisdiction_request_model(self) -> Model: unique_items=True, items=JsonSchema(type=JsonSchemaType.STRING, format='email'), ), - 'jurisdictionSummaryReportNotificationEmails': JsonSchema( - type=JsonSchemaType.ARRAY, - description='List of email addresses for summary report notifications', - min_items=1, - max_items=10, - unique_items=True, - items=JsonSchema(type=JsonSchemaType.STRING, format='email'), - ), 'licenseeRegistrationEnabled': JsonSchema( type=JsonSchemaType.BOOLEAN, description='Denotes whether licensee registration is enabled', @@ -1341,7 +1317,7 @@ def public_query_providers_response_model(self) -> Model: 'providers': JsonSchema( type=JsonSchemaType.ARRAY, max_length=100, - items=self._public_providers_response_schema, + items=self._public_license_search_response_schema, ), 'pagination': self._pagination_response_schema, 'query': JsonSchema( @@ -1367,6 +1343,12 @@ def public_query_providers_response_model(self) -> Model: max_length=100, description='Filter for providers with a family name', ), + 'licenseNumber': JsonSchema( + type=JsonSchemaType.STRING, + min_length=1, + max_length=100, + description='Filter for licenses with a specific license number', + ), }, ), 'sorting': self._sorting_schema, @@ -1657,6 +1639,32 @@ def provider_registration_request_model(self) -> Model: ) return self.api._v1_provider_registration_request_model + @property + def _public_license_search_response_schema(self): + """Schema for public query providers response.""" + stack: AppStack = AppStack.of(self.api) + return JsonSchema( + type=JsonSchemaType.OBJECT, + required=[ + 'providerId', + 'givenName', + 'familyName', + 'licenseJurisdiction', + 'compact', + 'licenseNumber', + ], + properties={ + 'providerId': JsonSchema(type=JsonSchemaType.STRING, pattern=cc_api.UUID4_FORMAT), + 'givenName': JsonSchema(type=JsonSchemaType.STRING, min_length=1, max_length=100), + 'familyName': JsonSchema(type=JsonSchemaType.STRING, min_length=1, max_length=100), + 'licenseJurisdiction': JsonSchema( + type=JsonSchemaType.STRING, enum=stack.node.get_context('jurisdictions') + ), + 'compact': JsonSchema(type=JsonSchemaType.STRING, enum=stack.node.get_context('compacts')), + 'licenseNumber': JsonSchema(type=JsonSchemaType.STRING, min_length=1, max_length=100), + }, + ) + @property def _public_providers_response_schema(self): return JsonSchema( diff --git a/backend/cosmetology-app/stacks/api_stack/v1_api/public_lookup_api.py b/backend/cosmetology-app/stacks/api_stack/v1_api/public_lookup_api.py index dc335b2f4..2c13505d0 100644 --- a/backend/cosmetology-app/stacks/api_stack/v1_api/public_lookup_api.py +++ b/backend/cosmetology-app/stacks/api_stack/v1_api/public_lookup_api.py @@ -5,6 +5,7 @@ from cdk_nag import NagSuppressions from common_constructs.cc_api import CCApi +from stacks import search_persistent_stack as sps from stacks.api_lambda_stack import ApiLambdaStack from .api_model import ApiModel @@ -17,6 +18,7 @@ def __init__( resource: Resource, api_model: ApiModel, api_lambda_stack: ApiLambdaStack, + search_persistent_stack: sps.SearchPersistentStack, ): super().__init__() @@ -32,9 +34,7 @@ def __init__( 'licenseType' ).add_resource('{licenseType}') - self._add_public_query_providers( - api_lambda_stack=api_lambda_stack, - ) + self._add_public_query_providers(search_persistent_stack=search_persistent_stack) self._add_public_get_provider( api_lambda_stack=api_lambda_stack, ) @@ -73,13 +73,10 @@ def _add_public_get_provider( ], ) - def _add_public_query_providers( - self, - api_lambda_stack: ApiLambdaStack, - ): + def _add_public_query_providers(self, search_persistent_stack: sps.SearchPersistentStack): query_resource = self.resource.add_resource('query') - handler = api_lambda_stack.public_lookup_lambdas.query_providers_handler + handler = search_persistent_stack.search_handler.public_handler public_query_provider_method = query_resource.add_method( 'POST', diff --git a/backend/cosmetology-app/stacks/search_persistent_stack/search_handler.py b/backend/cosmetology-app/stacks/search_persistent_stack/search_handler.py index 798083649..5c8ac3538 100644 --- a/backend/cosmetology-app/stacks/search_persistent_stack/search_handler.py +++ b/backend/cosmetology-app/stacks/search_persistent_stack/search_handler.py @@ -76,6 +76,53 @@ def __init__( alarm_topic=alarm_topic, ) + # Create Lambda function for public query providers + self.public_handler = PythonFunction( + self, + 'PublicSearchProvidersFunction', + description='Public search handler for OpenSearch license queries', + index=os.path.join('handlers', 'public_search.py'), + lambda_dir='search', + handler='public_search_api_handler', + role=lambda_role, + log_retention=RetentionDays.ONE_MONTH, + environment={ + 'OPENSEARCH_HOST_ENDPOINT': opensearch_domain.domain_endpoint, + **stack.common_env_vars, + }, + timeout=Duration.seconds(29), + memory_size=2048, + vpc=vpc_stack.vpc, + vpc_subnets=vpc_subnets, + security_groups=[vpc_stack.lambda_security_group], + alarm_topic=alarm_topic, + ) + opensearch_domain.grant_read(self.public_handler) + + # Create metric filter and alarm for public handler errors + public_error_log_metric = MetricFilter( + self, + 'PublicSearchHandlerErrorLogMetric', + log_group=self.public_handler.log_group, + metric_namespace='CompactConnect/Search', + metric_name='PublicSearchHandlerErrors', + filter_pattern=FilterPattern.string_value(json_field='$.level', comparison='=', value='ERROR'), + metric_value='1', + default_value=0, + ) + public_error_log_alarm = Alarm( + self, + 'PublicSearchHandlerErrorLogAlarm', + metric=public_error_log_metric.metric(statistic='Sum'), + evaluation_periods=1, + threshold=1, + actions_enabled=True, + alarm_description='The Public Search Handler Lambda logged an ERROR level message.', + comparison_operator=ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=TreatMissingData.NOT_BREACHING, + ) + public_error_log_alarm.add_alarm_action(SnsAction(alarm_topic)) + # Grant the handler read access to the OpenSearch domain opensearch_domain.grant_read(self.handler) diff --git a/backend/cosmetology-app/tests/app/test_api/test_public_lookup_api.py b/backend/cosmetology-app/tests/app/test_api/test_public_lookup_api.py index 045f40ede..d19a17616 100644 --- a/backend/cosmetology-app/tests/app/test_api/test_public_lookup_api.py +++ b/backend/cosmetology-app/tests/app/test_api/test_public_lookup_api.py @@ -85,8 +85,8 @@ def test_synth_generates_public_query_providers_endpoint(self): """Test that the POST /providers/query endpoint is configured correctly.""" api_stack = self.app.sandbox_backend_stage.api_stack api_stack_template = Template.from_stack(api_stack) - api_lambda_stack = self.app.sandbox_backend_stage.api_lambda_stack - api_lambda_stack_template = Template.from_stack(api_lambda_stack) + search_persistent_stack = self.app.sandbox_backend_stage.search_persistent_stack + search_persistent_stack_template = Template.from_stack(search_persistent_stack) # Ensure the resource is created with expected path api_stack_template.has_resource_properties( @@ -102,13 +102,13 @@ def test_synth_generates_public_query_providers_endpoint(self): # Ensure the lambda is created with expected code path in the ApiLambdaStack query_handler = TestApi.get_resource_properties_by_logical_id( - api_lambda_stack.get_logical_id( - api_lambda_stack.public_lookup_lambdas.query_providers_handler.node.default_child + search_persistent_stack.get_logical_id( + search_persistent_stack.search_handler.public_handler.node.default_child ), - api_lambda_stack_template.find_resources(CfnFunction.CFN_RESOURCE_TYPE_NAME), + search_persistent_stack_template.find_resources(CfnFunction.CFN_RESOURCE_TYPE_NAME), ) - self.assertEqual(query_handler['Handler'], 'handlers.public_lookup.public_query_providers') + self.assertEqual(query_handler['Handler'], 'handlers.public_search.public_search_api_handler') # Capture model logical IDs for verification request_model_logical_id_capture = Capture() @@ -120,9 +120,9 @@ def test_synth_generates_public_query_providers_endpoint(self): props={ 'HttpMethod': 'POST', 'Integration': TestApi.generate_expected_integration_object_for_imported_lambda( - api_lambda_stack, - api_lambda_stack_template, - api_lambda_stack.public_lookup_lambdas.query_providers_handler, + search_persistent_stack, + search_persistent_stack_template, + search_persistent_stack.search_handler.public_handler, ), 'RequestModels': { 'application/json': {'Ref': request_model_logical_id_capture}, diff --git a/backend/cosmetology-app/tests/resources/snapshots/GET_COMPACT_CONFIGURATION_RESPONSE_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/GET_COMPACT_CONFIGURATION_RESPONSE_SCHEMA.json index f3bf8e0d6..091491832 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/GET_COMPACT_CONFIGURATION_RESPONSE_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/GET_COMPACT_CONFIGURATION_RESPONSE_SCHEMA.json @@ -24,14 +24,6 @@ }, "type": "array" }, - "compactSummaryReportNotificationEmails": { - "description": "List of email addresses for summary report notifications", - "items": { - "format": "email", - "type": "string" - }, - "type": "array" - }, "licenseeRegistrationEnabled": { "description": "Denotes whether licensee registration is enabled", "type": "boolean" @@ -75,7 +67,6 @@ "compactName", "compactOperationsTeamEmails", "compactAdverseActionsNotificationEmails", - "compactSummaryReportNotificationEmails", "licenseeRegistrationEnabled", "configuredStates" ], diff --git a/backend/cosmetology-app/tests/resources/snapshots/GET_JURISDICTION_CONFIGURATION_RESPONSE_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/GET_JURISDICTION_CONFIGURATION_RESPONSE_SCHEMA.json index 54897ae1f..1cb9a9447 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/GET_JURISDICTION_CONFIGURATION_RESPONSE_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/GET_JURISDICTION_CONFIGURATION_RESPONSE_SCHEMA.json @@ -31,14 +31,6 @@ }, "type": "array" }, - "jurisdictionSummaryReportNotificationEmails": { - "description": "List of email addresses for summary report notifications", - "items": { - "format": "email", - "type": "string" - }, - "type": "array" - }, "licenseeRegistrationEnabled": { "description": "Denotes whether licensee registration is enabled", "type": "boolean" @@ -50,7 +42,6 @@ "postalAbbreviation", "jurisdictionOperationsTeamEmails", "jurisdictionAdverseActionsNotificationEmails", - "jurisdictionSummaryReportNotificationEmails", "licenseeRegistrationEnabled" ], "type": "object", diff --git a/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_REQUEST_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_REQUEST_SCHEMA.json index 1d3d3eeb7..0d12e375c 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_REQUEST_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_REQUEST_SCHEMA.json @@ -35,6 +35,12 @@ "description": "Filter for providers with a family name", "maxLength": 100, "type": "string" + }, + "licenseNumber": { + "description": "Filter for licenses with a specific license number", + "maxLength": 500, + "minLength": 1, + "type": "string" } }, "type": "object" diff --git a/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_RESPONSE_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_RESPONSE_SCHEMA.json index 380303be6..2c62fa41b 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_RESPONSE_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/PUBLIC_QUERY_PROVIDERS_RESPONSE_SCHEMA.json @@ -3,12 +3,6 @@ "providers": { "items": { "properties": { - "type": { - "enum": [ - "provider" - ], - "type": "string" - }, "providerId": { "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab]{1}[0-9a-f]{3}-[0-9a-f]{12}", "type": "string" @@ -18,27 +12,11 @@ "minLength": 1, "type": "string" }, - "middleName": { - "maxLength": 100, - "minLength": 1, - "type": "string" - }, "familyName": { "maxLength": 100, "minLength": 1, "type": "string" }, - "suffix": { - "maxLength": 100, - "minLength": 1, - "type": "string" - }, - "compact": { - "enum": [ - "cosm" - ], - "type": "string" - }, "licenseJurisdiction": { "enum": [ "al", @@ -54,18 +32,25 @@ ], "type": "string" }, - "dateOfUpdate": { - "format": "date-time", + "compact": { + "enum": [ + "cosm" + ], + "type": "string" + }, + "licenseNumber": { + "maxLength": 100, + "minLength": 1, "type": "string" } }, "required": [ - "type", "providerId", "givenName", "familyName", + "licenseJurisdiction", "compact", - "licenseJurisdiction" + "licenseNumber" ], "type": "object" }, @@ -130,6 +115,12 @@ "description": "Filter for providers with a family name", "maxLength": 100, "type": "string" + }, + "licenseNumber": { + "description": "Filter for licenses with a specific license number", + "maxLength": 100, + "minLength": 1, + "type": "string" } }, "type": "object" diff --git a/backend/cosmetology-app/tests/resources/snapshots/PUT_COMPACT_CONFIGURATION_REQUEST_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/PUT_COMPACT_CONFIGURATION_REQUEST_SCHEMA.json index 0e4657936..bcfe3e22d 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/PUT_COMPACT_CONFIGURATION_REQUEST_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/PUT_COMPACT_CONFIGURATION_REQUEST_SCHEMA.json @@ -23,17 +23,6 @@ "type": "array", "uniqueItems": true }, - "compactSummaryReportNotificationEmails": { - "description": "List of email addresses for summary report notifications", - "items": { - "format": "email", - "type": "string" - }, - "maxItems": 10, - "minItems": 1, - "type": "array", - "uniqueItems": true - }, "licenseeRegistrationEnabled": { "description": "Denotes whether licensee registration is enabled", "type": "boolean" @@ -76,7 +65,6 @@ "required": [ "compactOperationsTeamEmails", "compactAdverseActionsNotificationEmails", - "compactSummaryReportNotificationEmails", "licenseeRegistrationEnabled", "configuredStates" ], diff --git a/backend/cosmetology-app/tests/resources/snapshots/PUT_JURISDICTION_CONFIGURATION_REQUEST_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/PUT_JURISDICTION_CONFIGURATION_REQUEST_SCHEMA.json index cf7e87b39..7996e3d85 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/PUT_JURISDICTION_CONFIGURATION_REQUEST_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/PUT_JURISDICTION_CONFIGURATION_REQUEST_SCHEMA.json @@ -23,17 +23,6 @@ "type": "array", "uniqueItems": true }, - "jurisdictionSummaryReportNotificationEmails": { - "description": "List of email addresses for summary report notifications", - "items": { - "format": "email", - "type": "string" - }, - "maxItems": 10, - "minItems": 1, - "type": "array", - "uniqueItems": true - }, "licenseeRegistrationEnabled": { "description": "Denotes whether licensee registration is enabled", "type": "boolean" @@ -42,7 +31,6 @@ "required": [ "jurisdictionOperationsTeamEmails", "jurisdictionAdverseActionsNotificationEmails", - "jurisdictionSummaryReportNotificationEmails", "licenseeRegistrationEnabled" ], "type": "object", diff --git a/backend/cosmetology-app/tests/resources/snapshots/QUERY_PROVIDERS_REQUEST_SCHEMA.json b/backend/cosmetology-app/tests/resources/snapshots/QUERY_PROVIDERS_REQUEST_SCHEMA.json index 1d3d3eeb7..0d12e375c 100644 --- a/backend/cosmetology-app/tests/resources/snapshots/QUERY_PROVIDERS_REQUEST_SCHEMA.json +++ b/backend/cosmetology-app/tests/resources/snapshots/QUERY_PROVIDERS_REQUEST_SCHEMA.json @@ -35,6 +35,12 @@ "description": "Filter for providers with a family name", "maxLength": 100, "type": "string" + }, + "licenseNumber": { + "description": "Filter for licenses with a specific license number", + "maxLength": 500, + "minLength": 1, + "type": "string" } }, "type": "object" diff --git a/backend/cosmetology-app/tests/smoke/compact_configuration_smoke_tests.py b/backend/cosmetology-app/tests/smoke/compact_configuration_smoke_tests.py index d0b2ccf23..194ec810a 100644 --- a/backend/cosmetology-app/tests/smoke/compact_configuration_smoke_tests.py +++ b/backend/cosmetology-app/tests/smoke/compact_configuration_smoke_tests.py @@ -139,7 +139,6 @@ def test_compact_configuration(): 'licenseeRegistrationEnabled': False, 'compactOperationsTeamEmails': [notification_email], 'compactAdverseActionsNotificationEmails': [notification_email], - 'compactSummaryReportNotificationEmails': [notification_email], 'configuredStates': [], } @@ -259,7 +258,6 @@ def test_jurisdiction_configuration(jurisdiction: str = 'ne', recreate_compact_c jurisdiction_config = { 'jurisdictionOperationsTeamEmails': [notification_email], 'jurisdictionAdverseActionsNotificationEmails': [notification_email], - 'jurisdictionSummaryReportNotificationEmails': [notification_email], 'licenseeRegistrationEnabled': True, } diff --git a/backend/cosmetology-app/tests/smoke/rollback_license_upload_smoke_tests.py b/backend/cosmetology-app/tests/smoke/rollback_license_upload_smoke_tests.py index 266a82afc..2c92f4889 100644 --- a/backend/cosmetology-app/tests/smoke/rollback_license_upload_smoke_tests.py +++ b/backend/cosmetology-app/tests/smoke/rollback_license_upload_smoke_tests.py @@ -370,7 +370,6 @@ def create_privilege_for_provider(provider_id: str, compact: str): 'privilegeId': f'{license_type_abbr.upper()}-{privilege_jurisdiction.upper()}-12345', 'administratorSetStatus': 'active', 'compactTransactionId': 'test-transaction-12345', - 'compactTransactionIdGSIPK': f'COMPACT#{compact}#TX#test-transaction-12345#', } config.provider_user_dynamodb_table.put_item(Item=privilege_record)