diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..56b8755a8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +*.zip binary +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary diff --git a/artifacts/APP1/api_matrix.json b/artifacts/APP1/api_matrix.json new file mode 100644 index 000000000..d227bb8a5 --- /dev/null +++ b/artifacts/APP1/api_matrix.json @@ -0,0 +1,9 @@ +{ + "app": "APP1", + "routes": [ + {"path": "/api/quote", "methods": ["GET", "POST"], "roles": {"broker": "allow", "underwriter": "allow", "auditor": "read", "anonymous": "deny"}}, + {"path": "/api/claim", "methods": ["POST"], "roles": {"broker": "allow", "auditor": "deny", "customer": "allow"}}, + {"path": "/api/admin/quote/{quoteId}/approve", "methods": ["POST"], "roles": {"underwriter": "allow", "broker": "deny", "auditor": "read"}} + ], + "generated_at": "2025-10-28T13:10:00Z" +} diff --git a/artifacts/APP1/chaos_report.json b/artifacts/APP1/chaos_report.json new file mode 100644 index 000000000..38585aef4 --- /dev/null +++ b/artifacts/APP1/chaos_report.json @@ -0,0 +1,11 @@ +{ + "app": "APP1", + "experiments": [ + {"name": "pod_kill", "status": "pass", "error_rate": 0.006, "p95_ms": 410, "rollback": "scale restored"}, + {"name": "az_failure", "status": "pass", "failover_duration_s": 96, "p95_ms": 480}, + {"name": "network_partition", "status": "pass", "breaker_open_s": 9, "customer_impact": "retry-after 3s"}, + {"name": "disk_full", "status": "pass", "eviction_time_s": 70}, + {"name": "broker_failover", "status": "warn", "consumer_lag": 122, "notes": "lag exceeded SLO 100 messages"} + ], + "generated_at": "2025-10-28T13:20:00Z" +} diff --git a/artifacts/APP1/components.json b/artifacts/APP1/components.json new file mode 100644 index 000000000..3783cb10b --- /dev/null +++ b/artifacts/APP1/components.json @@ -0,0 +1,9 @@ +{ + "app": "APP1", + "components": [ + {"name": "frontend", "language": "React", "version": "18.2.0", "owner": "frontend-team"}, + {"name": "pricing-service", "language": "Node.js", "version": "20.10.0", "owner": "pricing-team"}, + {"name": "claims-processor", "language": "Python", "version": "3.11", "owner": "claims-team"}, + {"name": "terraform", "language": "HCL", "version": "1.7.5", "owner": "platform-team"} + ] +} diff --git a/artifacts/APP1/decisions.json b/artifacts/APP1/decisions.json new file mode 100644 index 000000000..d9e05a25d --- /dev/null +++ b/artifacts/APP1/decisions.json @@ -0,0 +1,10 @@ +{ + "app": "APP1", + "run_id": "7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11", + "decisions": [ + {"stage": "design-review", "status": "pass", "evidence": "inputs/APP1/design.csv"}, + {"stage": "supply-chain", "status": "fail", "reason": "Unsigned pricing image", "remediation_pr": "See reports/APP1_vc_summary.md#remediation"}, + {"stage": "runtime-controls", "status": "warn", "reason": "Kafka consumer lag"}, + {"stage": "release", "status": "blocked", "reason": "Policy gate: RDS public"} + ] +} diff --git a/artifacts/APP1/e2e_junit.xml b/artifacts/APP1/e2e_junit.xml new file mode 100644 index 000000000..76c5e18b4 --- /dev/null +++ b/artifacts/APP1/e2e_junit.xml @@ -0,0 +1,20 @@ + + + + + + + Policy enforcement missing on staging cluster + + + + + Observed p95=520ms vs SLO 500ms + + + + + + + + diff --git a/artifacts/APP1/k6_summary.json b/artifacts/APP1/k6_summary.json new file mode 100644 index 000000000..370cf7956 --- /dev/null +++ b/artifacts/APP1/k6_summary.json @@ -0,0 +1,8 @@ +{ + "scenario": "baseline", + "vus_max": 200, + "requests": 18000, + "http_req_duration": {"avg": 320, "p95": 520, "max": 890}, + "http_req_failed": 0.012, + "notes": "Exceeded p95 target during AZ failover replay; flagged for remediation" +} diff --git a/artifacts/APP1/metrics.json b/artifacts/APP1/metrics.json new file mode 100644 index 000000000..6989fb2ac --- /dev/null +++ b/artifacts/APP1/metrics.json @@ -0,0 +1,15 @@ +{ + "app": "APP1", + "run_id": "7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11", + "metrics": { + "critical_findings": 2, + "high_findings": 3, + "medium_findings": 5, + "policy_blocks": 2, + "tests_passed": 20, + "tests_failed": 2, + "tests_skipped": 1, + "mean_time_to_detect": "2m" + }, + "generated_at": "2025-10-28T13:22:00Z" +} diff --git a/artifacts/APP1/policy_results.json b/artifacts/APP1/policy_results.json new file mode 100644 index 000000000..c53cc3a84 --- /dev/null +++ b/artifacts/APP1/policy_results.json @@ -0,0 +1,12 @@ +{ + "app": "APP1", + "run_id": "7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11", + "results": [ + {"rule": "Database instances must not be publicly accessible.", "status": "fail", "resource": "aws_db_instance.customers"}, + {"rule": "Database must be encrypted at rest.", "status": "fail", "resource": "aws_db_instance.customers"}, + {"rule": "Ingress rules cannot expose 0.0.0.0/0.", "status": "pass", "resource": "aws_security_group.api"}, + {"rule": "Load balancers must enforce TLS 1.2+.", "status": "pass", "resource": "aws_lb_listener.app"}, + {"rule": "Base image older than 180 days", "status": "warn", "resource": "pricing-base:2024-03-01"} + ], + "evaluated_at": "2025-10-28T13:05:00Z" +} diff --git a/artifacts/APP1/run_manifest.json b/artifacts/APP1/run_manifest.json new file mode 100644 index 000000000..1f298f1ca --- /dev/null +++ b/artifacts/APP1/run_manifest.json @@ -0,0 +1,27 @@ +{ + "app": "APP1", + "run_id": "7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11", + "mode": "simulated-offline", + "host": "https://fixops.local", + "inputs": { + "design": "inputs/APP1/design.csv", + "sbom": "inputs/APP1/sbom.json", + "sarif": "inputs/APP1/results.sarif", + "cve_feed": "inputs/APP1/cve_feed.json", + "vex": "inputs/APP1/vex_doc.json", + "cnapp": "inputs/APP1/findings.json" + }, + "tests": { + "contract": "tests/APP1/contract_tests/openapi.yaml", + "authz": "tests/APP1/authz_tests/matrix.csv", + "idempotency": [ + "tests/APP1/idempotency_tests/quote_idempotency.yaml", + "tests/APP1/idempotency_tests/audit_replay.yaml" + ], + "performance": "tests/APP1/perf_k6.js", + "chaos": "tests/APP1/chaos_playbooks" + }, + "policy": ["policy/APP1/security_controls.rego"], + "backtests": ["2024-08-pricing-outage", "2023-credential-stuffing"], + "created_at": "2025-10-28T12:45:00Z" +} diff --git a/artifacts/APP1/tf_plan.json b/artifacts/APP1/tf_plan.json new file mode 100644 index 000000000..930b8c360 --- /dev/null +++ b/artifacts/APP1/tf_plan.json @@ -0,0 +1,25 @@ +{ + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_db_instance.customers", + "type": "aws_db_instance", + "change": { + "actions": ["update"], + "before": {"storage_encrypted": false, "publicly_accessible": true}, + "after": {"storage_encrypted": true, "publicly_accessible": false} + } + }, + { + "address": "aws_security_group.api", + "type": "aws_security_group", + "change": { + "actions": ["no-op"], + "after": {"ingress": [{"cidr_blocks": ["10.0.0.0/16"], "from_port": 443, "to_port": 443}]} + } + } + ] + } + } +} diff --git a/artifacts/APP2/api_matrix.json b/artifacts/APP2/api_matrix.json new file mode 100644 index 000000000..ce9ed257c --- /dev/null +++ b/artifacts/APP2/api_matrix.json @@ -0,0 +1,8 @@ +{ + "app": "APP2", + "routes": [ + {"path": "/graphql", "methods": ["POST"], "roles": {"viewer": "allow", "integration": "allow", "anonymous": "deny"}}, + {"path": "/api/webhooks/{partner}", "methods": ["POST"], "roles": {"integration": "allow", "viewer": "deny", "ops": "allow"}} + ], + "generated_at": "2025-10-28T13:46:00Z" +} diff --git a/artifacts/APP2/chaos_report.json b/artifacts/APP2/chaos_report.json new file mode 100644 index 000000000..7ffe8b7e1 --- /dev/null +++ b/artifacts/APP2/chaos_report.json @@ -0,0 +1,11 @@ +{ + "app": "APP2", + "experiments": [ + {"name": "pod_kill", "status": "pass", "p95_ms": 380}, + {"name": "az_failure", "status": "pass", "error_rate": 0.024}, + {"name": "broker_failover", "status": "fail", "dlq_messages": 5}, + {"name": "network_partition", "status": "pass", "fallback_hit_rate": 0.92}, + {"name": "disk_full", "status": "pass"} + ], + "generated_at": "2025-10-28T13:50:00Z" +} diff --git a/artifacts/APP2/components.json b/artifacts/APP2/components.json new file mode 100644 index 000000000..49ab22fa1 --- /dev/null +++ b/artifacts/APP2/components.json @@ -0,0 +1,9 @@ +{ + "app": "APP2", + "components": [ + {"name": "shell", "framework": "Next.js", "owner": "experience-team"}, + {"name": "graphql-gateway", "language": "TypeScript", "owner": "api-team"}, + {"name": "partner-transformer", "language": "Python", "owner": "platform-team"}, + {"name": "kong-gateway", "language": "Lua", "owner": "platform-team"} + ] +} diff --git a/artifacts/APP2/decisions.json b/artifacts/APP2/decisions.json new file mode 100644 index 000000000..b34286d7d --- /dev/null +++ b/artifacts/APP2/decisions.json @@ -0,0 +1,10 @@ +{ + "app": "APP2", + "run_id": "91e1f59b-fef7-4637-9d36-0a7ef5a547ab", + "decisions": [ + {"stage": "design-review", "status": "pass"}, + {"stage": "partner-security", "status": "fail", "reason": "Webhook signature plugin disabled"}, + {"stage": "runtime-controls", "status": "fail", "reason": "DLQ > 0 during chaos"}, + {"stage": "release", "status": "blocked", "reason": "Partner secret not from Secrets Manager"} + ] +} diff --git a/artifacts/APP2/e2e_junit.xml b/artifacts/APP2/e2e_junit.xml new file mode 100644 index 000000000..e5eb124c2 --- /dev/null +++ b/artifacts/APP2/e2e_junit.xml @@ -0,0 +1,16 @@ + + + + Missing exponential backoff header + + + + + + Exceeded threshold for 2 minutes + + + 5 messages moved to DLQ + + + diff --git a/artifacts/APP2/k6_summary.json b/artifacts/APP2/k6_summary.json new file mode 100644 index 000000000..e1e06c6a6 --- /dev/null +++ b/artifacts/APP2/k6_summary.json @@ -0,0 +1,8 @@ +{ + "scenario": "spike", + "vus_max": 300, + "requests": 25000, + "http_req_duration": {"avg": 290, "p95": 460, "max": 730}, + "http_req_failed": 0.018, + "notes": "Backoff header missing for 429 responses" +} diff --git a/artifacts/APP2/metrics.json b/artifacts/APP2/metrics.json new file mode 100644 index 000000000..bc31f1852 --- /dev/null +++ b/artifacts/APP2/metrics.json @@ -0,0 +1,14 @@ +{ + "app": "APP2", + "run_id": "91e1f59b-fef7-4637-9d36-0a7ef5a547ab", + "metrics": { + "critical_findings": 1, + "high_findings": 3, + "medium_findings": 4, + "policy_blocks": 2, + "tests_passed": 19, + "tests_failed": 3, + "mean_time_to_detect": "4m" + }, + "generated_at": "2025-10-28T13:51:00Z" +} diff --git a/artifacts/APP2/policy_results.json b/artifacts/APP2/policy_results.json new file mode 100644 index 000000000..c77a28271 --- /dev/null +++ b/artifacts/APP2/policy_results.json @@ -0,0 +1,12 @@ +{ + "app": "APP2", + "run_id": "91e1f59b-fef7-4637-9d36-0a7ef5a547ab", + "results": [ + {"rule": "Webhook route must enforce HMAC signature plugin", "status": "fail", "resource": "kong/route/webhooks"}, + {"rule": "Service must use https", "status": "pass", "resource": "kong/service/graphql"}, + {"rule": "CDN origins cannot be public buckets", "status": "pass", "resource": "cloudfront/app2"}, + {"rule": "Image older than 180 days", "status": "warn", "resource": "partner-base:2024-02"}, + {"rule": "Partner secrets from secrets manager", "status": "fail", "resource": "lambda/transformer"} + ], + "evaluated_at": "2025-10-28T13:45:00Z" +} diff --git a/artifacts/APP2/run_manifest.json b/artifacts/APP2/run_manifest.json new file mode 100644 index 000000000..d58a81f12 --- /dev/null +++ b/artifacts/APP2/run_manifest.json @@ -0,0 +1,31 @@ +{ + "app": "APP2", + "run_id": "91e1f59b-fef7-4637-9d36-0a7ef5a547ab", + "mode": "simulated-offline", + "host": "https://fixops.local", + "inputs": { + "design": "inputs/APP2/design.csv", + "sbom": "inputs/APP2/sbom.json", + "sarif": "inputs/APP2/results.sarif", + "cve_feed": "inputs/APP2/cve_feed.json", + "vex": "inputs/APP2/vex_doc.json", + "cnapp": "inputs/APP2/findings.json" + }, + "tests": { + "contract": [ + "tests/APP2/contract_tests/openapi.yaml", + "tests/APP2/contract_tests/partner_feed.asyncapi.yaml" + ], + "authz": "tests/APP2/authz_tests/matrix.csv", + "idempotency": [ + "tests/APP2/idempotency_tests/session_replay.yaml", + "tests/APP2/idempotency_tests/deletion_ack.yaml" + ], + "performance": "tests/APP2/perf_k6.js", + "chaos": "tests/APP2/chaos_playbooks" + }, + "policy": ["policy/APP2/security_controls.rego"], + "partner_simulators": "tests/APP2/partner_simulators", + "backtests": ["2024-partner-429", "2023-webhook-leak"], + "created_at": "2025-10-28T13:40:00Z" +} diff --git a/artifacts/APP2/tf_plan.json b/artifacts/APP2/tf_plan.json new file mode 100644 index 000000000..ec28f1aae --- /dev/null +++ b/artifacts/APP2/tf_plan.json @@ -0,0 +1,26 @@ +{ + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_cloudfront_distribution.partnerhub", + "type": "aws_cloudfront_distribution", + "change": { + "actions": ["update"], + "before": {"origin_groups": []}, + "after": {"origin_groups": [{"primary_origin_id": "origin-a", "failover_criteria": ["5xx"]}]} + } + }, + { + "address": "aws_lambda_function.transformer", + "type": "aws_lambda_function", + "change": { + "actions": ["update"], + "before": {"environment": {"variables": {"PARTNER_SECRET": "plaintext"}}}, + "after": {"environment": {"variables": {"PARTNER_SECRET": "{{secretsmanager:partner/secret}}"}}} + } + } + ] + } + } +} diff --git a/artifacts/APP3/api_matrix.json b/artifacts/APP3/api_matrix.json new file mode 100644 index 000000000..5542ab9da --- /dev/null +++ b/artifacts/APP3/api_matrix.json @@ -0,0 +1,8 @@ +{ + "app": "APP3", + "routes": [ + {"path": "/fhir/Patient", "methods": ["GET", "POST"], "roles": {"clinician": "allow", "patient": "deny", "auditor": "read"}}, + {"path": "/admin/audit/sign", "methods": ["POST"], "roles": {"auditor": "allow", "clinician": "deny"}} + ], + "generated_at": "2025-10-28T14:13:00Z" +} diff --git a/artifacts/APP3/chaos_report.json b/artifacts/APP3/chaos_report.json new file mode 100644 index 000000000..df06cd5da --- /dev/null +++ b/artifacts/APP3/chaos_report.json @@ -0,0 +1,11 @@ +{ + "app": "APP3", + "experiments": [ + {"name": "pod_kill", "status": "pass", "p95_ms": 420}, + {"name": "az_failure", "status": "pass", "failover_s": 108}, + {"name": "broker_failover", "status": "pass", "lag": 140}, + {"name": "network_partition", "status": "pass", "fallback_rate": 0.88}, + {"name": "disk_full", "status": "warn", "throttle_minutes": 7} + ], + "generated_at": "2025-10-28T14:15:00Z" +} diff --git a/artifacts/APP3/components.json b/artifacts/APP3/components.json new file mode 100644 index 000000000..b9a28fc32 --- /dev/null +++ b/artifacts/APP3/components.json @@ -0,0 +1,9 @@ +{ + "app": "APP3", + "components": [ + {"name": "patient-portal", "framework": "Angular", "owner": "experience-team"}, + {"name": "fhir-gateway", "language": "Java", "owner": "clinical-platform"}, + {"name": "ml-triage", "language": "Python", "owner": "ml-team"}, + {"name": "audit-ledger", "language": "Python", "owner": "security-team"} + ] +} diff --git a/artifacts/APP3/decisions.json b/artifacts/APP3/decisions.json new file mode 100644 index 000000000..3fe604f44 --- /dev/null +++ b/artifacts/APP3/decisions.json @@ -0,0 +1,10 @@ +{ + "app": "APP3", + "run_id": "f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d", + "decisions": [ + {"stage": "design-review", "status": "pass"}, + {"stage": "supply-chain", "status": "fail", "reason": "Spring Boot RCE (CVE-2024-34145)"}, + {"stage": "runtime-controls", "status": "warn", "reason": "Cosmos throttling 7m"}, + {"stage": "release", "status": "blocked", "reason": "Public admin ingress"} + ] +} diff --git a/artifacts/APP3/e2e_junit.xml b/artifacts/APP3/e2e_junit.xml new file mode 100644 index 000000000..efcfd5049 --- /dev/null +++ b/artifacts/APP3/e2e_junit.xml @@ -0,0 +1,18 @@ + + + + + + + + Exceeded 480ms target during surge + + + + + Cosmos autoscale capped for 7 minutes + + + + + diff --git a/artifacts/APP3/k6_summary.json b/artifacts/APP3/k6_summary.json new file mode 100644 index 000000000..92f8817cd --- /dev/null +++ b/artifacts/APP3/k6_summary.json @@ -0,0 +1,8 @@ +{ + "scenario": "surge", + "vus_max": 200, + "requests": 32000, + "http_req_duration": {"avg": 310, "p95": 490, "max": 780}, + "http_req_failed": 0.009, + "notes": "Slight SLO breach under surge, plan caching improvements" +} diff --git a/artifacts/APP3/metrics.json b/artifacts/APP3/metrics.json new file mode 100644 index 000000000..eae7e1f85 --- /dev/null +++ b/artifacts/APP3/metrics.json @@ -0,0 +1,15 @@ +{ + "app": "APP3", + "run_id": "f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d", + "metrics": { + "critical_findings": 2, + "high_findings": 2, + "medium_findings": 3, + "policy_blocks": 2, + "tests_passed": 22, + "tests_failed": 2, + "tests_skipped": 1, + "mean_time_to_detect": "3m" + }, + "generated_at": "2025-10-28T14:16:00Z" +} diff --git a/artifacts/APP3/policy_results.json b/artifacts/APP3/policy_results.json new file mode 100644 index 000000000..4fd27f1e6 --- /dev/null +++ b/artifacts/APP3/policy_results.json @@ -0,0 +1,12 @@ +{ + "app": "APP3", + "run_id": "f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d", + "results": [ + {"rule": "Admin ingress cannot be publicly accessible", "status": "fail", "resource": "ingress/admin"}, + {"rule": "Deployment must run as non-root", "status": "fail", "resource": "deployment/ehr-gateway"}, + {"rule": "Traffic manager endpoints must enforce HTTPS", "status": "pass", "resource": "traffic-manager/app3"}, + {"rule": "CosmosDB must use Strong consistency", "status": "warn", "resource": "cosmos/audit"}, + {"rule": "Base image older than 180 days", "status": "pass", "resource": "gateway-base:2025-07"} + ], + "evaluated_at": "2025-10-28T14:12:00Z" +} diff --git a/artifacts/APP3/run_manifest.json b/artifacts/APP3/run_manifest.json new file mode 100644 index 000000000..6e4f1d03f --- /dev/null +++ b/artifacts/APP3/run_manifest.json @@ -0,0 +1,28 @@ +{ + "app": "APP3", + "run_id": "f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d", + "mode": "simulated-offline", + "host": "https://fixops.local", + "inputs": { + "design": "inputs/APP3/design.csv", + "sbom": "inputs/APP3/sbom.json", + "sarif": "inputs/APP3/results.sarif", + "cve_feed": "inputs/APP3/cve_feed.json", + "vex": "inputs/APP3/vex_doc.json", + "cnapp": "inputs/APP3/findings.json" + }, + "tests": { + "contract": "tests/APP3/contract_tests/openapi.yaml", + "authz": "tests/APP3/authz_tests/matrix.csv", + "idempotency": [ + "tests/APP3/idempotency_tests/audit_append.yaml", + "tests/APP3/idempotency_tests/patient_create.yaml" + ], + "performance": "tests/APP3/perf_k6.js", + "chaos": "tests/APP3/chaos_playbooks" + }, + "policy": ["policy/APP3/security_controls.rego"], + "partner_simulators": "tests/APP3/partner_simulators", + "backtests": ["2025-fhir-wildcard", "2024-admin-ingress"], + "created_at": "2025-10-28T14:10:00Z" +} diff --git a/artifacts/APP3/tf_plan.json b/artifacts/APP3/tf_plan.json new file mode 100644 index 000000000..6a214101f --- /dev/null +++ b/artifacts/APP3/tf_plan.json @@ -0,0 +1,26 @@ +{ + "planned_values": { + "root_module": { + "resources": [ + { + "address": "azurerm_kubernetes_cluster.app3", + "type": "azurerm_kubernetes_cluster", + "change": { + "actions": ["update"], + "before": {"api_server_authorized_ip_ranges": ["0.0.0.0/0"]}, + "after": {"api_server_authorized_ip_ranges": ["10.20.0.0/16", "192.168.5.0/24"]} + } + }, + { + "address": "azurerm_linux_virtual_machine.kafka1", + "type": "azurerm_linux_virtual_machine", + "change": { + "actions": ["update"], + "before": {"admin_username": "root"}, + "after": {"admin_username": "app3svc"} + } + } + ] + } + } +} diff --git a/artifacts/APP4/api_matrix.json b/artifacts/APP4/api_matrix.json new file mode 100644 index 000000000..6844d010f --- /dev/null +++ b/artifacts/APP4/api_matrix.json @@ -0,0 +1,8 @@ +{ + "app": "APP4", + "routes": [ + {"path": "/checkout", "methods": ["POST"], "roles": {"store-device": "allow", "finance-ops": "deny"}}, + {"path": "/settlement/batch", "methods": ["POST"], "roles": {"finance-ops": "allow", "store-device": "deny"}} + ], + "generated_at": "2025-10-28T14:33:00Z" +} diff --git a/artifacts/APP4/chaos_report.json b/artifacts/APP4/chaos_report.json new file mode 100644 index 000000000..234d4d82f --- /dev/null +++ b/artifacts/APP4/chaos_report.json @@ -0,0 +1,11 @@ +{ + "app": "APP4", + "experiments": [ + {"name": "pod_kill", "status": "pass", "p95_ms": 380}, + {"name": "az_failure", "status": "pass", "error_rate": 0.019}, + {"name": "broker_failover", "status": "fail", "lag": 210}, + {"name": "network_partition", "status": "pass", "queue_drain_minutes": 4}, + {"name": "disk_full", "status": "pass"} + ], + "generated_at": "2025-10-28T14:35:00Z" +} diff --git a/artifacts/APP4/components.json b/artifacts/APP4/components.json new file mode 100644 index 000000000..b14c8f13f --- /dev/null +++ b/artifacts/APP4/components.json @@ -0,0 +1,9 @@ +{ + "app": "APP4", + "components": [ + {"name": "checkout-api", "language": "Go", "owner": "payments-team"}, + {"name": "settlement-batch", "language": "Python", "owner": "finance-team"}, + {"name": "pos-frontend", "language": "TypeScript", "owner": "device-team"}, + {"name": "edge-gateway", "language": "Rust", "owner": "platform-team"} + ] +} diff --git a/artifacts/APP4/decisions.json b/artifacts/APP4/decisions.json new file mode 100644 index 000000000..c6eb4a374 --- /dev/null +++ b/artifacts/APP4/decisions.json @@ -0,0 +1,10 @@ +{ + "app": "APP4", + "run_id": "6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b", + "decisions": [ + {"stage": "design-review", "status": "pass"}, + {"stage": "supply-chain", "status": "fail", "reason": "go-chi CVE and hardcoded HSM password"}, + {"stage": "runtime-controls", "status": "fail", "reason": "Broker failover produced lag 210"}, + {"stage": "release", "status": "blocked", "reason": "Policy violation: MQTT public"} + ] +} diff --git a/artifacts/APP4/e2e_junit.xml b/artifacts/APP4/e2e_junit.xml new file mode 100644 index 000000000..fe09a3da4 --- /dev/null +++ b/artifacts/APP4/e2e_junit.xml @@ -0,0 +1,16 @@ + + + + + + + Log scrubbing missing for settlement batch + + + Exceeded 420ms target + + + + Consumer lag exceeded SLO + + diff --git a/artifacts/APP4/k6_summary.json b/artifacts/APP4/k6_summary.json new file mode 100644 index 000000000..6b9332cc6 --- /dev/null +++ b/artifacts/APP4/k6_summary.json @@ -0,0 +1,8 @@ +{ + "scenario": "spike", + "vus_max": 400, + "requests": 28000, + "http_req_duration": {"avg": 260, "p95": 430, "max": 750}, + "http_req_failed": 0.015, + "notes": "Rate limiter tuning required" +} diff --git a/artifacts/APP4/metrics.json b/artifacts/APP4/metrics.json new file mode 100644 index 000000000..e2ce8317e --- /dev/null +++ b/artifacts/APP4/metrics.json @@ -0,0 +1,14 @@ +{ + "app": "APP4", + "run_id": "6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b", + "metrics": { + "critical_findings": 2, + "high_findings": 2, + "medium_findings": 2, + "policy_blocks": 2, + "tests_passed": 18, + "tests_failed": 3, + "mean_time_to_detect": "2m" + }, + "generated_at": "2025-10-28T14:36:00Z" +} diff --git a/artifacts/APP4/policy_results.json b/artifacts/APP4/policy_results.json new file mode 100644 index 000000000..efebd5769 --- /dev/null +++ b/artifacts/APP4/policy_results.json @@ -0,0 +1,12 @@ +{ + "app": "APP4", + "run_id": "6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b", + "results": [ + {"rule": "Lambda functions cannot store HSM credentials in environment variables", "status": "fail", "resource": "lambda/tokenizer"}, + {"rule": "MQTT listener cannot be public", "status": "fail", "resource": "sg/mqtt"}, + {"rule": "Checkout load balancers must use TLS", "status": "pass", "resource": "alb/checkout"}, + {"rule": "Settlement IAM must restrict KMS decrypt by VPC endpoint", "status": "warn", "resource": "iam/settlement-batch"}, + {"rule": "Base image older than 180 days", "status": "pass", "resource": "checkout-base:2025-05"} + ], + "evaluated_at": "2025-10-28T14:32:00Z" +} diff --git a/artifacts/APP4/run_manifest.json b/artifacts/APP4/run_manifest.json new file mode 100644 index 000000000..fb72e39ae --- /dev/null +++ b/artifacts/APP4/run_manifest.json @@ -0,0 +1,26 @@ +{ + "app": "APP4", + "run_id": "6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b", + "mode": "simulated-offline", + "host": "https://fixops.local", + "inputs": { + "design": "inputs/APP4/design.csv", + "sbom": "inputs/APP4/sbom.json", + "sarif": "inputs/APP4/results.sarif", + "cve_feed": "inputs/APP4/cve_feed.json", + "vex": "inputs/APP4/vex_doc.json", + "cnapp": "inputs/APP4/findings.json" + }, + "tests": { + "contract": "tests/APP4/contract_tests/openapi.yaml", + "authz": "tests/APP4/authz_tests/matrix.csv", + "idempotency": [ + "tests/APP4/idempotency_tests/checkout_idempotency.yaml", + "tests/APP4/idempotency_tests/audit_logs.yaml" + ], + "performance": "tests/APP4/perf_k6.js", + "chaos": "tests/APP4/chaos_playbooks" + }, + "policy": ["policy/APP4/security_controls.rego"], + "created_at": "2025-10-28T14:30:00Z" +} diff --git a/artifacts/APP4/tf_plan.json b/artifacts/APP4/tf_plan.json new file mode 100644 index 000000000..f868aef07 --- /dev/null +++ b/artifacts/APP4/tf_plan.json @@ -0,0 +1,25 @@ +{ + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_lambda_function.tokenizer", + "type": "aws_lambda_function", + "change": { + "actions": ["update"], + "before": {"environment": {"variables": {"HSM_PASSWORD": "plaintext"}}}, + "after": {"environment": {"variables": {"HSM_PASSWORD": "{{secretsmanager:/app4/hsm}}"}}} + } + }, + { + "address": "aws_security_group_rule.mqtt_public", + "type": "aws_security_group_rule", + "change": { + "actions": ["delete"], + "before": {"cidr_blocks": ["0.0.0.0/0"], "to_port": 8883} + } + } + ] + } + } +} diff --git a/artifacts/all_apps_reference.json b/artifacts/all_apps_reference.json new file mode 100644 index 000000000..600930508 --- /dev/null +++ b/artifacts/all_apps_reference.json @@ -0,0 +1,198 @@ +{ + "APP1": { + "run_id": "7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11", + "inputs": { + "design": "inputs/APP1/design.csv", + "sbom": "inputs/APP1/sbom.json", + "sarif": "inputs/APP1/results.sarif", + "cve_feed": "inputs/APP1/cve_feed.json", + "vex": "inputs/APP1/vex_doc.json", + "findings": "inputs/APP1/findings.json" + }, + "cli_commands": [ + { + "cmd": "fixops --list-commands --json", + "exit_code": 0 + }, + { + "cmd": "fixops api:enumerate --out artifacts/APP1_endpoints.json", + "exit_code": 0 + }, + { + "cmd": "fixops pipeline:run --module ssdlc --run-id 7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11", + "exit_code": 0 + } + ], + "api_calls": [ + { + "method": "POST", + "endpoint": "/inputs/design", + "status": 201 + }, + { + "method": "POST", + "endpoint": "/pipeline/run", + "status": 200 + } + ], + "artifacts": { + "evidence_bundle": { + "expected_path": "evidence/bundles/APP1_bundle.zip", + "generated_by": "cli/fixops-ci evidence bundle --tag APP1", + "tracked": false + }, + "policy_results": "artifacts/APP1/policy_results.json", + "decisions": "artifacts/APP1/decisions.json", + "metrics": "artifacts/APP1/metrics.json", + "components": "artifacts/APP1/components.json", + "remediation_pr": "artifacts/remediation_prs/APP1_remediation.md" + } + }, + "APP2": { + "run_id": "91e1f59b-fef7-4637-9d36-0a7ef5a547ab", + "inputs": { + "design": "inputs/APP2/design.csv", + "sbom": "inputs/APP2/sbom.json", + "sarif": "inputs/APP2/results.sarif", + "cve_feed": "inputs/APP2/cve_feed.json", + "vex": "inputs/APP2/vex_doc.json", + "findings": "inputs/APP2/findings.json" + }, + "cli_commands": [ + { + "cmd": "fixops --list-commands --json", + "exit_code": 0 + }, + { + "cmd": "fixops api:enumerate --app APP2 --out artifacts/APP2_endpoints.json", + "exit_code": 0 + }, + { + "cmd": "fixops pipeline:run --module ssdlc --run-id 91e1f59b-fef7-4637-9d36-0a7ef5a547ab", + "exit_code": 0 + } + ], + "api_calls": [ + { + "method": "POST", + "endpoint": "/inputs/design", + "status": 201 + }, + { + "method": "POST", + "endpoint": "/inputs/sbom", + "status": 201 + } + ], + "artifacts": { + "evidence_bundle": { + "expected_path": "evidence/bundles/APP2_bundle.zip", + "generated_by": "cli/fixops-ci evidence bundle --tag APP2", + "tracked": false + }, + "policy_results": "artifacts/APP2/policy_results.json", + "decisions": "artifacts/APP2/decisions.json", + "metrics": "artifacts/APP2/metrics.json", + "components": "artifacts/APP2/components.json", + "remediation_pr": "artifacts/remediation_prs/APP2_remediation.md" + } + }, + "APP3": { + "run_id": "f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d", + "inputs": { + "design": "inputs/APP3/design.csv", + "sbom": "inputs/APP3/sbom.json", + "sarif": "inputs/APP3/results.sarif", + "cve_feed": "inputs/APP3/cve_feed.json", + "vex": "inputs/APP3/vex_doc.json", + "findings": "inputs/APP3/findings.json" + }, + "cli_commands": [ + { + "cmd": "fixops --list-commands --json", + "exit_code": 0 + }, + { + "cmd": "fixops api:enumerate --app APP3 --out artifacts/APP3_endpoints.json", + "exit_code": 0 + }, + { + "cmd": "fixops pipeline:run --module ssdlc --run-id f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d", + "exit_code": 0 + } + ], + "api_calls": [ + { + "method": "POST", + "endpoint": "/inputs/design", + "status": 201 + }, + { + "method": "POST", + "endpoint": "/pipeline/run", + "status": 200 + } + ], + "artifacts": { + "evidence_bundle": { + "expected_path": "evidence/bundles/APP3_bundle.zip", + "generated_by": "cli/fixops-ci evidence bundle --tag APP3", + "tracked": false + }, + "policy_results": "artifacts/APP3/policy_results.json", + "decisions": "artifacts/APP3/decisions.json", + "metrics": "artifacts/APP3/metrics.json", + "components": "artifacts/APP3/components.json", + "remediation_pr": "artifacts/remediation_prs/APP3_remediation.md" + } + }, + "APP4": { + "run_id": "6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b", + "inputs": { + "design": "inputs/APP4/design.csv", + "sbom": "inputs/APP4/sbom.json", + "sarif": "inputs/APP4/results.sarif", + "cve_feed": "inputs/APP4/cve_feed.json", + "vex": "inputs/APP4/vex_doc.json", + "findings": "inputs/APP4/findings.json" + }, + "cli_commands": [ + { + "cmd": "fixops --list-commands --json", + "exit_code": 0 + }, + { + "cmd": "fixops api:enumerate --app APP4 --out artifacts/APP4_endpoints.json", + "exit_code": 0 + }, + { + "cmd": "fixops pipeline:run --module ssdlc --run-id 6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b", + "exit_code": 0 + } + ], + "api_calls": [ + { + "method": "POST", + "endpoint": "/inputs/design", + "status": 201 + }, + { + "method": "POST", + "endpoint": "/inputs/sbom", + "status": 201 + } + ], + "artifacts": { + "evidence_bundle": { + "expected_path": "evidence/bundles/APP4_bundle.zip", + "generated_by": "cli/fixops-ci evidence bundle --tag APP4", + "tracked": false + }, + "policy_results": "artifacts/APP4/policy_results.json", + "decisions": "artifacts/APP4/decisions.json", + "metrics": "artifacts/APP4/metrics.json", + "components": "artifacts/APP4/components.json", + "remediation_pr": "artifacts/remediation_prs/APP4_remediation.md" + } + } +} diff --git a/artifacts/remediation_prs/APP1_remediation.md b/artifacts/remediation_prs/APP1_remediation.md new file mode 100644 index 000000000..76a34abf0 --- /dev/null +++ b/artifacts/remediation_prs/APP1_remediation.md @@ -0,0 +1,49 @@ +# APP1 Remediation PR Bundle + +## Terraform: RDS Encryption + Private Access +```hcl +diff --git a/terraform/rds.tf b/terraform/rds.tf +@@ +-resource "aws_db_instance" "pricing" { +- publicly_accessible = true +- storage_encrypted = false +-} ++resource "aws_db_instance" "pricing" { ++ publicly_accessible = false ++ storage_encrypted = true ++ kms_key_id = aws_kms_key.rds_kms.arn ++ deletion_protection = true ++ backup_retention_period = 14 ++} +``` +- **Reason:** Release gate blocked on public, unencrypted RDS. Updating Terraform enforces encryption and removes internet access. +- **Verification:** `opa eval -d policy/APP1/security_controls.rego -i artifacts/APP1/tf_plan.json` now returns no deny messages. + +## Supply Chain: Signed Pricing Image +```diff +--- a/services/pricing/Dockerfile ++++ b/services/pricing/Dockerfile +@@ +-FROM node:20-alpine ++FROM ghcr.io/fixops/base-node:20.10-slsa3 ++LABEL org.opencontainers.image.source="https://github.com/fixops/pricing" ++RUN npm ci && npm run build ++RUN cosign sign --key env://SIGSTORE_KEY $IMAGE_DIGEST +``` +- **Reason:** Supply-chain stage failed because pricing image lacked signature. +- **Verification:** `cosign verify --key env://SIGSTORE_PUB ghcr.io/fixops/pricing:latest`. + +## Performance: Cache Warming +```diff +--- a/services/pricing/src/cache.ts ++++ b/services/pricing/src/cache.ts +@@ + export async function initCache(client: RedisClient) { +- await client.flushall(); ++ if (process.env.NODE_ENV === "production") { ++ await hydrateActuarialTables(client); ++ } + } +``` +- **Reason:** k6 run flagged p95=520ms. Hydrating cache before traffic keeps latency <350ms. +- **Verification:** Re-run `k6 run tests/APP1/perf_k6.js` and confirm new `artifacts/APP1/k6_summary.json` shows p95 < 350ms. diff --git a/artifacts/remediation_prs/APP2_remediation.md b/artifacts/remediation_prs/APP2_remediation.md new file mode 100644 index 000000000..7996162d3 --- /dev/null +++ b/artifacts/remediation_prs/APP2_remediation.md @@ -0,0 +1,56 @@ +# APP2 Remediation PR Bundle + +## Partner Security: Re-enable Webhook Signature Validation +```diff +--- a/services/partners/webhook_handler.py ++++ b/services/partners/webhook_handler.py +@@ +-def validate_signature(headers, payload): +- return True # temporarily bypassed for debugging ++def validate_signature(headers, payload): ++ signature = headers.get("X-Partner-Signature") ++ timestamp = headers.get("X-Partner-Timestamp") ++ if not signature or not timestamp: ++ raise Unauthorized("missing signature headers") ++ expected = hmac_sha256(SECRET_MANAGER.fetch("partner/webhook"), timestamp + payload) ++ if not hmac.compare_digest(signature, expected): ++ raise Unauthorized("signature mismatch") ++ return True +``` +- **Reason:** Partner-security stage failed because signature plugin was bypassed. +- **Verification:** Execute `tests/APP2/partner_simulators/invalid_signature.py` and confirm 401. + +## Runtime Controls: DLQ Recovery +```diff +--- a/infrastructure/terraform/kafka.tf ++++ b/infrastructure/terraform/kafka.tf +@@ + resource "kafka_topic" "session_events" { + name = "session-events" + partitions = 12 +- retention_ms = 86400000 ++ retention_ms = 604800000 ++ config = { ++ "min.insync.replicas" = "2" ++ } + } ++ ++module "dlq_consumer" { ++ source = "./modules/lambda_consumer" ++ topic = kafka_topic.session_events.name ++ target = aws_lambda_function.dlq_replay.arn ++} +``` +- **Reason:** Chaos tests left messages in DLQ; adding replay consumer clears backlog. +- **Verification:** Run `python tests/APP2/partner_simulators/server_error.py` followed by `python tests/APP2/partner_simulators/valid_signature.py`; confirm DLQ depth metric = 0. + +## Release Gate: Secrets Manager Source of Truth +```diff +--- a/config/default.yaml ++++ b/config/default.yaml +@@ +-partner_api_key: "test-test-test" ++partner_api_key: "${secretsmanager:partner/api/key}" +``` +- **Reason:** Release blocked because partner API key was hardcoded. +- **Verification:** `fixops secrets:lint --config config/default.yaml` passes; policy bundle no longer reports violation. diff --git a/artifacts/remediation_prs/APP3_remediation.md b/artifacts/remediation_prs/APP3_remediation.md new file mode 100644 index 000000000..14092825f --- /dev/null +++ b/artifacts/remediation_prs/APP3_remediation.md @@ -0,0 +1,58 @@ +# APP3 Remediation PR Bundle + +## Supply Chain: Patch Spring Boot CVE-2024-34145 +```diff +--- a/services/patient-api/build.gradle.kts ++++ b/services/patient-api/build.gradle.kts +@@ +-implementation("org.springframework.boot:spring-boot-starter-webflux:3.2.1") ++implementation("org.springframework.boot:spring-boot-starter-webflux:3.2.6") ++implementation("org.springframework.boot:spring-boot-starter-actuator:3.2.6") +``` +- **Reason:** SLSA attestation flagged vulnerable Spring Boot version. +- **Verification:** `./gradlew dependencyCheckAggregate` shows CVE remediated; `inputs/APP3/sbom.json` regenerated with version 3.2.6. + +## Release Gate: Lock Down Admin Ingress +```diff +--- a/infra/terraform/network.tf ++++ b/infra/terraform/network.tf +@@ +-resource "azurerm_application_gateway" "admin" { +- frontend_port = 443 +- frontend_ip_configuration { +- public_ip_address_id = azurerm_public_ip.admin.id +- } +-} ++resource "azurerm_application_gateway" "admin" { ++ frontend_port = 443 ++ frontend_ip_configuration { ++ subnet_id = azurerm_subnet.private_admin.id ++ } ++ ssl_certificate { ++ name = "admin-tls" ++ key_vault_secret_id = azurerm_key_vault_secret.admin_cert.id ++ } ++ firewall_policy_id = azurerm_web_application_firewall_policy.strict.id ++} +``` +- **Reason:** Release blocked because admin ingress exposed publicly. +- **Verification:** `opa eval -d policy/APP3/security_controls.rego -i artifacts/APP3/tf_plan.json` returns empty set. + +## Runtime Warning: Cosmos Throttling +```diff +--- a/services/telemetry/metrics.ts ++++ b/services/telemetry/metrics.ts +@@ +-export const COSMOS_RU_LIMIT = 40000; ++export const COSMOS_RU_LIMIT = 60000; +@@ +- if (usage > COSMOS_RU_LIMIT) { +- logger.warn("cosmos usage above threshold", { usage }); ++ if (usage > COSMOS_RU_LIMIT) { ++ circuitBreaker.open("cosmos_throttle"); ++ logger.warn("cosmos usage above threshold", { usage }); + } + } +``` +- **Reason:** Runtime controls emitted warning about throttling; raising limit and opening breaker protects downstream services. +- **Verification:** Replay soak test `k6 run tests/APP3/perf_k6.js` with `COSMOS_RU_LIMIT=60000`; ensure `artifacts/APP3/metrics.json` shows zero throttle events. diff --git a/artifacts/remediation_prs/APP4_remediation.md b/artifacts/remediation_prs/APP4_remediation.md new file mode 100644 index 000000000..a9ec381c7 --- /dev/null +++ b/artifacts/remediation_prs/APP4_remediation.md @@ -0,0 +1,83 @@ +# APP4 Remediation PR Bundle + +## Supply Chain: Harden Go Service Image +```diff +--- a/services/payments/Dockerfile ++++ b/services/payments/Dockerfile +@@ +-FROM golang:1.21-alpine +-RUN go build -o app ./cmd/payments ++FROM ghcr.io/fixops/base-go:1.21.5-slsa3 AS build ++RUN go build -trimpath -buildmode=pie -o /out/app ./cmd/payments ++ ++FROM gcr.io/distroless/base-debian12 ++COPY --from=build /out/app /usr/local/bin/payments ++ENTRYPOINT ["/usr/local/bin/payments"] +``` +- **Reason:** Supply-chain gate failed because base image outdated and binary unsigned. +- **Verification:** `cosign verify --key env://SIGSTORE_PUB gcr.io/fixops/payments:prod` and `grype` scan shows no go-chi CVE. + +## Runtime Controls: Broker Lag Mitigation +```diff +--- a/services/payments/stream/consumer.go ++++ b/services/payments/stream/consumer.go +@@ + func (c *Consumer) HandleBatch(ctx context.Context, msgs []Message) error { +- for _, msg := range msgs { +- process(msg) +- } ++ g, ctx := errgroup.WithContext(ctx) ++ sem := make(chan struct{}, 8) ++ for _, msg := range msgs { ++ sem <- struct{}{} ++ m := msg ++ g.Go(func() error { ++ defer func() { <-sem }() ++ return processWithRetry(ctx, m) ++ }) ++ } ++ if err := g.Wait(); err != nil { ++ return err ++ } + return nil + } +``` +- **Reason:** Chaos broker failover left lag 210; concurrency + retries clear backlog faster. +- **Verification:** Re-run `python tests/APP4/chaos_playbooks/broker_failover.md` scenario; confirm `artifacts/APP4/metrics.json` shows max lag < 30. + +## Release Gate: Secure MQTT Ingress +```diff +--- a/infra/terraform/mqtt.tf ++++ b/infra/terraform/mqtt.tf +@@ +-resource "aws_iot_endpoint" "public" {} ++resource "aws_iot_endpoint" "private" { ++ endpoint_type = "iot:Data-ATS" ++} + +-resource "aws_iot_policy" "all_devices" { +- policy = jsonencode({ +- Version = "2012-10-17" +- Statement = [{ +- Effect = "Allow" +- Action = "iot:*" +- Resource = "*" +- }] +- }) +-} ++resource "aws_iot_policy" "payments_devices" { ++ policy = jsonencode({ ++ Version = "2012-10-17" ++ Statement = [{ ++ Effect = "Allow" ++ Action = ["iot:Connect", "iot:Publish"] ++ Resource = "arn:aws:iot:*:*:topic/payments/*" ++ Condition = { ++ "Bool": {"iot:Connection.Thing.IsAttached": "true"} ++ } ++ }] ++ }) ++} +``` +- **Reason:** Release blocked: MQTT endpoint was public with wildcard policy. +- **Verification:** `opa eval -d policy/APP4/security_controls.rego -i artifacts/APP4/tf_plan.json` -> no denies; integration smoke test uses mutual TLS certificates from AWS IoT. diff --git a/artifacts/threat_matrices/APP1_threat_matrix.csv b/artifacts/threat_matrices/APP1_threat_matrix.csv new file mode 100644 index 000000000..2e0d7e863 --- /dev/null +++ b/artifacts/threat_matrices/APP1_threat_matrix.csv @@ -0,0 +1,9 @@ +threat_id,framework,category,description,attacker_objective,attack_vector,mitigation,test_reference +APP1-T1,STRIDE,spoofing,Forge customer identity to retrieve quotes,Steal competitor data,Compromised customer credentials,MFA on login + anomaly detection,tests/APP1/authz_tests/matrix.csv +APP1-T2,STRIDE,tampering,Modify pricing algorithm for lower premiums,Financial fraud,Supply chain attack on pricing service,Immutable builds + signed images,tests/APP1/chaos_playbooks/pipeline_supply_chain.md +APP1-T3,STRIDE,repudiation,Delete audit trail for quote approvals,Cover insider fraud,Privilege escalation in admin portal,Immutable audit logs + least privilege,tests/APP1/idempotency_tests/audit_replay.yaml +APP1-T4,STRIDE,information disclosure,Expose PHI from customer DB,Data exfiltration,Exposed LoadBalancer for postgres,Private networking + network policies,policy/APP1/deny_public_db.rego +APP1-T5,STRIDE,denial of service,Overwhelm quote API,Service disruption,API flood from botnet,Rate limiting + autoscaling,tests/APP1/perf_k6.js +APP1-P1,LINDDUN,linkability,Correlate quotes to individuals,Privacy breach,Re-identification via analytics dataset,Differential privacy + tokenization,tests/APP1/contract_tests/openapi.yaml +APP1-P2,LINDDUN,detectability,Detect claim history from response times,Competitive intelligence,Timing side-channel via risk score endpoint,Constant-time responses + caching,tests/APP1/perf_k6.js +APP1-P3,LINDDUN,non-compliance,Breach HIPAA logging requirements,Regulatory penalty,Improper access logging retention,Centralized audit pipeline + controls,artifacts/APP1/run_manifest.json diff --git a/artifacts/threat_matrices/APP1_threat_matrix.md b/artifacts/threat_matrices/APP1_threat_matrix.md new file mode 100644 index 000000000..9b6c7665e --- /dev/null +++ b/artifacts/threat_matrices/APP1_threat_matrix.md @@ -0,0 +1,24 @@ +# APP1 Insurance Platform - Threat & Attack Matrix + +| Threat ID | Framework | Category | Attacker Objective | Attack Vector | Mitigations | Validation | +|-----------|-----------|----------|--------------------|---------------|------------|-----------| +| APP1-T1 | STRIDE (Spoofing) | Identity Abuse | Steal competitor quotes by impersonating brokers | Credential stuffing via leaked broker emails | Enforce MFA, rate-limit login, anomaly detection | `tests/APP1/authz_tests/matrix.csv` | +| APP1-T2 | STRIDE (Tampering) | Code Integrity | Manipulate pricing model to offer fraudulent discounts | Supply chain compromise of pricing container image | Signed builds, admission controller, provenance attestation | `tests/APP1/chaos_playbooks/pipeline_supply_chain.md` | +| APP1-T3 | STRIDE (Repudiation) | Audit Evasion | Remove approval records to hide insider fraud | Abuse admin API to delete audit entries | Write-once audit store, approval workflow ledger | `tests/APP1/idempotency_tests/audit_replay.yaml` | +| APP1-T4 | STRIDE (Information Disclosure) | PHI Exposure | Exfiltrate policyholder SSNs | Public LoadBalancer on postgres service | Private subnets, service mesh mtls, network policies | `policy/APP1/deny_public_db.rego` | +| APP1-T5 | STRIDE (DoS) | Availability | Disrupt quote generation before renewal deadlines | Botnet API flooding | Global rate-limits, autoscaling, WAF challenge | `tests/APP1/perf_k6.js` | +| APP1-P1 | LINDDUN (Linkability) | Privacy | Link anonymized claims to individuals | Cross-correlation of analytics exports | Tokenization, aggregated reporting windows | `tests/APP1/contract_tests/openapi.yaml` | +| APP1-P2 | LINDDUN (Detectability) | Timing | Infer claim approval status by timing responses | Response time side-channels | Constant-time risk scoring, caching | `tests/APP1/perf_k6.js` | +| APP1-P3 | LINDDUN (Non-compliance) | Governance | Fail HIPAA retention obligations | Improper logging pipeline | Centralized logging, retention policy tests | `artifacts/APP1/run_manifest.json` | + +## Test Mapping + +- **Functional**: Contract tests cover quote, claim, billing flows with positive & negative cases. +- **Security**: AuthZ matrix covers broker, admin, auditor, and malicious roles. +- **Performance**: k6 scenarios for steady, spike, and failover loads with backtesting from Q2 incidents. +- **Chaos**: Pod kill, AZ failure, and DB failover validated with automated rollback scripts. + +## Backtesting Notes + +- Replayed the August 2024 outage (pricing pod restart loop) to confirm autoscaler mitigation. +- Simulated 2023 credential stuffing incident; MFA coverage prevented replay success. diff --git a/artifacts/threat_matrices/APP2_threat_matrix.csv b/artifacts/threat_matrices/APP2_threat_matrix.csv new file mode 100644 index 000000000..5412b4e10 --- /dev/null +++ b/artifacts/threat_matrices/APP2_threat_matrix.csv @@ -0,0 +1,8 @@ +threat_id,framework,category,description,attacker_objective,attack_vector,mitigation,test_reference +APP2-T1,STRIDE,spoofing,Partner impersonates webhook sender,Inject fraudulent bookings,Forged HMAC signature,Rotate secrets + verify timestamp,tests/APP2/partner_simulators/valid_signature.py +APP2-T2,STRIDE,tampering,Modify offer payload mid-flight,Manipulate pricing,MITM on CDN edge,Strict TLS pinning + integrity headers,tests/APP2/contract_tests/openapi.yaml +APP2-T3,STRIDE,information disclosure,Leak partner deals before embargo,Competitive advantage,Kong route exposing preview API,Access control policies + authz tests,tests/APP2/authz_tests/matrix.csv +APP2-T4,STRIDE,denial of service,Saturate GraphQL gateway,Downtime for partner launch,Botnet hitting persisted queries,Adaptive rate limiting + backoff,tests/APP2/perf_k6.js +APP2-L1,LINDDUN,linkability,Correlate partner user IDs,Privacy breach,Combining analytics tokens,Token anonymization,tests/APP2/idempotency_tests/session_replay.yaml +APP2-L2,LINDDUN,detectability,Infer partner contract terms,Competitive intel,Response length variations,Response padding + caching,tests/APP2/perf_k6.js +APP2-L3,LINDDUN,non-compliance,GDPR deletion violation,Regulatory fine,Async queue backlog,Deletion workflow tests,artifacts/APP2/run_manifest.json diff --git a/artifacts/threat_matrices/APP2_threat_matrix.md b/artifacts/threat_matrices/APP2_threat_matrix.md new file mode 100644 index 000000000..af760dfc1 --- /dev/null +++ b/artifacts/threat_matrices/APP2_threat_matrix.md @@ -0,0 +1,15 @@ +# APP2 Partner Hub - Threat & Attack Matrix + +| Threat | Framework | Objective | Attack Vector | Mitigations | Tests | +|--------|-----------|-----------|---------------|-------------|-------| +| APP2-T1 | STRIDE (Spoofing) | Inject fraudulent bookings via fake webhook | HMAC forged using leaked secret | Rotate partner secrets, strict timestamp drift, replay cache | `tests/APP2/partner_simulators/valid_signature.py` | +| APP2-T2 | STRIDE (Tampering) | Modify offer payload to include malicious URLs | CDN MITM / compromised edge worker | TLS 1.3 enforcement, signed payloads | `tests/APP2/contract_tests/openapi.yaml` | +| APP2-T3 | STRIDE (Information Disclosure) | Leak embargoed offers | Misconfigured Kong route with wildcard path | Zero-trust route policies, authz regression | `tests/APP2/authz_tests/matrix.csv` | +| APP2-T4 | STRIDE (DoS) | Overload GraphQL gateway | Botnet hitting persisted query IDs | Adaptive rate limiting, circuit breakers | `tests/APP2/perf_k6.js` | +| APP2-L1 | LINDDUN (Linkability) | Associate partner user IDs across campaigns | Analytics token re-use | Token rotation, privacy budget enforcement | `tests/APP2/idempotency_tests/session_replay.yaml` | +| APP2-L2 | LINDDUN (Detectability) | Infer partner contract tier | Response size/time differences | Response padding, caching | `tests/APP2/perf_k6.js` | +| APP2-L3 | LINDDUN (Non-compliance) | Miss GDPR deletion SLA | Queue backlog delaying delete events | Queue drain monitors, deletion tests | `artifacts/APP2/run_manifest.json` | + +## Backtesting +- Replayed March 2024 partner outage (HTTP 429 storm) verifying new exponential backoff. +- Simulated 2023 webhook credential leak; secret rotation + signature validation blocked forged requests. diff --git a/artifacts/threat_matrices/APP3_threat_matrix.csv b/artifacts/threat_matrices/APP3_threat_matrix.csv new file mode 100644 index 000000000..0d89536fd --- /dev/null +++ b/artifacts/threat_matrices/APP3_threat_matrix.csv @@ -0,0 +1,8 @@ +threat_id,framework,category,description,attacker_objective,attack_vector,mitigation,test_reference +APP3-T1,STRIDE,spoofing,Impersonate clinician to view records,Access PHI,Compromised Azure AD credentials,Conditional access + device checks,tests/APP3/authz_tests/matrix.csv +APP3-T2,STRIDE,tampering,Alter FHIR resource history,Medical fraud,Compromise FHIR gateway,Immutable audit ledger + signatures,tests/APP3/idempotency_tests/audit_append.yaml +APP3-T3,STRIDE,information disclosure,Leak remote monitoring data,Data breach,Open admin ingress 0.0.0.0/0,Network policy + WAF,policy/APP3/security_controls.rego +APP3-T4,STRIDE,denial of service,Exhaust FHIR gateway threads,Impact care coordination,Batched wildcard queries,Rate limiting + caching,tests/APP3/perf_k6.js +APP3-L1,LINDDUN,linkability,Link wearable data to patient identity,Privacy violation,Correlate ML triage outputs,Noise injection + pseudonyms,tests/APP3/contract_tests/openapi.yaml +APP3-L2,LINDDUN,non-compliance,Miss HIPAA audit trails,Regulatory penalty,Unsigned audit entries,Signing service + ledger tests,tests/APP3/idempotency_tests/audit_append.yaml +APP3-L3,LINDDUN,detectability,Infer mental health visits,Stigma risk,Timing differences in appointment API,Uniform response times,tests/APP3/perf_k6.js diff --git a/artifacts/threat_matrices/APP3_threat_matrix.md b/artifacts/threat_matrices/APP3_threat_matrix.md new file mode 100644 index 000000000..c3006222e --- /dev/null +++ b/artifacts/threat_matrices/APP3_threat_matrix.md @@ -0,0 +1,13 @@ +# APP3 Healthcare Portal - Threat & Attack Matrix + +| Threat | Framework | Objective | Attack Vector | Mitigation | Test | +|--------|-----------|-----------|---------------|------------|------| +| APP3-T1 | STRIDE (Spoofing) | Gain access to PHI via clinician impersonation | Compromised Azure AD credentials | Conditional access, hardware MFA | `tests/APP3/authz_tests/matrix.csv` | +| APP3-T2 | STRIDE (Tampering) | Modify patient record to falsify history | Exploit FHIR search injection | Signed audit ledger, parameterized queries | `tests/APP3/idempotency_tests/audit_append.yaml` | +| APP3-T3 | STRIDE (Information Disclosure) | Leak remote monitoring telemetry | Public admin ingress | Network policies, WAF, TLS 1.2+ | `policy/APP3/security_controls.rego` | +| APP3-T4 | STRIDE (DoS) | Exhaust FHIR gateway resources | Wildcard queries, patient export loops | Query throttles, caching | `tests/APP3/perf_k6.js` | +| APP3-L1 | LINDDUN (Linkability) | Link wearable metrics to identity | Cross-service correlation | Noise injection + pseudonyms | `tests/APP3/contract_tests/openapi.yaml` | +| APP3-L2 | LINDDUN (Non-compliance) | Break HIPAA audit logging | Unsigned audit entries | Hardware signing & immutable ledger | `tests/APP3/idempotency_tests/audit_append.yaml` | +| APP3-L3 | LINDDUN (Detectability) | Detect mental health visits | Timing side-channels | Uniform response times | `tests/APP3/perf_k6.js` | + +Backtests: March 2025 FHIR wildcard abuse incident, 2024 admin ingress misconfiguration, 2023 audit ledger tampering attempt. diff --git a/artifacts/threat_matrices/APP4_threat_matrix.csv b/artifacts/threat_matrices/APP4_threat_matrix.csv new file mode 100644 index 000000000..65b2ae469 --- /dev/null +++ b/artifacts/threat_matrices/APP4_threat_matrix.csv @@ -0,0 +1,7 @@ +threat_id,framework,category,description,attacker_objective,attack_vector,mitigation,test_reference +APP4-T1,STRIDE,spoofing,Impersonate store device to submit fraudulent transactions,Financial fraud,Stolen device certificates,Mutual TLS + device attestation,tests/APP4/authz_tests/matrix.csv +APP4-T2,STRIDE,tampering,Alter settlement batch totals,Financial manipulation,Compromise batch job container,Immutable infrastructure + signed artifacts,policy/APP4/security_controls.rego +APP4-T3,STRIDE,information disclosure,Exfiltrate PAN tokens,Data breach,Plaintext HSM credentials in lambda,Secrets manager + KMS integration,artifacts/APP4/tf_plan.json +APP4-T4,STRIDE,denial of service,Take down checkout API,Service outage,Botnet hitting /checkout,Rate limiting + WAF,tests/APP4/perf_k6.js +APP4-L1,LINDDUN,linkability,Link purchases to individuals,Privacy violation,Correlate inventory + receipt events,Tokenization + anonymized analytics,tests/APP4/contract_tests/openapi.yaml +APP4-L2,LINDDUN,non-compliance,Breach PCI logging rules,Regulatory penalty,Unmasked card tokens in logs,Log scrubbing + tests,tests/APP4/idempotency_tests/audit_logs.yaml diff --git a/artifacts/threat_matrices/APP4_threat_matrix.md b/artifacts/threat_matrices/APP4_threat_matrix.md new file mode 100644 index 000000000..013f31963 --- /dev/null +++ b/artifacts/threat_matrices/APP4_threat_matrix.md @@ -0,0 +1,12 @@ +# APP4 Retail Payments - Threat & Attack Matrix + +| Threat | Framework | Objective | Vector | Mitigation | Test | +|--------|-----------|-----------|--------|------------|------| +| APP4-T1 | STRIDE (Spoofing) | Submit fraudulent transactions | Stolen device certificates | Mutual TLS, device attestation, revocation lists | `tests/APP4/authz_tests/matrix.csv` | +| APP4-T2 | STRIDE (Tampering) | Modify settlement totals | Compromise batch job container | Signed pipelines, runtime attestation | `policy/APP4/security_controls.rego` | +| APP4-T3 | STRIDE (Information Disclosure) | Exfiltrate PAN tokens | Plaintext HSM credentials | Secrets manager, KMS envelope encryption | `artifacts/APP4/tf_plan.json` | +| APP4-T4 | STRIDE (DoS) | Checkout outage | API flooding | WAF rules, adaptive rate limiting | `tests/APP4/perf_k6.js` | +| APP4-L1 | LINDDUN (Linkability) | Link purchases to individuals | Combine receipt + inventory events | Pseudonymization, minimal logging | `tests/APP4/contract_tests/openapi.yaml` | +| APP4-L2 | LINDDUN (Non-compliance) | Breach PCI logging | Unmasked tokens in logs | Log scrubbers, audit tests | `tests/APP4/idempotency_tests/audit_logs.yaml` | + +Backtests: 2024 POS credential theft, 2023 settlement mismatch, 2022 WAF bypass outage. diff --git a/cli-tests/APP1/cli_failure.sh b/cli-tests/APP1/cli_failure.sh new file mode 100755 index 000000000..ee6c8d95c --- /dev/null +++ b/cli-tests/APP1/cli_failure.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-failure][APP1] $1"; } + +log "simulate pipeline failure" +if fixops pipeline:run --module ssdlc --run-id fake-run; then + log "expected failure but command succeeded" + exit 1 +else + log "captured failure exit code $?" +fi diff --git a/cli-tests/APP1/cli_smoke.sh b/cli-tests/APP1/cli_smoke.sh new file mode 100755 index 000000000..3a2cf9d5c --- /dev/null +++ b/cli-tests/APP1/cli_smoke.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +export FIXOPS_MODE=enterprise + +log() { echo "[cli-smoke][APP1] $1"; } + +log "version" +fixops --version || { log "missing fixops binary"; exit 1; } + +log "list commands" +fixops --list-commands --json > artifacts/APP1/cli_catalog.json + +log "upload design" +fixops design:upload inputs/APP1/design.csv || log "design upload failed (expected in offline mode)" + +log "upload sbom" +fixops sbom:upload inputs/APP1/sbom.json || log "sbom upload failed (expected in offline mode)" + +log "enumerate api" +fixops api:enumerate --out artifacts/APP1_endpoints.json || log "api enumerate simulated" + +log "matrix" +fixops api:matrix --routes artifacts/APP1_endpoints.json --out artifacts/APP1_api_matrix.json || log "api matrix simulated" + +log "done" diff --git a/cli-tests/APP2/cli_failure.sh b/cli-tests/APP2/cli_failure.sh new file mode 100755 index 000000000..f3be82a64 --- /dev/null +++ b/cli-tests/APP2/cli_failure.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-failure][APP2] $1"; } + +if fixops partner:webhook-test --partner demo --fail; then + log "Expected failure but succeeded" + exit 1 +else + log "Failure scenario captured" +fi diff --git a/cli-tests/APP2/cli_smoke.sh b/cli-tests/APP2/cli_smoke.sh new file mode 100755 index 000000000..dfec6373b --- /dev/null +++ b/cli-tests/APP2/cli_smoke.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-smoke][APP2] $1"; } + +fixops --version || log "fixops not installed" + +fixops --list-commands --json > artifacts/APP2/cli_catalog.json || log "catalog simulated" + +fixops api:enumerate --out artifacts/APP2_endpoints.json --app APP2 || log "enumerate simulated" +fixops api:matrix --routes artifacts/APP2_endpoints.json --out artifacts/APP2_api_matrix.json || log "matrix simulated" diff --git a/cli-tests/APP3/cli_failure.sh b/cli-tests/APP3/cli_failure.sh new file mode 100755 index 000000000..2bef20a07 --- /dev/null +++ b/cli-tests/APP3/cli_failure.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-failure][APP3] $1"; } + +if fixops healthcare:validate --run invalid; then + log "Expected failure" + exit 1 +else + log "Failure captured" +fi diff --git a/cli-tests/APP3/cli_smoke.sh b/cli-tests/APP3/cli_smoke.sh new file mode 100755 index 000000000..6656b79e5 --- /dev/null +++ b/cli-tests/APP3/cli_smoke.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-smoke][APP3] $1"; } + +fixops --version || log "fixops missing" +fixops --list-commands --json > artifacts/APP3/cli_catalog.json || log "catalog simulated" +fixops api:enumerate --app APP3 --out artifacts/APP3_endpoints.json || log "enumerate simulated" +fixops api:matrix --routes artifacts/APP3_endpoints.json --out artifacts/APP3_api_matrix.json || log "matrix simulated" diff --git a/cli-tests/APP4/cli_failure.sh b/cli-tests/APP4/cli_failure.sh new file mode 100755 index 000000000..4311a5ba2 --- /dev/null +++ b/cli-tests/APP4/cli_failure.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-failure][APP4] $1"; } + +if fixops pci:attest --run invalid; then + log "Expected failure" + exit 1 +else + log "Failure recorded" +fi diff --git a/cli-tests/APP4/cli_smoke.sh b/cli-tests/APP4/cli_smoke.sh new file mode 100755 index 000000000..ccfa52898 --- /dev/null +++ b/cli-tests/APP4/cli_smoke.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[cli-smoke][APP4] $1"; } + +fixops --version || log "fixops missing" +fixops --list-commands --json > artifacts/APP4/cli_catalog.json || log "catalog simulated" +fixops api:enumerate --app APP4 --out artifacts/APP4_endpoints.json || log "enumerate simulated" +fixops api:matrix --routes artifacts/APP4_endpoints.json --out artifacts/APP4_api_matrix.json || log "matrix simulated" diff --git a/evidence/README.md b/evidence/README.md new file mode 100644 index 000000000..492faba52 --- /dev/null +++ b/evidence/README.md @@ -0,0 +1,18 @@ +# Evidence Bundles + +The FixOps demo previously committed generated `.zip` bundles for APP1–APP4. Because +binary artifacts are not supported by our automation when opening pull requests, +these archives are no longer tracked in git. + +To reproduce them locally, run: + +``` +python -m cli.fixops_ci evidence bundle --tag APP1 --out evidence +python -m cli.fixops_ci evidence bundle --tag APP2 --out evidence +python -m cli.fixops_ci evidence bundle --tag APP3 --out evidence +python -m cli.fixops_ci evidence bundle --tag APP4 --out evidence +``` + +Each invocation recreates `evidence/bundles/_bundle.zip` along with the updated +`MANIFEST.yaml`. The repository `.gitignore` already excludes the generated archives, +so subsequent commits remain text-only. diff --git a/inputs/APP1/cve_feed.json b/inputs/APP1/cve_feed.json new file mode 100644 index 000000000..e09f801a2 --- /dev/null +++ b/inputs/APP1/cve_feed.json @@ -0,0 +1,8 @@ +{ + "generated": "2025-10-28T12:00:00Z", + "cves": [ + {"id": "CVE-2025-0001", "package": "pg", "version": "8.9.0", "cvss": 9.1, "published": "2025-10-01", "summary": "RCE via malformed query"}, + {"id": "CVE-2024-1709", "package": "express", "version": "4.18.2", "cvss": 7.8, "published": "2024-05-15", "summary": "Improper auth bypass"}, + {"id": "CVE-2024-31122", "package": "jsonwebtoken", "version": "9.0.2", "cvss": 8.3, "published": "2024-08-11", "summary": "Signature validation bypass"} + ] +} diff --git a/inputs/APP1/design.csv b/inputs/APP1/design.csv new file mode 100644 index 000000000..8695da576 --- /dev/null +++ b/inputs/APP1/design.csv @@ -0,0 +1,7 @@ +component,subcomponent,owner,data_class,description,control_scope +web,frontend,frontend-team,PII,"quote form collects name,email,ssn","SOC2,ISO27001,HIPAA" +api,pricing,pricing-team,PII,"quote calculation, stores customer id","SOC2,ISO27001" +db,customers-db,platform-team,PII/PHI,"postgres storing customer records","SOC2,ISO27001,HIPAA" +esb,billing,billing-team,PII,"payment callbacks and webhooks","PCI" +analytics,pricing-ml,ml-team,PII,"ML pricing model uses historical claim data","SOC2,ISO27001" +iac,terraform,platform-team,Config,"provisions eks cluster + postgres","SOC2,ISO27001" diff --git a/inputs/APP1/findings.json b/inputs/APP1/findings.json new file mode 100644 index 000000000..6cd00ac12 --- /dev/null +++ b/inputs/APP1/findings.json @@ -0,0 +1,8 @@ +{ + "scan_time": "2025-10-28T12:15:00Z", + "findings": [ + {"id": "F-001", "type": "k8s:misconfig", "severity": "high", "resource": "ns/prod/db", "description": "postgres service exposed via LoadBalancer (public)"}, + {"id": "F-002", "type": "iam:exposed_key", "severity": "medium", "resource": "aws/iam/user/deploy", "description": "long-lived access key active > 90 days"}, + {"id": "F-003", "type": "workload:vuln", "severity": "critical", "resource": "deployment/pricing", "description": "container image uses vulnerable openssl 1.1.1u"} + ] +} diff --git a/inputs/APP1/results.sarif b/inputs/APP1/results.sarif new file mode 100644 index 000000000..e1e6d7055 --- /dev/null +++ b/inputs/APP1/results.sarif @@ -0,0 +1,43 @@ +{ + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": "app1-sast", + "informationUri": "https://scanner.example.com/app1", + "version": "2025.10" + } + }, + "results": [ + { + "ruleId": "S001", + "level": "warning", + "message": {"text": "Potential SQL injection in /api/quote"}, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {"uri": "services/pricing.js"}, + "region": {"startLine": 123} + } + } + ] + }, + { + "ruleId": "S014", + "level": "error", + "message": {"text": "Hardcoded JWT secret in config"}, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {"uri": "config/default.json"}, + "region": {"startLine": 17} + } + } + ] + } + ] + } + ] +} diff --git a/inputs/APP1/sbom.json b/inputs/APP1/sbom.json new file mode 100644 index 000000000..58c6b6415 --- /dev/null +++ b/inputs/APP1/sbom.json @@ -0,0 +1,21 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-28T12:00:00Z", + "component": { + "type": "application", + "name": "APP1 Insurance Platform", + "version": "2.6.4", + "purl": "pkg:app/app1-insurance@2.6.4" + } + }, + "components": [ + {"type": "library", "name": "react", "version": "18.2.0", "purl": "pkg:npm/react@18.2.0"}, + {"type": "library", "name": "express", "version": "4.18.2", "purl": "pkg:npm/express@4.18.2"}, + {"type": "library", "name": "pg", "version": "8.9.0", "purl": "pkg:npm/pg@8.9.0"}, + {"type": "library", "name": "jsonwebtoken", "version": "9.0.2", "purl": "pkg:npm/jsonwebtoken@9.0.2"}, + {"type": "library", "name": "terraform", "version": "1.7.5", "purl": "pkg:generic/hashicorp/terraform@1.7.5"} + ] +} diff --git a/inputs/APP1/vex_doc.json b/inputs/APP1/vex_doc.json new file mode 100644 index 000000000..587cac9ac --- /dev/null +++ b/inputs/APP1/vex_doc.json @@ -0,0 +1,9 @@ +{ + "vex_id": "VEX-APP1-2025-10-28-001", + "statements": [ + {"cve": "CVE-2025-0001", "status": "not_affected", "justification": "Database uses pgbouncer with input sanitization"}, + {"cve": "CVE-2024-1709", "status": "fixed", "justification": "Upgraded to express 4.19.0 in October 2025"}, + {"cve": "CVE-2024-31122", "status": "under_investigation", "justification": "JWT tokens signed via HSM; verifying claim"} + ], + "generated": "2025-10-28T12:10:00Z" +} diff --git a/inputs/APP2/cve_feed.json b/inputs/APP2/cve_feed.json new file mode 100644 index 000000000..4eb3068f8 --- /dev/null +++ b/inputs/APP2/cve_feed.json @@ -0,0 +1,7 @@ +{ + "generated": "2025-10-28T13:32:00Z", + "cves": [ + {"id": "CVE-2024-45598", "package": "next", "version": "14.1.0", "cvss": 8.7, "published": "2024-09-10", "summary": "SSRF via image optimization"}, + {"id": "CVE-2025-11219", "package": "apollo-server", "version": "4.9.3", "cvss": 7.4, "published": "2025-02-18", "summary": "GraphQL batch auth bypass"} + ] +} diff --git a/inputs/APP2/design.csv b/inputs/APP2/design.csv new file mode 100644 index 000000000..d29f7f369 --- /dev/null +++ b/inputs/APP2/design.csv @@ -0,0 +1,7 @@ +component,subcomponent,owner,data_class,description,control_scope +frontend,shell,experience-team,Internal,"Micro-frontend shell orchestrating partner widgets","SOC2,ISO27001" +frontend,partner-widget,partner-integrations,Confidential,"Embeds partner travel offers","SOC2,ISO27001,GDPR" +api,aggregation,api-team,Confidential,"GraphQL gateway aggregating partner APIs","SOC2,ISO27001" +services,rate-limiter,platform-team,Operational,"Kong gateway enforcing policies","SOC2" +queue,event-bridge,platform-team,Confidential,"Handles partner webhook ingestion","SOC2,GDPR" +iac,terraform,cloud-team,Config,"Provisions CDN, gateway, lambda transformers","SOC2,ISO27001" diff --git a/inputs/APP2/findings.json b/inputs/APP2/findings.json new file mode 100644 index 000000000..a19051035 --- /dev/null +++ b/inputs/APP2/findings.json @@ -0,0 +1,8 @@ +{ + "scan_time": "2025-10-28T13:35:00Z", + "findings": [ + {"id": "F-201", "type": "cdn:misconfig", "severity": "high", "resource": "cloudfront/app2", "description": "Origin failover disabled leading to single point of failure"}, + {"id": "F-202", "type": "api:rate_limit", "severity": "medium", "resource": "kong/service/partner", "description": "429 threshold misaligned with contract"}, + {"id": "F-203", "type": "runtime:secret", "severity": "critical", "resource": "lambda/transformer", "description": "Partner secret stored in plaintext env"} + ] +} diff --git a/inputs/APP2/results.sarif b/inputs/APP2/results.sarif new file mode 100644 index 000000000..4b47a5ea8 --- /dev/null +++ b/inputs/APP2/results.sarif @@ -0,0 +1,33 @@ +{ + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": "app2-sast", + "informationUri": "https://scanner.example.com/app2", + "version": "2025.9" + } + }, + "results": [ + { + "ruleId": "GQL001", + "level": "error", + "message": {"text": "GraphQL resolver concatenates user-supplied URL"}, + "locations": [ + {"physicalLocation": {"artifactLocation": {"uri": "api/resolvers/partnerFeed.ts"}, "region": {"startLine": 87}}} + ] + }, + { + "ruleId": "WEB005", + "level": "warning", + "message": {"text": "Micro-frontend exposes analytics token in window scope"}, + "locations": [ + {"physicalLocation": {"artifactLocation": {"uri": "frontend/widgets/offers.tsx"}, "region": {"startLine": 52}}} + ] + } + ] + } + ] +} diff --git a/inputs/APP2/sbom.json b/inputs/APP2/sbom.json new file mode 100644 index 000000000..96153c466 --- /dev/null +++ b/inputs/APP2/sbom.json @@ -0,0 +1,20 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-28T13:30:00Z", + "component": { + "type": "application", + "name": "APP2 Partner Hub", + "version": "1.14.2", + "purl": "pkg:app/app2-partner-hub@1.14.2" + } + }, + "components": [ + {"type": "library", "name": "next", "version": "14.1.0", "purl": "pkg:npm/next@14.1.0"}, + {"type": "library", "name": "apollo-server", "version": "4.9.3", "purl": "pkg:npm/apollo-server@4.9.3"}, + {"type": "library", "name": "kong", "version": "3.5.0", "purl": "pkg:docker/kong@3.5.0"}, + {"type": "library", "name": "aws-sdk", "version": "3.540.0", "purl": "pkg:npm/@aws-sdk/client-sqs@3.540.0"} + ] +} diff --git a/inputs/APP2/vex_doc.json b/inputs/APP2/vex_doc.json new file mode 100644 index 000000000..981f69fc8 --- /dev/null +++ b/inputs/APP2/vex_doc.json @@ -0,0 +1,8 @@ +{ + "vex_id": "VEX-APP2-2025-10-28-001", + "statements": [ + {"cve": "CVE-2024-45598", "status": "affected", "justification": "Image proxy uses remote patterns; mitigation pending", "action": "deploy allowlist patch by 2025-11-05"}, + {"cve": "CVE-2025-11219", "status": "not_affected", "justification": "Gateway enforces persisted queries only"} + ], + "generated": "2025-10-28T13:33:00Z" +} diff --git a/inputs/APP3/cve_feed.json b/inputs/APP3/cve_feed.json new file mode 100644 index 000000000..d9a30ceef --- /dev/null +++ b/inputs/APP3/cve_feed.json @@ -0,0 +1,7 @@ +{ + "generated": "2025-10-28T14:02:00Z", + "cves": [ + {"id": "CVE-2024-34145", "package": "spring-boot", "version": "3.3.2", "cvss": 9.8, "published": "2024-07-05", "summary": "RCE in Spring expression language"}, + {"id": "CVE-2025-07701", "package": "azure-storage-blob", "version": "12.24.0", "cvss": 6.9, "published": "2025-01-12", "summary": "Missing TLS hostname verification"} + ] +} diff --git a/inputs/APP3/design.csv b/inputs/APP3/design.csv new file mode 100644 index 000000000..8e8351578 --- /dev/null +++ b/inputs/APP3/design.csv @@ -0,0 +1,7 @@ +component,subcomponent,owner,data_class,description,control_scope +web,patient-portal,experience-team,PHI,"Patient portal for scheduling and messaging","SOC2,HIPAA" +api,ehr-gateway,clinical-platform,PHI,"FHIR API proxy for EMR integrations","SOC2,HIPAA" +services,ml-triage,ml-team,PHI,"Risk scoring service for remote monitoring","SOC2,HIPAA" +queue,event-stream,platform-team,PHI,"HL7 event ingestion via Kafka","SOC2,HIPAA" +db,audit-ledger,security-team,PHI,"Immutable audit trail on PostgreSQL","SOC2,HIPAA" +iac,terraform,platform-team,Config,"Provision AKS cluster, Key Vault, CosmosDB","SOC2,HIPAA" diff --git a/inputs/APP3/findings.json b/inputs/APP3/findings.json new file mode 100644 index 000000000..9ac65cc2a --- /dev/null +++ b/inputs/APP3/findings.json @@ -0,0 +1,8 @@ +{ + "scan_time": "2025-10-28T14:05:00Z", + "findings": [ + {"id": "F-301", "type": "k8s:privilege", "severity": "high", "resource": "deployment/ehr-gateway", "description": "Container running as root"}, + {"id": "F-302", "type": "iam:overprivileged", "severity": "medium", "resource": "azure/ad/app/app3-svc", "description": "Service principal has directory.write"}, + {"id": "F-303", "type": "network:exposure", "severity": "critical", "resource": "aks/app3-loadbalancer", "description": "Ingress allows 0.0.0.0/0 to admin API"} + ] +} diff --git a/inputs/APP3/results.sarif b/inputs/APP3/results.sarif new file mode 100644 index 000000000..f0453a110 --- /dev/null +++ b/inputs/APP3/results.sarif @@ -0,0 +1,23 @@ +{ + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": {"driver": {"name": "app3-sast", "version": "2025.8"}}, + "results": [ + { + "ruleId": "FHIR001", + "level": "error", + "message": {"text": "FHIR search endpoint allows wildcard injection"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "gateway/src/main/java/com/app3/FhirSearchController.java"}, "region": {"startLine": 142}}}] + }, + { + "ruleId": "AUDIT003", + "level": "warning", + "message": {"text": "Audit trail missing tamper-evident signature"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "audit/service.py"}, "region": {"startLine": 61}}}] + } + ] + } + ] +} diff --git a/inputs/APP3/sbom.json b/inputs/APP3/sbom.json new file mode 100644 index 000000000..2ec4f48a8 --- /dev/null +++ b/inputs/APP3/sbom.json @@ -0,0 +1,20 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-28T14:00:00Z", + "component": { + "type": "application", + "name": "APP3 Healthcare Portal", + "version": "5.3.1", + "purl": "pkg:app/app3-healthcare@5.3.1" + } + }, + "components": [ + {"type": "library", "name": "angular", "version": "17.1.0", "purl": "pkg:npm/@angular/core@17.1.0"}, + {"type": "library", "name": "spring-boot", "version": "3.3.2", "purl": "pkg:maven/org.springframework.boot/spring-boot-starter-web@3.3.2"}, + {"type": "library", "name": "azure-storage-blob", "version": "12.24.0", "purl": "pkg:maven/com.azure/azure-storage-blob@12.24.0"}, + {"type": "library", "name": "numpy", "version": "1.26.4", "purl": "pkg:pypi/numpy@1.26.4"} + ] +} diff --git a/inputs/APP3/vex_doc.json b/inputs/APP3/vex_doc.json new file mode 100644 index 000000000..6802b6b55 --- /dev/null +++ b/inputs/APP3/vex_doc.json @@ -0,0 +1,8 @@ +{ + "vex_id": "VEX-APP3-2025-10-28-001", + "statements": [ + {"cve": "CVE-2024-34145", "status": "affected", "justification": "Spring boot version pending upgrade", "action": "Upgrade to 3.3.4 and retest by 2025-11-10"}, + {"cve": "CVE-2025-07701", "status": "not_affected", "justification": "Client configured with enforceHostnameVerification=true"} + ], + "generated": "2025-10-28T14:03:00Z" +} diff --git a/inputs/APP4/cve_feed.json b/inputs/APP4/cve_feed.json new file mode 100644 index 000000000..9ff794678 --- /dev/null +++ b/inputs/APP4/cve_feed.json @@ -0,0 +1,7 @@ +{ + "generated": "2025-10-28T14:22:00Z", + "cves": [ + {"id": "CVE-2024-55781", "package": "go-chi", "version": "5.0.10", "cvss": 8.2, "published": "2024-11-02", "summary": "Routing bypass allows path traversal"}, + {"id": "CVE-2025-01911", "package": "grpc", "version": "1.62.0", "cvss": 7.9, "published": "2025-03-01", "summary": "TLS downgrade vulnerability"} + ] +} diff --git a/inputs/APP4/design.csv b/inputs/APP4/design.csv new file mode 100644 index 000000000..af9f90d52 --- /dev/null +++ b/inputs/APP4/design.csv @@ -0,0 +1,7 @@ +component,subcomponent,owner,data_class,description,control_scope +mobile,pos-app,device-team,PCI,"Tablet POS app capturing card data","PCI-DSS" +api,checkout,payments-team,PCI,"Golang checkout API processing tokens","PCI-DSS" +services,settlement,finance-team,PCI,"Nightly settlement batch jobs","PCI-DSS,SOX" +queue,event-stream,platform-team,Operational,"Kafka topics for receipts and inventory","PCI-DSS" +infrastructure,edge-gateway,platform-team,Operational,"IoT gateway bridging stores to cloud","PCI-DSS" +iac,terraform,platform-team,Config,"Provisions EKS, MSK, DynamoDB, KMS","PCI-DSS,ISO27001" diff --git a/inputs/APP4/findings.json b/inputs/APP4/findings.json new file mode 100644 index 000000000..d762330bb --- /dev/null +++ b/inputs/APP4/findings.json @@ -0,0 +1,8 @@ +{ + "scan_time": "2025-10-28T14:25:00Z", + "findings": [ + {"id": "F-401", "type": "pci:unencrypted", "severity": "critical", "resource": "lambda/tokenizer", "description": "HSM credentials stored as plaintext env"}, + {"id": "F-402", "type": "network:exposure", "severity": "high", "resource": "edge-gateway", "description": "MQTT listener open to internet"}, + {"id": "F-403", "type": "compliance:logging", "severity": "medium", "resource": "checkout-api", "description": "Audit logs missing card token masking"} + ] +} diff --git a/inputs/APP4/results.sarif b/inputs/APP4/results.sarif new file mode 100644 index 000000000..56e7cf43c --- /dev/null +++ b/inputs/APP4/results.sarif @@ -0,0 +1,23 @@ +{ + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": {"driver": {"name": "app4-sast", "version": "2025.7"}}, + "results": [ + { + "ruleId": "GOSEC-001", + "level": "error", + "message": {"text": "Hardcoded HSM credentials in checkout handler"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "checkout/handler.go"}, "region": {"startLine": 88}}}] + }, + { + "ruleId": "NODE-SEC-021", + "level": "warning", + "message": {"text": "POS frontend uses outdated crypto library"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "pos/package.json"}, "region": {"startLine": 32}}}] + } + ] + } + ] +} diff --git a/inputs/APP4/sbom.json b/inputs/APP4/sbom.json new file mode 100644 index 000000000..c8229ef16 --- /dev/null +++ b/inputs/APP4/sbom.json @@ -0,0 +1,20 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-28T14:20:00Z", + "component": { + "type": "application", + "name": "APP4 Retail Payments", + "version": "3.9.0", + "purl": "pkg:app/app4-retail@3.9.0" + } + }, + "components": [ + {"type": "library", "name": "go-chi", "version": "5.0.10", "purl": "pkg:golang/github.com/go-chi/chi@5.0.10"}, + {"type": "library", "name": "aws-lambda-go", "version": "1.45.0", "purl": "pkg:golang/github.com/aws/aws-lambda-go@1.45.0"}, + {"type": "library", "name": "grpc", "version": "1.62.0", "purl": "pkg:golang/google.golang.org/grpc@1.62.0"}, + {"type": "library", "name": "node", "version": "20.10.0", "purl": "pkg:docker/node@20.10.0"} + ] +} diff --git a/inputs/APP4/vex_doc.json b/inputs/APP4/vex_doc.json new file mode 100644 index 000000000..303a1290e --- /dev/null +++ b/inputs/APP4/vex_doc.json @@ -0,0 +1,8 @@ +{ + "vex_id": "VEX-APP4-2025-10-28-001", + "statements": [ + {"cve": "CVE-2024-55781", "status": "affected", "justification": "Checkout uses vulnerable router", "action": "Upgrade to go-chi 5.0.11 and add regression test"}, + {"cve": "CVE-2025-01911", "status": "not_affected", "justification": "Mutual TLS enforced with TLS1.3 only"} + ], + "generated": "2025-10-28T14:23:00Z" +} diff --git a/policy/APP1/security_controls.rego b/policy/APP1/security_controls.rego new file mode 100644 index 000000000..578a812f7 --- /dev/null +++ b/policy/APP1/security_controls.rego @@ -0,0 +1,52 @@ +package app1.policy + +deny[msg] { + some change + change := input.resource_changes[_] + change.change.after.type == "aws_db_instance" + change.change.after.publicly_accessible + msg := "Database instances must not be publicly accessible." +} + +deny[msg] { + some change + change := input.resource_changes[_] + change.change.after.type == "aws_db_instance" + not change.change.after.storage_encrypted + msg := "Database must be encrypted at rest." +} + +deny[msg] { + some change + change := input.resource_changes[_] + change.change.after.type == "aws_security_group" + some ingress + ingress := change.change.after.ingress[_] + ingress.cidr_blocks[_] == "0.0.0.0/0" + msg := "Ingress rules cannot expose 0.0.0.0/0." +} + +deny[msg] { + some change + change := input.resource_changes[_] + change.change.after.type == "kubernetes_secret" + re_match("(?i)key|password|secret", change.change.after.metadata.name) + msg := sprintf("Secret %s must be sourced from external vault", [change.change.after.metadata.name]) +} + +deny[msg] { + some change + change := input.resource_changes[_] + change.change.after.type == "aws_lb_listener" + not startswith(change.change.after.protocol, "TLS") + msg := "Load balancers must enforce TLS 1.2+." +} + +deny[msg] { + some image + image := input.images[_] + now := time.now_ns() / 1000000000 + age := now - image.metadata.created + age > 180 * 24 * 3600 + msg := sprintf("Base image %s older than 180 days", [image.name]) +} diff --git a/policy/APP2/security_controls.rego b/policy/APP2/security_controls.rego new file mode 100644 index 000000000..fdcf79ce5 --- /dev/null +++ b/policy/APP2/security_controls.rego @@ -0,0 +1,32 @@ +package app2.policy + +violation[msg] { + input.gateway.routes[_].path == "/api/webhooks/{partner}" + not input.gateway.routes[_].plugins.signatures.enabled + msg := "Webhook route must enforce HMAC signature plugin" +} + +violation[msg] { + some svc + svc := input.kong_services[_] + svc.protocol == "http" + msg := sprintf("Service %s must use https", [svc.name]) +} + +violation[msg] { + input.infrastructure.cdn.public_origin + msg := "CDN origins cannot be public buckets" +} + +violation[msg] { + some image + image := input.images[_] + image.base_age_days > 180 + msg := sprintf("Image %s older than 180 days", [image.name]) +} + +violation[msg] { + input.lambda_env[_].key == "PARTNER_SECRET" + input.lambda_env[_].source != "secrets-manager" + msg := "Partner secrets must be sourced from AWS Secrets Manager" +} diff --git a/policy/APP3/security_controls.rego b/policy/APP3/security_controls.rego new file mode 100644 index 000000000..ed04c3f7e --- /dev/null +++ b/policy/APP3/security_controls.rego @@ -0,0 +1,32 @@ +package app3.policy + +deny[msg] { + input.kubernetes.ingress[_].spec.rules[_].host == "admin.app3.example.com" + input.kubernetes.ingress[_].spec.rules[_].http.paths[_].backend.service.port.number == 8080 + input.kubernetes.ingress[_].spec.rules[_].http.paths[_].backend.service.allow_public + msg := "Admin ingress cannot be publicly accessible" +} + +deny[msg] { + some deploy + deploy := input.kubernetes.deployments[_] + deploy.spec.template.spec.containers[_].securityContext.runAsNonRoot == false + msg := sprintf("Deployment %s must run as non-root", [deploy.metadata.name]) +} + +deny[msg] { + input.azure.traffic_manager.profile.protocols[_] != "https" + msg := "Traffic manager endpoints must enforce HTTPS" +} + +deny[msg] { + input.cosmos.consistency != "Strong" + msg := "CosmosDB must use Strong consistency for audit ledger" +} + +deny[msg] { + some image + image := input.images[_] + image.base_age_days > 180 + msg := sprintf("Base image %s older than 180 days", [image.name]) +} diff --git a/policy/APP4/security_controls.rego b/policy/APP4/security_controls.rego new file mode 100644 index 000000000..deccb6325 --- /dev/null +++ b/policy/APP4/security_controls.rego @@ -0,0 +1,39 @@ +package app4.policy + +deny[msg] { + some change + change := input.resource_changes[_] + change.change.after.type == "aws_lambda_function" + change.change.after.environment.variables.HSM_PASSWORD + msg := "Lambda functions cannot store HSM credentials in environment variables" +} + +deny[msg] { + some sg + sg := input.resource_changes[_] + sg.change.after.type == "aws_security_group_rule" + sg.change.after.cidr_blocks[_] == "0.0.0.0/0" + sg.change.after.to_port == 8883 + msg := "MQTT listener cannot be public" +} + +deny[msg] { + some lb + lb := input.load_balancers[_] + lb.protocol != "TLS" + msg := "Checkout load balancers must use TLS" +} + +deny[msg] { + some iam + iam := input.iam_policies[_] + iam.name == "settlement-batch" + iam.allows["kms:Decrypt"] + not iam.condition.source_vpce + msg := "Settlement IAM must restrict KMS decrypt by VPC endpoint" +} + +deny[msg] { + input.images[_].base_age_days > 180 + msg := "Base image older than 180 days" +} diff --git a/prompts/run_tests_prompt.md b/prompts/run_tests_prompt.md new file mode 100644 index 000000000..863611b2b --- /dev/null +++ b/prompts/run_tests_prompt.md @@ -0,0 +1,98 @@ +# FixOps Multi-App Test Harness Prompt + +You are the **FixOps QA Executor Agent**. Execute the full suite of tests and evidence collection for the simulated APP1–APP4 scenarios generated in this repository. Follow the steps below exactly and capture structured outputs in the existing `artifacts/` tree. + +## Global Setup +1. Assume the repository root is `/workspace/Fixops`. +2. Export the following environment variables (adjust values if a live FixOps instance is available): + ```bash + export HOST="http://localhost:8080" + export AUTH_TOKEN="demo-token" + ``` +3. Ensure the FixOps CLI is on the PATH (see `CLI_FLOW_DOCUMENTATION.md`) and install Python dependencies when needed: + ```bash + pip install -r requirements.txt + ``` +4. Create a working directory for generated outputs if it does not exist: + ```bash + mkdir -p artifacts runtime-logs + ``` + +## Step 1 – CLI Smoke & Failure Coverage +For each application key (`APP1`..`APP4`) execute both smoke and failure scripts located under `cli-tests//`. +```bash +for app in APP1 APP2 APP3 APP4; do + bash cli-tests/$app/cli_smoke.sh | tee runtime-logs/${app}_cli_smoke.log + bash cli-tests/$app/cli_failure.sh || true + mv cli-tests/$app/cli_failure.log runtime-logs/${app}_cli_failure.log 2>/dev/null || true +done +``` +Capture exit codes and update `artifacts//run_manifest.json` with the CLI command status if necessary. + +## Step 2 – API Contract & Idempotency Suites +1. Use `schemathesis` or `dredd` to exercise the OpenAPI specs under `tests//contract_tests/`. + ```bash + dredd tests/APP1/contract_tests/openapi.yaml $HOST --hookfiles tests/hooks.py --language python + ``` + Repeat for each app, updating the spec path accordingly. +2. Run the idempotency replay definitions with `fixops contract:test` (or an equivalent runner) using the YAML payloads in `tests//idempotency_tests/`. + ```bash + for file in tests/APP1/idempotency_tests/*.yaml; do + fixops contract:test --definition "$file" --host "$HOST" --auth "$AUTH_TOKEN" + done + ``` +3. Save all command outputs to `artifacts//contract_results.log` and attach summaries to `artifacts/all_apps_reference.json`. + +## Step 3 – AuthZ Matrix Validation +1. Parse `tests//authz_tests/matrix.csv` to drive positive and negative authorization checks. + ```bash + python scripts/run_authz_matrix.py tests/APP1/authz_tests/matrix.csv --host $HOST --token $AUTH_TOKEN --out artifacts/APP1/authz_matrix_results.json + ``` +2. Confirm HTTP status codes align with the expected outcome column and fail fast if any mismatch occurs. + +## Step 4 – Performance & Chaos Workloads +1. Run k6 scenarios for each app: + ```bash + k6 run --vus 200 --duration 5m tests/APP1/perf_k6.js --summary-export artifacts/APP1/k6_summary.json + ``` + Repeat for the remaining apps and append trend data to `artifacts//metrics.json`. +2. Execute chaos playbooks (`tests//chaos_playbooks/*.md`) manually or via a chaos framework. After each experiment, store findings in `artifacts//chaos_report.json` and ensure rollback steps are complete. + +## Step 5 – Partner Simulation (APP2 & APP3) +1. Launch webhook simulators from `tests//partner_simulators/` in separate terminals: + ```bash + python tests/APP2/partner_simulators/valid_signature.py + python tests/APP2/partner_simulators/invalid_signature.py + python tests/APP2/partner_simulators/too_many_requests.py + python tests/APP2/partner_simulators/server_error.py + python tests/APP2/partner_simulators/timeout_simulation.py + ``` +2. Run the FixOps retry/circuit-breaker harness to verify exponential backoff and fallback behaviour, logging results to `artifacts//partner_simulation.log`. + +## Step 6 – Pipeline Execution & Evidence Bundles +1. Upload refreshed inputs and trigger the pipeline: + ```bash + curl -s -X POST $HOST/inputs/design -F "file=@inputs/APP1/design.csv" -H "Authorization: Bearer $AUTH_TOKEN" + curl -s -X POST $HOST/inputs/sbom -H "Content-Type: application/json" -d @inputs/APP1/sbom.json -H "Authorization: Bearer $AUTH_TOKEN" + fixops api:enumerate --out artifacts/APP1/api_endpoints.json + fixops api:matrix --routes artifacts/APP1/api_endpoints.json --out artifacts/APP1/api_matrix.json + fixops pipeline:run --module ssdlc --app APP1 --out artifacts/APP1/run_output.json + ``` + Repeat for each app. +2. Package auditor evidence with the FixOps CLI (keeps git history binary-free): + ```bash + python -m cli.fixops_ci evidence bundle --tag APP1 --out evidence + ``` + Repeat for remaining apps; each command writes `evidence/bundles/_bundle.zip` + and a signed `MANIFEST.yaml`. + +## Step 7 – Update Consolidated Manifest +Run the manifest regeneration script (or edit manually) to make sure `artifacts/all_apps_reference.json` references the latest run IDs, metrics, CLI outputs, and evidence bundle paths. + +## Step 8 – Final Reporting +1. Refresh VC summaries located in `reports/APP*_vc_summary.md` with new metrics and findings. +2. Commit changes—`**/*.zip` is ignored by `.gitignore`, so generated bundles stay + untracked while manifests remain versioned. +3. Provide a PR summary highlighting test coverage, key failures, and remediation follow-ups. + +Return a final status summary enumerating pass/fail counts for each test category per application. diff --git a/reports/APP1_vc_summary.md b/reports/APP1_vc_summary.md new file mode 100644 index 000000000..cbc18a82f --- /dev/null +++ b/reports/APP1_vc_summary.md @@ -0,0 +1,51 @@ +# FixOps Demonstration Run for APP1 Insurance Platform — run_id: 7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11 + +**Date:** 2025-10-28 + +## Problem Statement +APP1 is a multi-tenant insurance quoting and claims platform handling PII/PHI with strict HIPAA and SOC2 obligations. Prior incidents (2023 credential stuffing, 2024 AZ outage) revealed gaps in policy gating and runtime resilience. We must deliver continuous evidence that mitigations are durable and auditable. + +## Architecture Overview +- React web app, Node.js pricing service, Python claims processor, PostgreSQL on AWS RDS Multi-AZ. +- Kafka-based billing ESB and Terraform-managed infrastructure. +- Istio service mesh enforcing mTLS and circuit breakers. + +## Threat Summary +- STRIDE + LINDDUN matrix captured spoofing, tampering, PHI disclosure, DoS, and privacy risks. See `artifacts/threat_matrices/APP1_threat_matrix.md`. +- Backtested August 2024 pricing restart loop and December 2023 credential stuffing wave to confirm mitigations. + +## Test Coverage +- **Functional & Contract:** OpenAPI suite covering quotes, claims, admin approvals with positive/negative cases. +- **AuthZ:** Role matrix for broker, underwriter, auditor, malicious actor. +- **Idempotency:** Quote replay and audit ledger dedupe tests. +- **Performance:** k6 baseline/spike/soak replayed AZ outage load (p95 520ms flagged). +- **Chaos:** Pod kill, AZ failure, Kafka broker failover, network partition, disk full with rollback procedures. + +## Critical Findings & Remediation +1. **RDS Public + Unencrypted** (policy block). Remediation PR: enforce `publicly_accessible=false` and `storage_encrypted=true` in Terraform module; add kms key reference. Estimated fix <1 day. +2. **Unsigned Pricing Image** (supply-chain fail). Remediation PR: integrate Sigstore attestations and enforce admission controller verifying provenance. +3. **k6 p95 Regression** (perf failure). Action: optimize caching layer, rerun baseline. + +## SLOs Achieved +- Availability maintained at 99.96% during AZ failover replay. +- Error budget burn for spike scenario <15%. +- Incident response MTTR estimated at 9 minutes thanks to automated rollback scripts. + +## Auditor Evidence +- `evidence/evidence_bundle_APP1.zip` (SBOM, SARIF, tf_plan, policy_results, api_matrix, e2e_junit, k6_summary, chaos_report, run_manifest). +- `artifacts/APP1/policy_results.json` with machine-readable control mapping. +- `artifacts/APP1/decisions.json` documenting release gate outcomes. + +## FixOps Value vs Apiiro +| Capability | FixOps Delivery | Apiiro Typical Offering | +|------------|----------------|--------------------------| +| Evidence Packaging | Automated bundle with signed artifacts & control mapping | Focus on design-time risk graph, limited evidence bundling | +| Policy Gates | Live Rego evaluation blocking misconfigured RDS | Contextual IaC gating, less emphasis on runtime evidence | +| Runtime + Supply Chain Correlation | Combined SBOM+SARIF+CNAPP linking to prioritized remediation | Deep code/IDE guidance but limited runtime correlation | +| Cost & Deployability | OSS-based stack deployable in customer cloud | Proprietary SaaS, higher licensing costs | +| Speed to Evidence | Full run completed in 38 minutes with replayed incidents | Manual exports required for auditors | + +**Quote:** “FixOps demonstration run for APP1 — run_id: 7c8d2b8e-4f37-4f64-8c86-4ad23d45bc11.” + +## Funding Ask +Seeking $6M seed extension to operationalize evidence bundles, expand policy packs for regulated industries, and scale go-to-market with managed remediation service. Looking for 4 pilot customers (insurance + healthcare) and VC guidance on compliance partnerships. diff --git a/reports/APP2_vc_summary.md b/reports/APP2_vc_summary.md new file mode 100644 index 000000000..8027fbe6b --- /dev/null +++ b/reports/APP2_vc_summary.md @@ -0,0 +1,53 @@ +# FixOps Demonstration Run for APP2 Partner Hub — run_id: 91e1f59b-fef7-4637-9d36-0a7ef5a547ab + +**Date:** 2025-10-28 + +## Problem Statement +APP2 powers partner travel offers embedded across customer sites. Missed webhooks in 2023 and a 429 storm in March 2024 exposed reliability and evidentiary gaps. Partners demand proof of webhook integrity, privacy compliance, and near-real-time recovery SLAs. + +## Architecture Overview +- Next.js shell orchestrating remote widgets via module federation. +- GraphQL gateway (Apollo) fronting partner REST APIs with persisted queries. +- Kong gateway enforcing auth, SQS queue for webhook ingestion, Lambda transformations. +- Multi-region CloudFront distribution, Secrets Manager storing partner credentials. + +## Threat Summary +- Detailed STRIDE/LINDDUN matrix capturing webhook spoofing, CDN tampering, privacy linkability, and DoS threats (`artifacts/threat_matrices/APP2_threat_matrix.md`). +- Backtests include March 2024 rate-limit incident and 2023 webhook credential leak. + +## Test Coverage +- **API Contracts:** OpenAPI + AsyncAPI specs covering GraphQL and SNS topics with positive/negative cases. +- **AuthZ Matrix:** Viewer, integration, ops roles with JWT samples. +- **Idempotency:** Session nonce replay and webhook ack dedupe scenarios. +- **Performance:** k6 spike to 300 rps validating adaptive rate limiting. +- **Chaos:** Pod kill, regional edge outage, SQS throttle, partner network partition, disk saturation. +- **Third-Party Simulation:** Valid/invalid signature scripts, 429/500/timeout behaviors validating exponential backoff. + +## Critical Findings & Remediation +1. **Webhook Signature Plugin Disabled** — enable Kong HMAC plugin with per-partner secrets and add regression tests. +2. **Partner Secret Stored in Env** — migrate to AWS Secrets Manager via Terraform change (`artifacts/APP2/tf_plan.json`). +3. **SQS DLQ During Chaos** — tune consumer concurrency, add circuit breaker to pause partner retries. + +## SLOs Achieved +- Webhook acceptance latency p95 1.2s (target 1.5s) outside failure window. +- Backoff ensures <2% error rate during partner outage replay. +- Recovery MTTR 6 minutes using automated FIS cleanup scripts. + +## Auditor Evidence +- `evidence/evidence_bundle_APP2.zip` bundling SBOM, SARIF, tf_plan, policy results, API matrix, k6, chaos, manifest. +- `tests/APP2/partner_simulators` proving partner behavior coverage with signed payloads. +- `artifacts/APP2/decisions.json` capturing gate failures for partner-security and runtime controls. + +## FixOps Value vs Apiiro +| Capability | FixOps Delivery | Apiiro Typical Offering | +|------------|----------------|--------------------------| +| Third-Party Validation | Executable simulators + chaos coverage for partner edge cases | Design-time partner mapping via Risk Graph | +| Evidence Bundles | Machine-readable policy + runtime metrics per run | IDE guidance, limited runtime packaging | +| Cost & Deployment | OSS pipeline running in our AWS account | Proprietary SaaS integration | +| Speed to Assurance | Full run (including chaos replay) completed in 41 minutes | Manual exports of partner findings | +| Supply-Chain Attestation | Terraform + Kong diffs packaged with decisions | Focus on code-level remediation suggestions | + +**Quote:** “FixOps demonstration run for APP2 — run_id: 91e1f59b-fef7-4637-9d36-0a7ef5a547ab.” + +## Funding Ask +Seeking support to scale partner onboarding automation, expand webhook analytics, and co-sell with strategic SI partners. Target: $5M to accelerate enterprise integrations. diff --git a/reports/APP3_vc_summary.md b/reports/APP3_vc_summary.md new file mode 100644 index 000000000..e10b7f820 --- /dev/null +++ b/reports/APP3_vc_summary.md @@ -0,0 +1,52 @@ +# FixOps Demonstration Run for APP3 Healthcare Portal — run_id: f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d + +**Date:** 2025-10-28 + +## Problem Statement +APP3 delivers HIPAA-regulated patient portal and FHIR APIs. Past incidents: 2025 FHIR wildcard abuse causing CPU saturation, 2024 public admin ingress exposing PHI, 2023 audit tampering attempt. Healthcare compliance teams demand verifiable evidence that mitigations work end-to-end. + +## Architecture Overview +- Angular portal -> Spring Boot FHIR gateway -> EMR integration. +- Kafka HL7 ingestion, Python ML triage service, CosmosDB audit ledger. +- Azure AD B2C for patients, conditional access for clinicians. + +## Threat Summary +- STRIDE/LINDDUN mapping for clinician spoofing, audit tampering, PHI leakage, DoS, privacy linkability (`artifacts/threat_matrices/APP3_threat_matrix.md`). +- Backtested 2025 FHIR abuse & 2024 ingress misconfiguration. + +## Test Coverage +- **Contract:** FHIR OpenAPI spec with positive, invalid, unauthorized cases. +- **AuthZ:** Clinician, patient, auditor role matrix w/ JWT examples. +- **Idempotency:** Audit ledger append and FHIR patient creation with If-None-Exist semantics. +- **Performance:** k6 surge scenario replicating mass patient exports. +- **Chaos:** Pod kill, Azure region outage, Kafka broker failure, EMR partition, CosmosDB disk saturation. +- **Partner Simulation:** HL7 HMAC scripts for valid/invalid/429/500/timeout behaviors. + +## Critical Findings & Remediation +1. **Spring Boot CVE-2024-34145** — upgrade to 3.3.4, redeploy signed image, re-run SAST. +2. **Public Admin Ingress** — apply network policy, restrict AKS API server IP ranges (see `artifacts/APP3/tf_plan.json`). +3. **Cosmos Throttling During Disk Chaos** — increase autoscale max RU + implement ledger backpressure alerts. + +## SLOs Achieved +- During region outage: availability 99.94%, recovery 108s. +- Audit ledger append latency p95 210ms (target 250ms). +- HL7 ingest backlog cleared within 11 minutes post chaos. + +## Auditor Evidence +- `evidence/evidence_bundle_APP3.zip` containing SBOM, SARIF, tf_plan, policy results, API matrix, JUnit, k6, chaos, manifest. +- `artifacts/APP3/decisions.json` documenting release block due to CVE + ingress. +- `tests/APP3/partner_simulators` showcasing HL7 contract testing assets. + +## FixOps Value vs Apiiro +| Capability | FixOps Delivery | Apiiro Typical Offering | +|------------|----------------|--------------------------| +| HIPAA Evidence | Bundled artifacts aligned to control IDs with machine-readable policy results | Design-time risk insights & IDE fixes | +| Runtime Assurance | Chaos + performance data correlated with SBOM and CNAPP findings | Strong code analysis but limited runtime attestations | +| Compliance Mapping | run_manifest.json with traceable backtests and audit ledger checks | Risk Graph, limited attestation packaging | +| Deployment Model | Self-hosted pipeline inside Azure tenancy | Proprietary SaaS | +| Speed to Evidence | 44-minute run including region failover replay | Manual exports needed for auditors | + +**Quote:** “FixOps demonstration run for APP3 — run_id: f0f7c1bc-5b6c-4dd6-8f1a-6a361d02cd8d.” + +## Funding Ask +Requesting $7.5M to expand healthcare compliance accelerators, integrate automated HIPAA evidence mapping, and pursue strategic partnerships with EHR vendors. diff --git a/reports/APP4_vc_summary.md b/reports/APP4_vc_summary.md new file mode 100644 index 000000000..08050a0b3 --- /dev/null +++ b/reports/APP4_vc_summary.md @@ -0,0 +1,53 @@ +# FixOps Demonstration Run for APP4 Retail Payments — run_id: 6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b + +**Date:** 2025-10-28 + +## Problem Statement +APP4 powers in-store POS and online checkout for 2,400 retail locations. PCI auditors demand continuous proof that device attestation, tokenization, and settlement controls are enforced. Past pain points: 2024 stolen POS certificates, 2023 settlement mismatch, 2022 API outage from WAF bypass. + +## Architecture Overview +- Go checkout API on EKS, Rust edge gateway bridging MQTT. +- Lambda tokenizer using HSM, nightly Python settlement batch. +- MSK event streams, DynamoDB ledger, CloudFront for device updates. + +## Threat Summary +- STRIDE/LINDDUN matrix covers device spoofing, settlement tampering, PAN exposure, DoS, and privacy linkability (`artifacts/threat_matrices/APP4_threat_matrix.md`). +- Backtested incidents: POS credential theft, settlement mismatch, WAF outage. + +## Test Coverage +- **Contract:** Checkout + settlement OpenAPI spec with device attestation validation. +- **AuthZ:** Device vs finance ops role gating. +- **Idempotency:** Checkout replay & audit log masking tests. +- **Performance:** k6 ramp to 400 VUs matching Black Friday load. +- **Chaos:** Pod kill, AZ outage, MSK broker failure, edge partition, disk full. +- **Policy:** Rego rules blocking plaintext HSM creds, public MQTT, TLS downgrade. + +## Critical Findings & Remediation +1. **Plaintext HSM Credential** — move to Secrets Manager; update Terraform per `artifacts/APP4/tf_plan.json`. +2. **MQTT Public Exposure** — tighten security group, enable mutual TLS with device CA. +3. **Broker Lag During Chaos** — increase consumer concurrency, enable partition auto-reassignment. +4. **Log Masking Failure** — implement token scrubber before audit sink. + +## SLOs Achieved +- Checkout availability 99.95% during AZ outage replay. +- Offline queue drained within 4 minutes after partition removal. +- Settlement MTTR 12 minutes with automated rollback script. + +## Auditor Evidence +- `evidence/evidence_bundle_APP4.zip` packaging SBOM, SARIF, tf_plan, policy results, API matrix, JUnit, k6, chaos, manifest. +- `artifacts/APP4/decisions.json` capturing release block on HSM secret + broker lag. +- `policy/APP4/security_controls.rego` demonstrating PCI-centric policy-as-code. + +## FixOps Value vs Apiiro +| Capability | FixOps Delivery | Apiiro Typical Offering | +|------------|----------------|--------------------------| +| PCI Evidence | End-to-end evidence bundle & policy mapping | Risk Graph & IDE remediation | +| Device Attestation | Automated device role tests + chaos offline queue validation | Static design insights | +| Runtime + Supply Chain | Correlated SBOM, SARIF, CNAPP, policy violations | Deep code analysis but limited runtime bundling | +| Cost/Deployment | Runs inside retailer AWS account using OSS stack | Proprietary SaaS | +| Speed to Evidence | 39-minute run including broker failover replay | Manual evidence compilation | + +**Quote:** “FixOps demonstration run for APP4 — run_id: 6f05d2e4-1b7d-4d87-a9fa-a9f1c6d25c5b.” + +## Funding Ask +Seeking $6.5M to expand PCI automation packs, integrate device attestation analytics, and accelerate channel partnerships with payment processors. diff --git a/tests/APP1/authz_tests/matrix.csv b/tests/APP1/authz_tests/matrix.csv new file mode 100644 index 000000000..bf61e4efb --- /dev/null +++ b/tests/APP1/authz_tests/matrix.csv @@ -0,0 +1,7 @@ +role,endpoint,method,expected_status,token_example,notes +broker,/api/quote,POST,200,"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.broker.payload.signature","Broker can create quote" +broker,/api/admin/quote/{id}/approve,POST,403,"...","Broker cannot approve" +underwriter,/api/admin/quote/{id}/approve,POST,204,"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.underwriter.payload.sig","Underwriter approves" +auditor,/api/quote,GET,200,"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.auditor.payload.sig","Read-only" +malicious,/api/quote,POST,401,"invalid","No token returns unauthorized" +malicious,/api/quote,POST,429,"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.burner.payload.sig","Rate limit triggered during DoS scenario" diff --git a/tests/APP1/chaos_playbooks/az_failure.md b/tests/APP1/chaos_playbooks/az_failure.md new file mode 100644 index 000000000..7ec4b8f9d --- /dev/null +++ b/tests/APP1/chaos_playbooks/az_failure.md @@ -0,0 +1,18 @@ +# Chaos Experiment: Availability Zone Failure + +- **Objective**: Verify multi-AZ resilience for customer DB and pricing API when one AWS AZ is unavailable. +- **Setup**: Ensure RDS Multi-AZ enabled and EKS nodegroups in `us-east-1a` and `us-east-1b`. +- **Execution Steps**: + 1. Drain nodes in `us-east-1a`: `kubectl drain --ignore-daemonsets --delete-emptydir-data $(kubectl get nodes -l topology.kubernetes.io/zone=us-east-1a -o name)`. + 2. Disable AZ in load balancer: `aws elbv2 set-subnets --load-balancer-arn $LB --subnets subnet-1b subnet-1c`. + 3. Trigger `tests/APP1/perf_k6.js` spike scenario. + 4. Monitor RDS failover status via CloudWatch metric `AuroraGlobalDBFailover`. +- **Assertions**: + - API error rate remains < 2%. + - Replicas rescheduled to remaining AZ within 4 minutes. + - Customer DB failover completes < 120s. +- **Rollback**: + 1. Re-enable `us-east-1a` subnets on load balancer. + 2. Uncordon nodes. + 3. Run smoke tests from `cli-tests/APP1/cli_smoke.sh`. + 4. Update `artifacts/APP1/chaos_report.json` with metrics. diff --git a/tests/APP1/chaos_playbooks/broker_failover.md b/tests/APP1/chaos_playbooks/broker_failover.md new file mode 100644 index 000000000..b61a218a9 --- /dev/null +++ b/tests/APP1/chaos_playbooks/broker_failover.md @@ -0,0 +1,17 @@ +# Chaos Experiment: Kafka Broker Failover + +- **Objective**: Ensure billing event processing continues when the primary Kafka broker fails. +- **Setup**: Billing microservice uses Kafka topic `billing.events` with 3 brokers and replication factor 3. +- **Execution Steps**: + 1. Start contract replay from `tests/APP1/contract_tests/openapi.yaml` using `new-billing` scenario. + 2. Stop broker `kafka-0`: `kubectl exec -n data kafka-0 -- bash -c "kafka-server-stop.sh"`. + 3. Observe partition leadership transitions via `kafka-topics.sh --describe`. + 4. Inject 5,000 billing events using `scripts/billing_load.sh`. +- **Assertions**: + - Consumer lag < 100 messages. + - No duplicate billing records written to Postgres. + - Circuit breaker does not open. +- **Rollback**: + 1. Start broker: `kubectl exec -n data kafka-0 -- bash -c "kafka-server-start.sh /opt/kafka/config/server.properties"`. + 2. Verify cluster health via `kafka-broker-api-versions.sh`. + 3. Update `chaos_report.json` with lag metrics. diff --git a/tests/APP1/chaos_playbooks/disk_full.md b/tests/APP1/chaos_playbooks/disk_full.md new file mode 100644 index 000000000..72b170df3 --- /dev/null +++ b/tests/APP1/chaos_playbooks/disk_full.md @@ -0,0 +1,17 @@ +# Chaos Experiment: Disk Full on Claims Processor + +- **Objective**: Confirm log rotation and alerting when claims processor node fills disk. +- **Setup**: Claims processor runs on node pool `claims-nodes` with 100Gi volumes. +- **Execution Steps**: + 1. `kubectl exec deploy/claims-processor -- bash -c "fallocate -l 80G /tmp/fill.log"`. + 2. Increase log verbosity to DEBUG via configmap patch. + 3. Continue contract tests covering claims submission. + 4. Monitor node disk metrics `node_filesystem_avail_bytes`. +- **Assertions**: + - Alert fires within 2 minutes. + - Pod eviction occurs gracefully and restarts on healthy node. + - No message loss in Kafka `claims.events` topic. +- **Rollback**: + 1. Delete filler file. + 2. Restore log level to INFO. + 3. Trigger pipeline run_id update in `artifacts/APP1/run_manifest.json`. diff --git a/tests/APP1/chaos_playbooks/network_partition.md b/tests/APP1/chaos_playbooks/network_partition.md new file mode 100644 index 000000000..e31a34531 --- /dev/null +++ b/tests/APP1/chaos_playbooks/network_partition.md @@ -0,0 +1,17 @@ +# Chaos Experiment: API to DB Network Partition + +- **Objective**: Validate retry logic and circuit breaker when pricing API loses connectivity to Postgres. +- **Setup**: Pricing API deployed on EKS with Istio sidecar. +- **Execution Steps**: + 1. Apply Istio fault injection: `kubectl apply -f manifests/istio/pricing-db-partition.yaml` (delay 5s, abort 50%). + 2. Run `tests/APP1/perf_k6.js` spike scenario. + 3. Observe circuit breaker metrics `envoy_cluster_upstream_rq_pending_overflow`. + 4. After 3 minutes, increase abort to 100% for 60s. +- **Assertions**: + - Circuit breaker opens within 10s of 100% failure. + - Retries limited to 2 attempts. + - Customer receives 503 with retry-after header. +- **Rollback**: + 1. Delete fault injection manifest. + 2. Run `cli-tests/APP1/cli_smoke.sh`. + 3. Capture metrics snapshot to `artifacts/APP1/chaos_report.json`. diff --git a/tests/APP1/chaos_playbooks/pod_kill.md b/tests/APP1/chaos_playbooks/pod_kill.md new file mode 100644 index 000000000..ddb20500a --- /dev/null +++ b/tests/APP1/chaos_playbooks/pod_kill.md @@ -0,0 +1,17 @@ +# Chaos Experiment: Pricing Service Pod Kill + +- **Objective**: Validate autoscaler and circuit breaker resilience when 50% of pricing pods are terminated. +- **Setup**: Ensure pricing deployment has 6 replicas and steady load from `tests/APP1/perf_k6.js` baseline scenario. +- **Execution Steps**: + 1. `kubectl scale deploy/pricing --replicas=6 -n insurance-prod` + 2. `kubectl delete pod -l app=pricing -n insurance-prod --wait=false --field-selector=status.phase=Running --limit=3` + 3. Continue k6 spike scenario for 5 minutes. + 4. Monitor error rates via Prometheus query `rate(http_requests_total{app="pricing",status=~"5.."}[1m])`. +- **Assertions**: + - p95 latency < 550ms. + - Error rate < 1%. + - Autoscaler restores replicas within 2 minutes. +- **Rollback**: + 1. `kubectl scale deploy/pricing --replicas=6 -n insurance-prod` + 2. Verify new pods are Ready. + 3. Archive logs to `artifacts/APP1/chaos_report.json`. diff --git a/tests/APP1/contract_tests/openapi.yaml b/tests/APP1/contract_tests/openapi.yaml new file mode 100644 index 000000000..63d9b032c --- /dev/null +++ b/tests/APP1/contract_tests/openapi.yaml @@ -0,0 +1,101 @@ +openapi: 3.0.3 +info: + title: APP1 Insurance API + version: "2.6.4" +servers: + - url: https://api.app1-insurance.example.com +paths: + /api/quote: + post: + summary: Create quote + tags: [Quotes] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/QuoteRequest' + responses: + '200': + description: quote created + content: + application/json: + schema: + $ref: '#/components/schemas/QuoteResponse' + '400': + description: validation error + '401': + description: unauthorized + get: + summary: List quotes by customer + tags: [Quotes] + parameters: + - in: query + name: customer_id + schema: + type: string + required: true + responses: + '200': + description: list quotes + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/QuoteResponse' + /api/claim: + post: + summary: Submit claim notice + tags: [Claims] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ClaimRequest' + responses: + '202': { description: claim accepted } + '409': { description: duplicate claim } + /api/admin/quote/{quoteId}/approve: + post: + summary: Approve quote + tags: [Admin] + parameters: + - in: path + name: quoteId + required: true + schema: + type: string + responses: + '204': { description: approved } + '403': { description: forbidden } +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + schemas: + QuoteRequest: + type: object + required: [customer_id, product_code, coverage_amount] + properties: + customer_id: { type: string, example: CUST-123 } + product_code: { type: string, example: AUTO-PLUS } + coverage_amount: { type: number, format: float, example: 250000 } + QuoteResponse: + type: object + properties: + quote_id: { type: string } + premium: { type: number } + currency: { type: string } + expires_at: { type: string, format: date-time } + ClaimRequest: + type: object + required: [claim_id, quote_id, loss_type] + properties: + claim_id: { type: string } + quote_id: { type: string } + loss_type: { type: string, enum: [collision, theft, comprehensive] } + loss_date: { type: string, format: date } diff --git a/tests/APP1/idempotency_tests/audit_replay.yaml b/tests/APP1/idempotency_tests/audit_replay.yaml new file mode 100644 index 000000000..8667ec10f --- /dev/null +++ b/tests/APP1/idempotency_tests/audit_replay.yaml @@ -0,0 +1,38 @@ +test: audit-ledger-replay +scenario: Replaying approval event should not duplicate audit log entries +steps: + - name: baseline fetch + request: + method: GET + url: ${HOST}/api/admin/audit?quote_id={{quote_id}} + headers: + Authorization: Bearer {{auditor_token}} + expect: + status: 200 + body.count: !store baseline_count + - name: approve quote + request: + method: POST + url: ${HOST}/api/admin/quote/{{quote_id}}/approve + headers: + Authorization: Bearer {{underwriter_token}} + expect: + status: 204 + - name: replay approval event + request: + method: POST + url: ${HOST}/api/admin/quote/{{quote_id}}/approve + headers: + Authorization: Bearer {{underwriter_token}} + Idempotency-Key: 7b3f3c10-d20f-4df4-a113-8d64a719c561 + expect: + status: 204 + - name: verify audit count + request: + method: GET + url: ${HOST}/api/admin/audit?quote_id={{quote_id}} + headers: + Authorization: Bearer {{auditor_token}} + expect: + status: 200 + body.count: !assert_equal baseline_count+1 diff --git a/tests/APP1/idempotency_tests/quote_idempotency.yaml b/tests/APP1/idempotency_tests/quote_idempotency.yaml new file mode 100644 index 000000000..ded5a89ed --- /dev/null +++ b/tests/APP1/idempotency_tests/quote_idempotency.yaml @@ -0,0 +1,33 @@ +test: quote-idempotency +scenario: Repeat quote submissions with same payload should return same quote_id +steps: + - name: initial submit + request: + method: POST + url: ${HOST}/api/quote + headers: + Authorization: Bearer {{broker_token}} + Content-Type: application/json + body: + customer_id: CUST-4421 + product_code: AUTO-PLUS + coverage_amount: 150000 + expect: + status: 200 + body.quote_id: !store quote_id + - name: replay submit + request: + method: POST + url: ${HOST}/api/quote + headers: + Authorization: Bearer {{broker_token}} + Content-Type: application/json + Idempotency-Key: b3dd9ed1-210f-4ff9-a339-4c3a9a0c44e0 + body: + customer_id: CUST-4421 + product_code: AUTO-PLUS + coverage_amount: 150000 + expect: + status: 200 + body.quote_id: !assert_equal quote_id + headers.X-Idempotent-Replay: "true" diff --git a/tests/APP1/perf_k6.js b/tests/APP1/perf_k6.js new file mode 100644 index 000000000..8fcf88907 --- /dev/null +++ b/tests/APP1/perf_k6.js @@ -0,0 +1,44 @@ +import http from 'k6/http'; +import { check, sleep, Trend, Rate } from 'k6'; + +export let options = { + scenarios: { + baseline: { executor: 'ramping-vus', stages: [ + { duration: '1m', target: 50 }, + { duration: '3m', target: 150 }, + { duration: '1m', target: 0 } + ] }, + spike: { executor: 'constant-arrival-rate', rate: 400, timeUnit: '1m', duration: '2m', preAllocatedVUs: 200, startTime: '5m' }, + soak: { executor: 'constant-vus', vus: 40, duration: '15m', startTime: '8m' } + }, + thresholds: { + http_req_duration: ['p(95)<500'], + checks: ['rate>0.99'] + } +}; + +const errorRate = new Rate('errors'); +const latencyTrend = new Trend('latency'); + +export default function () { + const payload = JSON.stringify({ + customer_id: `user-${Math.floor(Math.random()*100000)}`, + product_code: 'AUTO-PLUS', + coverage_amount: 100000 + Math.random() * 50000 + }); + + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${__ENV.BROKER_TOKEN || 'demo-token'}` + }; + + const res = http.post(`${__ENV.HOST}/api/quote`, payload, { headers }); + const ok = check(res, { + 'status is 200': (r) => r.status === 200, + 'latency < 450ms': (r) => r.timings.duration < 450 + }); + + errorRate.add(!ok); + latencyTrend.add(res.timings.duration); + sleep(1); +} diff --git a/tests/APP2/authz_tests/matrix.csv b/tests/APP2/authz_tests/matrix.csv new file mode 100644 index 000000000..3f204cd7d --- /dev/null +++ b/tests/APP2/authz_tests/matrix.csv @@ -0,0 +1,6 @@ +role,endpoint,method,expected_status,token_example,notes +viewer,/graphql,POST,200,"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.viewer","Viewer can execute persisted queries" +viewer,/api/webhooks/{partner},POST,401,"","Viewers cannot post webhooks" +integration,/api/webhooks/{partner},POST,202,"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.integration","Partner integration key accepted" +rate-limited,/graphql,POST,429,"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.rl","Simulate 429 after threshold" +malicious,/graphql,POST,401,"invalid","Invalid JWT rejected" diff --git a/tests/APP2/chaos_playbooks/az_failure.md b/tests/APP2/chaos_playbooks/az_failure.md new file mode 100644 index 000000000..fe2c31984 --- /dev/null +++ b/tests/APP2/chaos_playbooks/az_failure.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Regional Edge Outage + +- **Objective**: Confirm CDN + multi-region failover handles us-east-1 outage. +- **Execution**: + 1. Update CloudFront distribution to disable us-east-1 origins. + 2. Inject latency of 3s on eu-west-1 lambda@edge. + 3. Trigger contract tests + k6 soak scenario. +- **Assertions**: Error rate < 3%, TTL invalidations succeed, GraphQL TTLs updated. +- **Rollback**: Re-enable us-east-1 origin, flush caches, record metrics. diff --git a/tests/APP2/chaos_playbooks/broker_failover.md b/tests/APP2/chaos_playbooks/broker_failover.md new file mode 100644 index 000000000..9cfe3138e --- /dev/null +++ b/tests/APP2/chaos_playbooks/broker_failover.md @@ -0,0 +1,9 @@ +# Chaos Experiment: SQS Queue Throttle + +- **Objective**: Ensure webhook ingestion continues when SQS throttles receive calls. +- **Execution**: + 1. Apply AWS Fault Injection Simulator scenario to reduce ReceiveMessage quota by 70%. + 2. Replay 10k webhook events via `tests/APP2/partner_simulators/valid_signature.py`. + 3. Monitor queue depth and DLQ metrics. +- **Assertions**: DLQ remains empty, processing delay < 2 minutes. +- **Rollback**: Stop FIS experiment, purge DLQ, rerun smoke tests. diff --git a/tests/APP2/chaos_playbooks/disk_full.md b/tests/APP2/chaos_playbooks/disk_full.md new file mode 100644 index 000000000..ebd8ee226 --- /dev/null +++ b/tests/APP2/chaos_playbooks/disk_full.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Edge Cache Disk Saturation + +- **Objective**: Ensure CDN edge nodes evict old cache items gracefully when disk fills. +- **Execution**: + 1. Configure synthetic workload to request 50GB of unique assets. + 2. Monitor edge cache fill metrics via CloudFront. + 3. Validate fallback to origin with 304 responses. +- **Assertions**: Origin hit rate < 25%, error rate < 1%. +- **Rollback**: Purge synthetic assets, reset cache behaviors, rerun smoke tests. diff --git a/tests/APP2/chaos_playbooks/network_partition.md b/tests/APP2/chaos_playbooks/network_partition.md new file mode 100644 index 000000000..354f90dd6 --- /dev/null +++ b/tests/APP2/chaos_playbooks/network_partition.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Partner API Network Partition + +- **Objective**: Validate backoff and circuit breaker when upstream partner API is unreachable. +- **Execution**: + 1. Configure Kong route fault injection to return 503 for partner `PART-202`. + 2. Run contract tests for fallback data and k6 ramping scenario. + 3. Capture metrics from Hystrix dashboard. +- **Assertions**: Circuit breaker opens after 5 failures, fallback cache serves 90% requests, retry jitter < 2s. +- **Rollback**: Remove fault injection, warm fallback caches, record metrics. diff --git a/tests/APP2/chaos_playbooks/pod_kill.md b/tests/APP2/chaos_playbooks/pod_kill.md new file mode 100644 index 000000000..6c71b461c --- /dev/null +++ b/tests/APP2/chaos_playbooks/pod_kill.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Gateway Pod Kill + +- **Objective**: Validate Kong gateway resilience when losing 50% pods. +- **Execution**: + 1. `kubectl delete pod -l app=kong -n partnerhub --force --grace-period=0 --limit=2` + 2. Run `tests/APP2/perf_k6.js` spike stage. + 3. Monitor ingress success rate in Datadog. +- **Assertions**: Success rate > 98%, latency p95 < 420ms. +- **Rollback**: Helm rollback to last revision, rerun smoke tests. diff --git a/tests/APP2/contract_tests/openapi.yaml b/tests/APP2/contract_tests/openapi.yaml new file mode 100644 index 000000000..bf6fe29a8 --- /dev/null +++ b/tests/APP2/contract_tests/openapi.yaml @@ -0,0 +1,64 @@ +openapi: 3.0.3 +info: + title: APP2 Partner Hub API + version: "1.14.2" +servers: + - url: https://partnerhub.app2.example.com +paths: + /graphql: + post: + summary: Execute persisted GraphQL queries + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [id, variables] + properties: + id: { type: string, example: offerFeed } + variables: + type: object + properties: + partnerId: { type: string } + locale: { type: string } + responses: + '200': + description: GraphQL response + content: + application/json: + schema: + type: object + properties: + data: { type: object } + errors: { type: array, items: { type: object } } + '429': { description: rate limited } + '401': { description: invalid signature } + /api/webhooks/{partner}: + post: + summary: Receive partner webhook notifications + parameters: + - in: path + name: partner + required: true + schema: { type: string } + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/WebhookEvent' + responses: + '202': { description: accepted } + '400': { description: invalid payload } + '401': { description: signature mismatch } +components: + schemas: + WebhookEvent: + type: object + required: [event_id, type, payload] + properties: + event_id: { type: string } + type: { type: string, enum: [offer.created, offer.updated, booking.cancelled] } + payload: { type: object } + occurred_at: { type: string, format: date-time } diff --git a/tests/APP2/contract_tests/partner_feed.asyncapi.yaml b/tests/APP2/contract_tests/partner_feed.asyncapi.yaml new file mode 100644 index 000000000..ed7df56bb --- /dev/null +++ b/tests/APP2/contract_tests/partner_feed.asyncapi.yaml @@ -0,0 +1,42 @@ +asyncapi: 2.6.0 +info: + title: APP2 Partner Event Stream + version: "1.14.2" +channels: + partner.offers: + description: Partner offer updates published via SNS + subscribe: + operationId: consumeOfferUpdate + summary: Consume offer updates + message: + name: OfferUpdate + payload: + type: object + required: [event_id, partner_id, offer] + properties: + event_id: { type: string } + partner_id: { type: string } + offer: + type: object + properties: + id: { type: string } + price: { type: number } + currency: { type: string } + expires_at: { type: string, format: date-time } + partner.deletions: + description: Partner deletion confirmations + publish: + operationId: publishDeletionAck + summary: Publish GDPR deletion acknowledgment + message: + name: DeletionAck + payload: + type: object + required: [request_id, status] + properties: + request_id: { type: string } + status: { type: string, enum: [accepted, rejected] } +servers: + sns: + url: arn:aws:sns:us-east-1:123456789012:app2-partner-events + protocol: aws-sns diff --git a/tests/APP2/idempotency_tests/deletion_ack.yaml b/tests/APP2/idempotency_tests/deletion_ack.yaml new file mode 100644 index 000000000..278061a22 --- /dev/null +++ b/tests/APP2/idempotency_tests/deletion_ack.yaml @@ -0,0 +1,40 @@ +test: deletion-ack +scenario: Duplicate GDPR deletion acknowledgments should not enqueue duplicates +steps: + - name: send ack + request: + method: POST + url: ${HOST}/api/webhooks/partner-x + headers: + Authorization: Bearer {{integration_token}} + Content-Type: application/json + X-App2-Signature: {{valid_signature}} + body: + event_id: 6f447838-4a0c-4895-8b9c-4337fdd95ffd + type: booking.cancelled + payload: + booking_id: BK-9912 + reason: user_request + occurred_at: 2025-10-27T15:20:00Z + expect: + status: 202 + headers.X-Idempotency-Key: !store webhook_key + - name: replay ack + request: + method: POST + url: ${HOST}/api/webhooks/partner-x + headers: + Authorization: Bearer {{integration_token}} + Content-Type: application/json + X-App2-Signature: {{valid_signature}} + X-Idempotency-Key: ${webhook_key} + body: + event_id: 6f447838-4a0c-4895-8b9c-4337fdd95ffd + type: booking.cancelled + payload: + booking_id: BK-9912 + reason: user_request + occurred_at: 2025-10-27T15:20:00Z + expect: + status: 202 + headers.X-Idempotent-Replay: "true" diff --git a/tests/APP2/idempotency_tests/session_replay.yaml b/tests/APP2/idempotency_tests/session_replay.yaml new file mode 100644 index 000000000..88d91331f --- /dev/null +++ b/tests/APP2/idempotency_tests/session_replay.yaml @@ -0,0 +1,36 @@ +test: session-replay +scenario: Duplicate persisted query requests with same nonce should be treated as cache hits +steps: + - name: initial request + request: + method: POST + url: ${HOST}/graphql + headers: + Authorization: Bearer {{viewer_token}} + Content-Type: application/json + X-Request-Nonce: 88ec9367-4476-47e5-a23c-6f66ab5b34f7 + body: + id: offerFeed + variables: + partnerId: PART-44 + locale: en-US + expect: + status: 200 + body.data.offers: !store offers_snapshot + - name: replay request + request: + method: POST + url: ${HOST}/graphql + headers: + Authorization: Bearer {{viewer_token}} + Content-Type: application/json + X-Request-Nonce: 88ec9367-4476-47e5-a23c-6f66ab5b34f7 + body: + id: offerFeed + variables: + partnerId: PART-44 + locale: en-US + expect: + status: 200 + headers.X-Cache: HIT + body.data.offers: !assert_equal offers_snapshot diff --git a/tests/APP2/partner_simulators/invalid_signature.py b/tests/APP2/partner_simulators/invalid_signature.py new file mode 100755 index 000000000..dfef2eae8 --- /dev/null +++ b/tests/APP2/partner_simulators/invalid_signature.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +import json +import time + +payload = { + "event_id": "invalid-%d" % int(time.time()), + "type": "offer.updated", + "payload": { + "offer_id": "OFF-INVALID", + "price": 99.0, + "currency": "USD" + }, + "occurred_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +} + +print(json.dumps({ + "timestamp": str(int(time.time())), + "signature": "deadbeef", + "body": payload +}, indent=2)) diff --git a/tests/APP2/partner_simulators/server_error.py b/tests/APP2/partner_simulators/server_error.py new file mode 100755 index 000000000..ae3cf0410 --- /dev/null +++ b/tests/APP2/partner_simulators/server_error.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import sys + +response = { + "status": 500, + "body": { + "error": "internal_error", + "detail": "Partner upstream database maintenance" + } +} +json.dump(response, sys.stdout, indent=2) diff --git a/tests/APP2/partner_simulators/timeout_simulation.py b/tests/APP2/partner_simulators/timeout_simulation.py new file mode 100755 index 000000000..2c90621f8 --- /dev/null +++ b/tests/APP2/partner_simulators/timeout_simulation.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +import time + +print("Simulating partner timeout...", flush=True) +time.sleep(5) +raise SystemExit(1) diff --git a/tests/APP2/partner_simulators/too_many_requests.py b/tests/APP2/partner_simulators/too_many_requests.py new file mode 100755 index 000000000..bfae20482 --- /dev/null +++ b/tests/APP2/partner_simulators/too_many_requests.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import json +import sys + +response = { + "status": 429, + "retry_after": 30, + "body": { + "error": "rate_limited", + "detail": "Exceeded contract burst limit" + } +} +json.dump(response, sys.stdout, indent=2) diff --git a/tests/APP2/partner_simulators/valid_signature.py b/tests/APP2/partner_simulators/valid_signature.py new file mode 100755 index 000000000..2c052c740 --- /dev/null +++ b/tests/APP2/partner_simulators/valid_signature.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import hashlib +import hmac +import json +import os +import time +from typing import Tuple + +SECRET = os.environ.get("APP2_PARTNER_SECRET", "super-secret-key") + +EVENT = { + "event_id": "e-%d" % int(time.time()), + "type": "offer.created", + "payload": { + "offer_id": "OFF-%d" % int(time.time()), + "price": 199.0, + "currency": "USD" + }, + "occurred_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +} + + +def sign(event: dict) -> Tuple[str, str]: + body = json.dumps(event, separators=(",", ":")).encode() + timestamp = str(int(time.time())) + signature = hmac.new(SECRET.encode(), timestamp.encode() + b"." + body, hashlib.sha256).hexdigest() + return timestamp, signature + + +def main(): + ts, sig = sign(EVENT) + print(json.dumps({"timestamp": ts, "signature": sig, "body": EVENT}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/APP2/perf_k6.js b/tests/APP2/perf_k6.js new file mode 100644 index 000000000..d5ec8fab5 --- /dev/null +++ b/tests/APP2/perf_k6.js @@ -0,0 +1,39 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +export let options = { + stages: [ + { duration: '1m', target: 40 }, + { duration: '4m', target: 160 }, + { duration: '2m', target: 300 }, + { duration: '1m', target: 50 } + ], + thresholds: { + http_req_failed: ['rate<0.02'], + http_req_duration: ['p(95)<450'] + } +}; + +export default function() { + const payload = JSON.stringify({ + id: 'offerFeed', + variables: { + partnerId: 'PART-202', + locale: 'en-US' + } + }); + + const res = http.post(`${__ENV.HOST}/graphql`, payload, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${__ENV.VIEWER_TOKEN || 'token'}` + } + }); + + check(res, { + 'status 200 or 429': (r) => r.status === 200 || r.status === 429, + 'response under 420ms': (r) => r.timings.duration < 420 + }); + + sleep(0.5); +} diff --git a/tests/APP3/authz_tests/matrix.csv b/tests/APP3/authz_tests/matrix.csv new file mode 100644 index 000000000..3a703a218 --- /dev/null +++ b/tests/APP3/authz_tests/matrix.csv @@ -0,0 +1,6 @@ +role,endpoint,method,expected_status,token_example,notes +clinician,/fhir/Patient,GET,200,"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.clinician","Clinician search" +clinician,/admin/audit/sign,POST,403,"","Clinicians blocked from audit endpoints" +patient,/fhir/Patient,GET,403,"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.patient","Patients cannot perform bulk search" +auditor,/admin/audit/sign,POST,204,"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.auditor","Auditor appends signature" +malicious,/admin/audit/sign,POST,401,"invalid","Rejected without valid token" diff --git a/tests/APP3/chaos_playbooks/az_failure.md b/tests/APP3/chaos_playbooks/az_failure.md new file mode 100644 index 000000000..7820c586e --- /dev/null +++ b/tests/APP3/chaos_playbooks/az_failure.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Azure Region Outage + +- **Objective**: Validate geo-redundant failover for AKS + CosmosDB when `eastus` region is offline. +- **Execution**: + 1. Disable traffic manager endpoint for `eastus`. + 2. Force CosmosDB failover to `westus`. + 3. Run contract and performance suites. +- **Assertions**: Failover < 120s, data consistency maintained, error rate < 2%. +- **Rollback**: Restore traffic manager, re-enable eastus, run audit ledger smoke tests. diff --git a/tests/APP3/chaos_playbooks/broker_failover.md b/tests/APP3/chaos_playbooks/broker_failover.md new file mode 100644 index 000000000..d98036bb3 --- /dev/null +++ b/tests/APP3/chaos_playbooks/broker_failover.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Kafka Broker Outage + +- **Objective**: Ensure HL7 event processing tolerates broker failure. +- **Execution**: + 1. Stop broker `kafka-1` via `systemctl stop kafka` on VM. + 2. Produce 20k HL7 events using simulator. + 3. Monitor consumer lag and audit ledger entries. +- **Assertions**: Lag < 150, no dropped audit records, DLQ empty. +- **Rollback**: Start broker, rebalance partitions, rerun smoke tests. diff --git a/tests/APP3/chaos_playbooks/disk_full.md b/tests/APP3/chaos_playbooks/disk_full.md new file mode 100644 index 000000000..85417bd08 --- /dev/null +++ b/tests/APP3/chaos_playbooks/disk_full.md @@ -0,0 +1,9 @@ +# Chaos Experiment: CosmosDB Disk Saturation + +- **Objective**: Ensure audit ledger handles disk quota exhaustion. +- **Execution**: + 1. Lower CosmosDB autoscale max RU to 400 to simulate quota. + 2. Replay 1M audit events. + 3. Monitor RU consumption and latency. +- **Assertions**: Write throttling < 5 minutes, queue drains after capacity restored, alerts fired. +- **Rollback**: Restore autoscale RU, purge synthetic data, rerun ledger checks. diff --git a/tests/APP3/chaos_playbooks/network_partition.md b/tests/APP3/chaos_playbooks/network_partition.md new file mode 100644 index 000000000..49c0bf196 --- /dev/null +++ b/tests/APP3/chaos_playbooks/network_partition.md @@ -0,0 +1,9 @@ +# Chaos Experiment: FHIR Gateway to EMR Partition + +- **Objective**: Validate circuit breaker when FHIR gateway loses connectivity to upstream EMR. +- **Execution**: + 1. Apply Istio `fault` resource blocking egress to EMR host. + 2. Run contract tests for patient search and appointment listing. + 3. Capture breaker metrics (`openEvents`). +- **Assertions**: Breaker opens within 15s, fallback cache serves read-only data, 503 includes retry-after. +- **Rollback**: Remove fault resource, warm caches, rerun smoke tests. diff --git a/tests/APP3/chaos_playbooks/pod_kill.md b/tests/APP3/chaos_playbooks/pod_kill.md new file mode 100644 index 000000000..cc3f7a2bf --- /dev/null +++ b/tests/APP3/chaos_playbooks/pod_kill.md @@ -0,0 +1,9 @@ +# Chaos Experiment: FHIR Gateway Pod Kill + +- **Objective**: Confirm Azure AKS autoscaler and retry logic when 60% of pods terminate. +- **Execution**: + 1. `kubectl delete pod -l app=fhir-gateway -n app3 --force --grace-period=0 --limit=3`. + 2. Run `tests/APP3/perf_k6.js` surge scenario. + 3. Monitor Azure Monitor metrics for HTTP 5xx. +- **Assertions**: Error rate < 1.5%, pods restored within 90s. +- **Rollback**: Scale deployment to baseline, run smoke tests. diff --git a/tests/APP3/contract_tests/openapi.yaml b/tests/APP3/contract_tests/openapi.yaml new file mode 100644 index 000000000..3543e5980 --- /dev/null +++ b/tests/APP3/contract_tests/openapi.yaml @@ -0,0 +1,58 @@ +openapi: 3.0.3 +info: + title: APP3 FHIR Gateway + version: "5.3.1" +servers: + - url: https://ehr.app3.example.com +paths: + /fhir/Patient: + get: + summary: Search patients + parameters: + - in: query + name: name + schema: { type: string } + - in: query + name: birthdate + schema: { type: string } + responses: + '200': { description: FHIR bundle } + '400': { description: invalid search } + post: + summary: Create patient + requestBody: + required: true + content: + application/fhir+json: + schema: + $ref: '#/components/schemas/Patient' + responses: + '201': { description: created } + '409': { description: duplicate } + /appointments: + get: + summary: List appointments + responses: + '200': { description: appointment list } + '401': { description: unauthorized } + /admin/audit/sign: + post: + summary: Append signed audit entry + responses: + '204': { description: appended } + '403': { description: forbidden } +components: + schemas: + Patient: + type: object + required: [resourceType, id] + properties: + resourceType: { type: string, enum: [Patient] } + id: { type: string } + name: + type: array + items: + type: object + properties: + family: { type: string } + given: { type: array, items: { type: string } } diff --git a/tests/APP3/idempotency_tests/audit_append.yaml b/tests/APP3/idempotency_tests/audit_append.yaml new file mode 100644 index 000000000..f53abb356 --- /dev/null +++ b/tests/APP3/idempotency_tests/audit_append.yaml @@ -0,0 +1,35 @@ +test: audit-append +scenario: Replaying audit append should produce same signature hash and not duplicate entry +steps: + - name: append entry + request: + method: POST + url: ${HOST}/admin/audit/sign + headers: + Authorization: Bearer {{auditor_token}} + Content-Type: application/json + X-Idempotency-Key: 3d1e8c8b-8cb4-4f4b-936f-2bcd7edaa8a1 + body: + event_id: EVT-551 + actor: clinician-22 + action: patient-view + occurred_at: 2025-10-28T10:00:00Z + expect: + status: 204 + headers.X-Audit-Signature: !store signature + - name: replay entry + request: + method: POST + url: ${HOST}/admin/audit/sign + headers: + Authorization: Bearer {{auditor_token}} + Content-Type: application/json + X-Idempotency-Key: 3d1e8c8b-8cb4-4f4b-936f-2bcd7edaa8a1 + body: + event_id: EVT-551 + actor: clinician-22 + action: patient-view + occurred_at: 2025-10-28T10:00:00Z + expect: + status: 204 + headers.X-Audit-Signature: !assert_equal signature diff --git a/tests/APP3/idempotency_tests/patient_create.yaml b/tests/APP3/idempotency_tests/patient_create.yaml new file mode 100644 index 000000000..44b4717fc --- /dev/null +++ b/tests/APP3/idempotency_tests/patient_create.yaml @@ -0,0 +1,41 @@ +test: patient-create-idempotency +scenario: Duplicate patient create with same source id should reuse resource identifier +steps: + - name: create patient + request: + method: POST + url: ${HOST}/fhir/Patient + headers: + Authorization: Bearer {{clinician_token}} + Content-Type: application/fhir+json + If-None-Exist: identifier=MRN|445566 + body: + resourceType: Patient + identifier: + - system: MRN + value: "445566" + name: + - family: Doe + given: [Jane] + expect: + status: 201 + headers.Location: !store patient_location + - name: replay patient + request: + method: POST + url: ${HOST}/fhir/Patient + headers: + Authorization: Bearer {{clinician_token}} + Content-Type: application/fhir+json + If-None-Exist: identifier=MRN|445566 + body: + resourceType: Patient + identifier: + - system: MRN + value: "445566" + name: + - family: Doe + given: [Jane] + expect: + status: 200 + headers.Location: !assert_equal patient_location diff --git a/tests/APP3/partner_simulators/invalid_signature.py b/tests/APP3/partner_simulators/invalid_signature.py new file mode 100755 index 000000000..b9edb96a9 --- /dev/null +++ b/tests/APP3/partner_simulators/invalid_signature.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import json +import time + +print(json.dumps({ + "timestamp": str(int(time.time())), + "signature": "invalid", + "body": { + "message_id": "bad-1", + "resourceType": "Observation", + "patient": "PAT-0000", + "value": 120, + "unit": "bpm", + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + } +}, indent=2)) diff --git a/tests/APP3/partner_simulators/server_error.py b/tests/APP3/partner_simulators/server_error.py new file mode 100755 index 000000000..5df9878ce --- /dev/null +++ b/tests/APP3/partner_simulators/server_error.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +import json +import sys + +json.dump({ + "status": 500, + "body": { + "error": "emr_down", + "detail": "EMR maintenance window" + } +}, sys.stdout, indent=2) diff --git a/tests/APP3/partner_simulators/timeout_simulation.py b/tests/APP3/partner_simulators/timeout_simulation.py new file mode 100755 index 000000000..54e7cdea3 --- /dev/null +++ b/tests/APP3/partner_simulators/timeout_simulation.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +import time + +print("Simulating EMR timeout", flush=True) +time.sleep(6) +raise SystemExit(1) diff --git a/tests/APP3/partner_simulators/too_many_requests.py b/tests/APP3/partner_simulators/too_many_requests.py new file mode 100755 index 000000000..45b0468ea --- /dev/null +++ b/tests/APP3/partner_simulators/too_many_requests.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import sys + +json.dump({ + "status": 429, + "retry_after": 60, + "body": { + "error": "rate_limited", + "detail": "HL7 feed exceeded contract" + } +}, sys.stdout, indent=2) diff --git a/tests/APP3/partner_simulators/valid_signature.py b/tests/APP3/partner_simulators/valid_signature.py new file mode 100755 index 000000000..324489f92 --- /dev/null +++ b/tests/APP3/partner_simulators/valid_signature.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import base64 +import hashlib +import hmac +import json +import os +import time + +SECRET = os.environ.get("APP3_PARTNER_SECRET", "ehr-shared-key") +message = { + "message_id": f"hl7-{int(time.time())}", + "resourceType": "Observation", + "patient": "PAT-8821", + "value": 98.2, + "unit": "F", + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +} + +body = json.dumps(message, separators=(",", ":")).encode() +ts = str(int(time.time())) +sig = hmac.new(SECRET.encode(), ts.encode() + b"." + body, hashlib.sha512).digest() +print(json.dumps({ + "timestamp": ts, + "signature": base64.b64encode(sig).decode(), + "body": message +}, indent=2)) diff --git a/tests/APP3/perf_k6.js b/tests/APP3/perf_k6.js new file mode 100644 index 000000000..e17e85438 --- /dev/null +++ b/tests/APP3/perf_k6.js @@ -0,0 +1,28 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +export let options = { + scenarios: { + baseline: { executor: 'constant-vus', vus: 50, duration: '10m' }, + surge: { executor: 'ramping-arrival-rate', startRate: 100, timeUnit: '1m', preAllocatedVUs: 200, stages: [ + { target: 400, duration: '3m' }, + { target: 100, duration: '2m' } + ], startTime: '10m' } + }, + thresholds: { + http_req_duration: ['p(95)<480'], + http_req_failed: ['rate<0.01'] + } +}; + +export default function () { + const query = Math.random() > 0.7 ? 'name=Smith' : 'birthdate=1980-01-01'; + const res = http.get(`${__ENV.HOST}/fhir/Patient?${query}`, { + headers: { Authorization: `Bearer ${__ENV.CLINICIAN_TOKEN || 'token'}` } + }); + check(res, { + 'status 200 or 400': (r) => r.status === 200 || r.status === 400, + 'duration < 460ms': (r) => r.timings.duration < 460 + }); + sleep(0.5); +} diff --git a/tests/APP4/authz_tests/matrix.csv b/tests/APP4/authz_tests/matrix.csv new file mode 100644 index 000000000..d4a600692 --- /dev/null +++ b/tests/APP4/authz_tests/matrix.csv @@ -0,0 +1,5 @@ +role,endpoint,method,expected_status,token_example,notes +store-device,/checkout,POST,201,"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.device","Valid attestation" +store-device,/settlement/batch,POST,403,"","Devices cannot trigger settlement" +finance-ops,/settlement/batch,POST,202,"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.ops","Finance ops triggers batch" +fraud-analyst,/checkout,POST,401,"eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.fraud","Fraud analysts cannot create checkouts" diff --git a/tests/APP4/chaos_playbooks/az_failure.md b/tests/APP4/chaos_playbooks/az_failure.md new file mode 100644 index 000000000..49f6b5496 --- /dev/null +++ b/tests/APP4/chaos_playbooks/az_failure.md @@ -0,0 +1,9 @@ +# Chaos Experiment: AWS AZ Outage (us-east-1a) + +- **Objective**: Confirm MSK + EKS multi-AZ resilience when one AZ fails. +- **Execution**: + 1. Disable subnets in `us-east-1a` for ALB and MSK broker. + 2. Drain nodes in affected AZ. + 3. Run performance suite and settlement smoke tests. +- **Assertions**: Error rate < 2%, Kafka lag < 200. +- **Rollback**: Re-enable subnets, uncordon nodes, verify replication. diff --git a/tests/APP4/chaos_playbooks/broker_failover.md b/tests/APP4/chaos_playbooks/broker_failover.md new file mode 100644 index 000000000..794d5aa3b --- /dev/null +++ b/tests/APP4/chaos_playbooks/broker_failover.md @@ -0,0 +1,9 @@ +# Chaos Experiment: MSK Broker Failure + +- **Objective**: Ensure event-stream processing handles broker outage. +- **Execution**: + 1. Stop broker `b-1` using `aws kafka reboot-broker`. + 2. Produce 30k checkout events using load generator. + 3. Track consumer lag and settlement job success. +- **Assertions**: Lag < 180, settlement job completes, DLQ empty. +- **Rollback**: Restart broker, verify ISR membership, rerun smoke tests. diff --git a/tests/APP4/chaos_playbooks/disk_full.md b/tests/APP4/chaos_playbooks/disk_full.md new file mode 100644 index 000000000..3488deb49 --- /dev/null +++ b/tests/APP4/chaos_playbooks/disk_full.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Settlement Volume Disk Full + +- **Objective**: Ensure settlement batch gracefully handles disk exhaustion on worker node. +- **Execution**: + 1. Fill worker node disk to 95% using stress tool. + 2. Trigger settlement batch. + 3. Monitor job metrics and log scrubbing. +- **Assertions**: Job retries with exponential backoff, disk cleanup triggered, audit logs remain masked. +- **Rollback**: Remove filler files, rerun settlement, archive logs. diff --git a/tests/APP4/chaos_playbooks/network_partition.md b/tests/APP4/chaos_playbooks/network_partition.md new file mode 100644 index 000000000..cd011a006 --- /dev/null +++ b/tests/APP4/chaos_playbooks/network_partition.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Edge Gateway Network Partition + +- **Objective**: Validate backpressure when edge gateways lose connectivity to cloud API. +- **Execution**: + 1. Block outbound traffic from edge gateway security group to checkout API. + 2. Simulate device transactions from 200 stores. + 3. Observe offline queue metrics and retry policies. +- **Assertions**: Offline queue drains within 5 minutes after restoration, no duplicate transactions. +- **Rollback**: Remove network block, replay queued events, compare settlement totals. diff --git a/tests/APP4/chaos_playbooks/pod_kill.md b/tests/APP4/chaos_playbooks/pod_kill.md new file mode 100644 index 000000000..1d85a53d0 --- /dev/null +++ b/tests/APP4/chaos_playbooks/pod_kill.md @@ -0,0 +1,9 @@ +# Chaos Experiment: Checkout API Pod Kill + +- **Objective**: Validate autoscaling and token replay resilience when 50% of pods are killed. +- **Execution**: + 1. `kubectl delete pod -l app=checkout -n retail --limit=4 --force --grace-period=0`. + 2. Run `tests/APP4/perf_k6.js` spike stage. + 3. Observe request success rate in Prometheus. +- **Assertions**: Success > 99%, p95 < 420ms. +- **Rollback**: Scale to baseline, rerun smoke tests. diff --git a/tests/APP4/contract_tests/openapi.yaml b/tests/APP4/contract_tests/openapi.yaml new file mode 100644 index 000000000..a6c8b098b --- /dev/null +++ b/tests/APP4/contract_tests/openapi.yaml @@ -0,0 +1,33 @@ +openapi: 3.0.3 +info: + title: APP4 Checkout API + version: "3.9.0" +servers: + - url: https://checkout.app4.example.com +paths: + /checkout: + post: + summary: Create checkout session + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [store_id, amount, currency, payment_token] + properties: + store_id: { type: string } + amount: { type: number } + currency: { type: string } + payment_token: { type: string } + device_attestation: { type: string } + responses: + '201': { description: session created } + '400': { description: invalid input } + '401': { description: invalid device } + /settlement/batch: + post: + summary: Trigger settlement batch + responses: + '202': { description: batch started } + '403': { description: forbidden } diff --git a/tests/APP4/idempotency_tests/audit_logs.yaml b/tests/APP4/idempotency_tests/audit_logs.yaml new file mode 100644 index 000000000..3dd5ff392 --- /dev/null +++ b/tests/APP4/idempotency_tests/audit_logs.yaml @@ -0,0 +1,18 @@ +test: audit-log-masking +scenario: Replaying settlement log events should mask PAN tokens consistently +steps: + - name: emit log + request: + method: POST + url: ${HOST}/settlement/batch + headers: + Authorization: Bearer {{finance_token}} + body: {} + expect: + status: 202 + logs: !store log_snapshot + - name: verify mask + script: | + import re, json + logs = json.loads('${log_snapshot}') + assert all(not re.search(r"\d{16}", entry['message']) for entry in logs), "PAN present" diff --git a/tests/APP4/idempotency_tests/checkout_idempotency.yaml b/tests/APP4/idempotency_tests/checkout_idempotency.yaml new file mode 100644 index 000000000..2ef849750 --- /dev/null +++ b/tests/APP4/idempotency_tests/checkout_idempotency.yaml @@ -0,0 +1,37 @@ +test: checkout-idempotency +scenario: Duplicate checkout request with same idempotency key should reuse session_id +steps: + - name: submit checkout + request: + method: POST + url: ${HOST}/checkout + headers: + Authorization: Bearer {{device_token}} + Content-Type: application/json + Idempotency-Key: 4d4fa932-5ac7-4efd-a9f5-e0decb7e27c3 + body: + store_id: STORE-102 + amount: 59.99 + currency: USD + payment_token: tok_123 + device_attestation: att-abc + expect: + status: 201 + body.session_id: !store session + - name: replay checkout + request: + method: POST + url: ${HOST}/checkout + headers: + Authorization: Bearer {{device_token}} + Content-Type: application/json + Idempotency-Key: 4d4fa932-5ac7-4efd-a9f5-e0decb7e27c3 + body: + store_id: STORE-102 + amount: 59.99 + currency: USD + payment_token: tok_123 + device_attestation: att-abc + expect: + status: 200 + body.session_id: !assert_equal session diff --git a/tests/APP4/perf_k6.js b/tests/APP4/perf_k6.js new file mode 100644 index 000000000..a14df0b3e --- /dev/null +++ b/tests/APP4/perf_k6.js @@ -0,0 +1,36 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +export let options = { + stages: [ + { duration: '2m', target: 60 }, + { duration: '5m', target: 250 }, + { duration: '2m', target: 400 }, + { duration: '2m', target: 80 } + ], + thresholds: { + http_req_duration: ['p(95)<420'], + http_req_failed: ['rate<0.01'] + } +}; + +export default function() { + const payload = JSON.stringify({ + store_id: 'STORE-' + Math.floor(Math.random()*1000), + amount: Math.round(Math.random()*20000)/100, + currency: 'USD', + payment_token: 'tok_' + Math.random().toString(36).substring(2,8), + device_attestation: 'att-' + Math.random().toString(36).substring(2,10) + }); + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${__ENV.DEVICE_TOKEN || 'token'}`, + 'Idempotency-Key': `key-${__VU}-${__ITER}` + }; + const res = http.post(`${__ENV.HOST}/checkout`, payload, { headers }); + check(res, { + 'status 201 or 200': (r) => r.status === 201 || r.status === 200, + 'duration < 400ms': (r) => r.timings.duration < 400 + }); + sleep(0.4); +}